From a73d983b0534444496e8b7bd29d1460dcb62a1f1 Mon Sep 17 00:00:00 2001 From: radiken Date: Tue, 10 Nov 2026 21:14:52 +0000 Subject: [PATCH 1/2] Read a run's metrics back from the DST dashboard when there is no local scrape run_metrics returns /metrics if the run was scraped there. Otherwise it finds the run's experiment on the dashboard and writes the stored rows to the same folder with the scraper's own CSV writer. fetch_run does this for run folders from the command line. Co-Authored-By: Claude Opus 5.5 --- src/analysis/fetch_run.py | 34 ++++++++ src/analysis/post_run/fetch.py | 83 +++++++++++++++++++ src/analysis/post_run/tests/test_fetch.py | 98 +++++++++++++++++++++++ 3 files changed, 215 insertions(+) create mode 100644 src/analysis/fetch_run.py create mode 100644 src/analysis/post_run/fetch.py create mode 100644 src/analysis/post_run/tests/test_fetch.py diff --git a/src/analysis/fetch_run.py b/src/analysis/fetch_run.py new file mode 100644 index 00000000..fef5d694 --- /dev/null +++ b/src/analysis/fetch_run.py @@ -0,0 +1,34 @@ +"""Put published runs' metrics in their run folders, unless they were scraped there.""" + +import argparse +import logging +import sys +from pathlib import Path + +from src.analysis.post_run.fetch import run_metrics +from src.analysis.utils.log_utils import init_logger + +logger = logging.getLogger(__name__) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("runs", nargs="+", help="Run folders, named as the run on the dashboard") + args = parser.parse_args() + init_logger(logging.getLogger(), verbosity=1) + + failed = [] + for arg in args.runs: + run = Path(arg).resolve() + try: + print(run_metrics(run)) + except Exception: + logger.exception(f"Fetching `{run.name}` failed") + failed.append(run.name) + + if failed: + sys.exit(f"Not fetched: {failed}") + + +if __name__ == "__main__": + main() diff --git a/src/analysis/post_run/fetch.py b/src/analysis/post_run/fetch.py new file mode 100644 index 00000000..df1d1e14 --- /dev/null +++ b/src/analysis/post_run/fetch.py @@ -0,0 +1,83 @@ +"""Read a published run's metrics back from the DST dashboard into the local scrape layout.""" + +import logging +import shutil +from datetime import datetime, timezone +from pathlib import Path +from typing import List + +import requests + +from src.analysis.data.data_request_handler import DataRequestHandler +from src.analysis.post_run.metrics import run_scrape_config +from src.analysis.post_run.publish import DASHBOARD_URL, TIMEOUT_S + +logger = logging.getLogger(__name__) + + +def run_metrics(run: Path, url: str = DASHBOARD_URL) -> Path: + """The run's metric dump: `/metrics` as scraped, else fetched there from the dashboard.""" + dump = run / "metrics" + if any(dump.glob("*/*.csv")): + return dump + if dump.exists(): + raise FileExistsError(f"`{dump}` holds no .csv files; rename or move it to fetch the run") + return fetch_run_metrics(run.name, dump, url) + + +def find_experiment(run_name: str, url: str = DASHBOARD_URL) -> dict: + response = requests.get(f"{url}/experiments", timeout=TIMEOUT_S) + response.raise_for_status() + found = [e for e in response.json() if (e.get("metadata") or {}).get("run") == run_name] + if len(found) != 1: + titles = [e["title"] for e in found] + raise LookupError(f"Expected one dashboard experiment for `{run_name}`, found {titles}") + return found[0] + + +def _utc_seconds(timestamp: str) -> float: + dt = datetime.fromisoformat(timestamp) + return (dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)).timestamp() + + +def _query_response(rows: List[dict]) -> dict: + """The stored rows as the VM query_range response they came from.""" + series = {} + for row in rows: + labels = {k: v for k, v in row.items() if k not in ("timestamp", "value")} + key = tuple(sorted(labels.items())) + series.setdefault(key, (labels, []))[1].append( + [_utc_seconds(row["timestamp"]), row["value"]] + ) + return {"data": {"result": [{"metric": m, "values": v} for m, v in series.values()]}} + + +def fetch_run_metrics(run_name: str, dump: Path, url: str = DASHBOARD_URL) -> Path: + """Write the run's stored datasets to `dump` with the scraper's own CSV writer.""" + experiment = find_experiment(run_name, url) + response = requests.get(f"{url}/experiments/{experiment['id']}/datasets", timeout=TIMEOUT_S) + response.raise_for_status() + stored = {d["name"]: d["data"] for d in response.json()["datasets"]} + + partial = dump.with_name(f"{dump.name}.partial") + shutil.rmtree(partial, ignore_errors=True) + config = run_scrape_config(experiment["metadata"], partial, run_name) + unknown = set(stored) - {m.folder_name.strip("/") for m in config.metrics_to_scrape} + if unknown: + logger.warning(f"`{run_name}` has stored datasets no current metric matches: {unknown}") + for metric in config.metrics_to_scrape: + rows = stored.get(metric.folder_name.strip("/")) + if not rows: + logger.warning(f"`{run_name}` has no stored {metric.name} data") + continue + handler = DataRequestHandler(_query_response(rows)) + handler.create_dataframe_from_request( + metric.extract_field, metric.container, metric.metrics_path + ) + handler.dump_dataframe((partial / metric.folder_name / f"{run_name}.csv").as_posix()) + + if not partial.exists(): + raise LookupError(f"`{run_name}` has no stored metrics on the dashboard") + partial.rename(dump) + logger.info(f"Fetched `{run_name}` from the dashboard ({experiment['id']}) into `{dump}`") + return dump diff --git a/src/analysis/post_run/tests/test_fetch.py b/src/analysis/post_run/tests/test_fetch.py new file mode 100644 index 00000000..1044ee7e --- /dev/null +++ b/src/analysis/post_run/tests/test_fetch.py @@ -0,0 +1,98 @@ +from datetime import datetime, timezone + +import pandas as pd +import pytest + +from src.analysis.data.data_request_handler import DataRequestHandler +from src.analysis.post_run import fetch + +EXP = { + "experiment": "nimlibp2p", + "stack": {"namespace": "zerotesting"}, + "results": {"stable": {"start": "2026-09-16T10:10:00", "end": "2026-09-16T10:11:00"}}, +} +T0 = datetime(2026, 9, 16, 10, 10, tzinfo=timezone.utc).timestamp() +VM_RESPONSE = { + "data": { + "result": [ + { + "metric": {"instance": "10.0.0.2:8008", "type": "Yamux", "dir": "in", "job": "j"}, + "values": [[T0, "3"], [T0 + 60, "4"]], + }, + { + "metric": {"instance": "10.0.0.1:8008", "type": "Yamux", "dir": "out", "job": "j"}, + "values": [[T0, "5"], [T0 + 60, "6.5"]], + }, + ] + } +} + + +class _Response: + def __init__(self, body): + self._body = body + + def json(self): + return self._body + + def raise_for_status(self): + pass + + +def _stored_rows(response): + """What the dashboard keeps of a VM response: one row per sample, schema labels only.""" + return [ + { + "timestamp": pd.to_datetime(ts, unit="s").isoformat(), + "value": float(value), + **{k: series["metric"][k] for k in ("instance", "type", "dir")}, + } + for series in response["data"]["result"] + for ts, value in series["values"] + ] + + +def test_a_fetched_run_matches_its_local_scrape(tmp_path, monkeypatch): + def get(url, **kwargs): + if url.endswith("/experiments"): + return _Response([{"id": "e1", "title": "run", "metadata": {**EXP, "run": "run"}}]) + rows = _stored_rows(VM_RESPONSE) + return _Response({"datasets": [{"name": "libp2p-open-streams", "data": rows}]}) + + monkeypatch.setattr(fetch.requests, "get", get) + dump = fetch.run_metrics(tmp_path / "run", url="http://dash") + + scraped = DataRequestHandler(VM_RESPONSE) + scraped.create_dataframe_from_request("instance-type-dir", None, None) + scraped.dump_dataframe(str(tmp_path / "scraped.csv")) + read = lambda path: pd.read_csv(path, index_col="Time", parse_dates=["Time"]) + pd.testing.assert_frame_equal( + read(dump / "libp2p-open-streams" / "run.csv"), + read(tmp_path / "scraped.csv"), + check_dtype=False, + ) + + +def test_a_local_scrape_is_used_without_the_dashboard(tmp_path, monkeypatch): + local = tmp_path / "run" / "metrics" / "libp2p-in" + local.mkdir(parents=True) + (local / "quic.csv").write_text("Time,pod-0\n") + + def get(*args, **kwargs): + raise AssertionError("contacted the dashboard") + + monkeypatch.setattr(fetch.requests, "get", get) + assert fetch.run_metrics(tmp_path / "run") == tmp_path / "run" / "metrics" + + +def test_a_metrics_folder_without_csvs_is_not_overwritten(tmp_path, monkeypatch): + local = tmp_path / "run" / "metrics" / "libp2p-in" + local.mkdir(parents=True) + (local / "quic").write_text("Time,pod-0\n") + + def get(*args, **kwargs): + raise AssertionError("contacted the dashboard") + + monkeypatch.setattr(fetch.requests, "get", get) + with pytest.raises(FileExistsError): + fetch.run_metrics(tmp_path / "run") From d85e33c750a5ea5477ab6978c054f1b0fd5c93f4 Mon Sep 17 00:00:00 2001 From: radiken Date: Tue, 17 Nov 2026 11:32:17 +0000 Subject: [PATCH 2/2] Fetch: find the run by its title, fail on missing rows or another query, keep the CSV names A run folder with metadata.json is matched on the exact title publish gives it, so runs sharing a folder name stay apart. A listed dataset with no rows, or stored under another query than today's scrape, fails the fetch instead of leaving a partial or mislabelled dump. Publish records the name the run's CSVs share, and the fetch writes the same file names. Co-Authored-By: Claude Opus 5.5 --- src/analysis/post_run/fetch.py | 55 +++++++++++++++------ src/analysis/post_run/logos_messaging.py | 2 +- src/analysis/post_run/nimlibp2p.py | 2 +- src/analysis/post_run/publish.py | 37 ++++++++++---- src/analysis/post_run/tests/test_fetch.py | 53 +++++++++++++++++--- src/analysis/post_run/tests/test_publish.py | 4 +- src/analysis/publish_run.py | 4 +- 7 files changed, 120 insertions(+), 37 deletions(-) diff --git a/src/analysis/post_run/fetch.py b/src/analysis/post_run/fetch.py index df1d1e14..70f41d30 100644 --- a/src/analysis/post_run/fetch.py +++ b/src/analysis/post_run/fetch.py @@ -1,16 +1,23 @@ """Read a published run's metrics back from the DST dashboard into the local scrape layout.""" +import json import logging import shutil from datetime import datetime, timezone from pathlib import Path -from typing import List +from typing import List, Optional import requests from src.analysis.data.data_request_handler import DataRequestHandler from src.analysis.post_run.metrics import run_scrape_config -from src.analysis.post_run.publish import DASHBOARD_URL, TIMEOUT_S +from src.analysis.post_run.publish import ( + DASHBOARD_URL, + SCRAPE_LABEL, + TIMEOUT_S, + metric_dataset, + run_title, +) logger = logging.getLogger(__name__) @@ -22,16 +29,26 @@ def run_metrics(run: Path, url: str = DASHBOARD_URL) -> Path: return dump if dump.exists(): raise FileExistsError(f"`{dump}` holds no .csv files; rename or move it to fetch the run") - return fetch_run_metrics(run.name, dump, url) + title = None + if (run / "metadata.json").exists(): + exp = json.loads((run / "metadata.json").read_text()) + title = run_title(run.name, run_scrape_config(exp, name=run.name)) + return fetch_run_metrics(run.name, dump, url, title) -def find_experiment(run_name: str, url: str = DASHBOARD_URL) -> dict: +def find_experiment(run_name: str, url: str = DASHBOARD_URL, title: Optional[str] = None) -> dict: + """The experiment titled `title`, else the only one published from a folder named `run_name`.""" response = requests.get(f"{url}/experiments", timeout=TIMEOUT_S) response.raise_for_status() - found = [e for e in response.json() if (e.get("metadata") or {}).get("run") == run_name] + if title: + found = [e for e in response.json() if e["title"] == title] + else: + found = [e for e in response.json() if (e.get("metadata") or {}).get("run") == run_name] if len(found) != 1: titles = [e["title"] for e in found] - raise LookupError(f"Expected one dashboard experiment for `{run_name}`, found {titles}") + raise LookupError( + f"Expected one dashboard experiment for `{title or run_name}`, found {titles}" + ) return found[0] @@ -52,9 +69,12 @@ def _query_response(rows: List[dict]) -> dict: return {"data": {"result": [{"metric": m, "values": v} for m, v in series.values()]}} -def fetch_run_metrics(run_name: str, dump: Path, url: str = DASHBOARD_URL) -> Path: +def fetch_run_metrics( + run_name: str, dump: Path, url: str = DASHBOARD_URL, title: Optional[str] = None +) -> Path: """Write the run's stored datasets to `dump` with the scraper's own CSV writer.""" - experiment = find_experiment(run_name, url) + experiment = find_experiment(run_name, url, title) + queries = {d["name"]: d["query"] for d in experiment.get("datasets", [])} response = requests.get(f"{url}/experiments/{experiment['id']}/datasets", timeout=TIMEOUT_S) response.raise_for_status() stored = {d["name"]: d["data"] for d in response.json()["datasets"]} @@ -62,19 +82,22 @@ def fetch_run_metrics(run_name: str, dump: Path, url: str = DASHBOARD_URL) -> Pa partial = dump.with_name(f"{dump.name}.partial") shutil.rmtree(partial, ignore_errors=True) config = run_scrape_config(experiment["metadata"], partial, run_name) - unknown = set(stored) - {m.folder_name.strip("/") for m in config.metrics_to_scrape} - if unknown: - logger.warning(f"`{run_name}` has stored datasets no current metric matches: {unknown}") + label = experiment["metadata"].get(SCRAPE_LABEL, run_name) for metric in config.metrics_to_scrape: - rows = stored.get(metric.folder_name.strip("/")) - if not rows: - logger.warning(f"`{run_name}` has no stored {metric.name} data") + name = metric.folder_name.strip("/") + if name not in queries: + logger.warning(f"`{run_name}` was published without {metric.name}") continue - handler = DataRequestHandler(_query_response(rows)) + query, current = queries[name], metric_dataset(metric, config)["query"] + if (query.get("expr"), query.get("step")) != (current["expr"], current["step"]): + raise ValueError(f"`{run_name}` stored {name} as {query}, not as today's {current}") + if not stored.get(name): + raise LookupError(f"`{run_name}` has no stored rows for {name}") + handler = DataRequestHandler(_query_response(stored[name])) handler.create_dataframe_from_request( metric.extract_field, metric.container, metric.metrics_path ) - handler.dump_dataframe((partial / metric.folder_name / f"{run_name}.csv").as_posix()) + handler.dump_dataframe((partial / metric.folder_name / f"{label}.csv").as_posix()) if not partial.exists(): raise LookupError(f"`{run_name}` has no stored metrics on the dashboard") diff --git a/src/analysis/post_run/logos_messaging.py b/src/analysis/post_run/logos_messaging.py index 00279a9c..19d73f58 100644 --- a/src/analysis/post_run/logos_messaging.py +++ b/src/analysis/post_run/logos_messaging.py @@ -400,7 +400,7 @@ def scrape_and_plot(experiment: "LogosMessagingExperiment") -> None: label = f"{experiment.config.num_relay_nodes} nodes" dump = scrape_run_metrics(experiment.metadata, out / "metrics", name=label) plot_run_metrics({label: dump}, out / "plots", xlabel=experiment.namespace) - publish_run_metrics(experiment.metadata, out.name) + publish_run_metrics(experiment.metadata, out) def _safe(label: str, query, default): diff --git a/src/analysis/post_run/nimlibp2p.py b/src/analysis/post_run/nimlibp2p.py index 8c10266f..de53cdbb 100644 --- a/src/analysis/post_run/nimlibp2p.py +++ b/src/analysis/post_run/nimlibp2p.py @@ -134,4 +134,4 @@ def _scrape_and_plot(experiment: "NimLibp2pExperiment") -> None: experiment.output_folder / "plots", xlabel=f"{experiment.namespace}, {experiment.config.num_relay_nodes} nodes", ) - publish_run_metrics(exp, experiment.output_folder.name) + publish_run_metrics(exp, experiment.output_folder) diff --git a/src/analysis/post_run/publish.py b/src/analysis/post_run/publish.py index a8f91937..6b5f1c96 100644 --- a/src/analysis/post_run/publish.py +++ b/src/analysis/post_run/publish.py @@ -3,6 +3,7 @@ import logging import os from datetime import datetime, timezone +from pathlib import Path from typing import List, Optional import requests @@ -16,6 +17,8 @@ TOKEN_ENV = "DST_DASHBOARD_TOKEN" DATASOURCE = "victoria-metrics" TIMEOUT_S = 600 +# The name the run's scraped CSVs share, kept so a fetch writes the same files. +SCRAPE_LABEL = "scrape_label" class PublishError(Exception): @@ -26,7 +29,7 @@ def _utc(dt: datetime) -> str: return (dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt).isoformat() -def _dataset(metric: MetricToScrape, config: ScrapeConfig) -> dict: +def metric_dataset(metric: MetricToScrape, config: ScrapeConfig) -> dict: labels = metric.extract_field.split("-") labels += [label for label in ("container", "metrics_path") if getattr(metric, label)] return { @@ -42,16 +45,29 @@ def _dataset(metric: MetricToScrape, config: ScrapeConfig) -> dict: } -def experiment_body(exp: dict, config: ScrapeConfig, run_name: str) -> dict: +def run_title(run_name: str, config: ScrapeConfig) -> str: + return f"{run_name} {config.start:%Y-%m-%d %H:%M}" + + +def scrape_label(run: Path) -> Optional[str]: + """The name the run's scraped CSVs share (`metrics//