Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions src/analysis/fetch_run.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
"""Put published runs' metrics in their run folders, unless they were scraped there."""

import argparse
import logging
import sys
from pathlib import Path

from src.analysis.post_run.fetch import run_metrics
from src.analysis.utils.log_utils import init_logger

logger = logging.getLogger(__name__)


def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("runs", nargs="+", help="Run folders, named as the run on the dashboard")
args = parser.parse_args()
init_logger(logging.getLogger(), verbosity=1)

failed = []
for arg in args.runs:
run = Path(arg).resolve()
try:
print(run_metrics(run))
except Exception:
logger.exception(f"Fetching `{run.name}` failed")
failed.append(run.name)

if failed:
sys.exit(f"Not fetched: {failed}")


if __name__ == "__main__":
main()
106 changes: 106 additions & 0 deletions src/analysis/post_run/fetch.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
"""Read a published run's metrics back from the DST dashboard into the local scrape layout."""

import json
import logging
import shutil
from datetime import datetime, timezone
from pathlib import Path
from typing import List, Optional

import requests

from src.analysis.data.data_request_handler import DataRequestHandler
from src.analysis.post_run.metrics import run_scrape_config
from src.analysis.post_run.publish import (
DASHBOARD_URL,
SCRAPE_LABEL,
TIMEOUT_S,
metric_dataset,
run_title,
)

logger = logging.getLogger(__name__)


def run_metrics(run: Path, url: str = DASHBOARD_URL) -> Path:
"""The run's metric dump: `<run>/metrics` as scraped, else fetched there from the dashboard."""
dump = run / "metrics"
if any(dump.glob("*/*.csv")):
return dump
if dump.exists():
raise FileExistsError(f"`{dump}` holds no .csv files; rename or move it to fetch the run")
title = None
if (run / "metadata.json").exists():
exp = json.loads((run / "metadata.json").read_text())
title = run_title(run.name, run_scrape_config(exp, name=run.name))
return fetch_run_metrics(run.name, dump, url, title)


def find_experiment(run_name: str, url: str = DASHBOARD_URL, title: Optional[str] = None) -> dict:
"""The experiment titled `title`, else the only one published from a folder named `run_name`."""
response = requests.get(f"{url}/experiments", timeout=TIMEOUT_S)
response.raise_for_status()
if title:
found = [e for e in response.json() if e["title"] == title]
else:
found = [e for e in response.json() if (e.get("metadata") or {}).get("run") == run_name]
if len(found) != 1:
titles = [e["title"] for e in found]
raise LookupError(
f"Expected one dashboard experiment for `{title or run_name}`, found {titles}"
)
return found[0]


def _utc_seconds(timestamp: str) -> float:
dt = datetime.fromisoformat(timestamp)
return (dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)).timestamp()


def _query_response(rows: List[dict]) -> dict:
"""The stored rows as the VM query_range response they came from."""
series = {}
for row in rows:
labels = {k: v for k, v in row.items() if k not in ("timestamp", "value")}
key = tuple(sorted(labels.items()))
series.setdefault(key, (labels, []))[1].append(
[_utc_seconds(row["timestamp"]), row["value"]]
)
return {"data": {"result": [{"metric": m, "values": v} for m, v in series.values()]}}


def fetch_run_metrics(
run_name: str, dump: Path, url: str = DASHBOARD_URL, title: Optional[str] = None
) -> Path:
"""Write the run's stored datasets to `dump` with the scraper's own CSV writer."""
experiment = find_experiment(run_name, url, title)
queries = {d["name"]: d["query"] for d in experiment.get("datasets", [])}
response = requests.get(f"{url}/experiments/{experiment['id']}/datasets", timeout=TIMEOUT_S)
response.raise_for_status()
stored = {d["name"]: d["data"] for d in response.json()["datasets"]}

partial = dump.with_name(f"{dump.name}.partial")
shutil.rmtree(partial, ignore_errors=True)
config = run_scrape_config(experiment["metadata"], partial, run_name)
label = experiment["metadata"].get(SCRAPE_LABEL, run_name)
for metric in config.metrics_to_scrape:
name = metric.folder_name.strip("/")
if name not in queries:
logger.warning(f"`{run_name}` was published without {metric.name}")
continue
query, current = queries[name], metric_dataset(metric, config)["query"]
if (query.get("expr"), query.get("step")) != (current["expr"], current["step"]):
raise ValueError(f"`{run_name}` stored {name} as {query}, not as today's {current}")
if not stored.get(name):
raise LookupError(f"`{run_name}` has no stored rows for {name}")
handler = DataRequestHandler(_query_response(stored[name]))
handler.create_dataframe_from_request(
metric.extract_field, metric.container, metric.metrics_path
)
handler.dump_dataframe((partial / metric.folder_name / f"{label}.csv").as_posix())

if not partial.exists():
raise LookupError(f"`{run_name}` has no stored metrics on the dashboard")
partial.rename(dump)
logger.info(f"Fetched `{run_name}` from the dashboard ({experiment['id']}) into `{dump}`")
return dump
2 changes: 1 addition & 1 deletion src/analysis/post_run/logos_messaging.py
Original file line number Diff line number Diff line change
Expand Up @@ -400,7 +400,7 @@ def scrape_and_plot(experiment: "LogosMessagingExperiment") -> None:
label = f"{experiment.config.num_relay_nodes} nodes"
dump = scrape_run_metrics(experiment.metadata, out / "metrics", name=label)
plot_run_metrics({label: dump}, out / "plots", xlabel=experiment.namespace)
publish_run_metrics(experiment.metadata, out.name)
publish_run_metrics(experiment.metadata, out)


def _safe(label: str, query, default):
Expand Down
2 changes: 1 addition & 1 deletion src/analysis/post_run/nimlibp2p.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,4 +134,4 @@ def _scrape_and_plot(experiment: "NimLibp2pExperiment") -> None:
experiment.output_folder / "plots",
xlabel=f"{experiment.namespace}, {experiment.config.num_relay_nodes} nodes",
)
publish_run_metrics(exp, experiment.output_folder.name)
publish_run_metrics(exp, experiment.output_folder)
37 changes: 27 additions & 10 deletions src/analysis/post_run/publish.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import logging
import os
from datetime import datetime, timezone
from pathlib import Path
from typing import List, Optional

import requests
Expand All @@ -16,6 +17,8 @@
TOKEN_ENV = "DST_DASHBOARD_TOKEN"
DATASOURCE = "victoria-metrics"
TIMEOUT_S = 600
# The name the run's scraped CSVs share, kept so a fetch writes the same files.
SCRAPE_LABEL = "scrape_label"


class PublishError(Exception):
Expand All @@ -26,7 +29,7 @@ def _utc(dt: datetime) -> str:
return (dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt).isoformat()


def _dataset(metric: MetricToScrape, config: ScrapeConfig) -> dict:
def metric_dataset(metric: MetricToScrape, config: ScrapeConfig) -> dict:
labels = metric.extract_field.split("-")
labels += [label for label in ("container", "metrics_path") if getattr(metric, label)]
return {
Expand All @@ -42,16 +45,29 @@ def _dataset(metric: MetricToScrape, config: ScrapeConfig) -> dict:
}


def experiment_body(exp: dict, config: ScrapeConfig, run_name: str) -> dict:
def run_title(run_name: str, config: ScrapeConfig) -> str:
return f"{run_name} {config.start:%Y-%m-%d %H:%M}"


def scrape_label(run: Path) -> Optional[str]:
"""The name the run's scraped CSVs share (`metrics/<metric>/<label>.csv`), if one."""
labels = {path.stem for path in (run / "metrics").glob("*/*.csv")}
return labels.pop() if len(labels) == 1 else None


def experiment_body(
exp: dict, config: ScrapeConfig, run_name: str, label: Optional[str] = None
) -> dict:
"""One dataset per scraped metric, over the same window and step as the local CSVs."""
return {
"title": f"{run_name} {config.start:%Y-%m-%d %H:%M}",
"title": run_title(run_name, config),
"family": exp["experiment"]["name"],
"date": f"{config.start:%Y-%m-%d}",
"docker_image": exp.get("params", {}).get("image"),
"metadata": {"run": run_name, "experiment": exp["experiment"]["name"]}
| {key: exp[key] for key in ("params", "code", "stack", "results") if key in exp},
"datasets": [_dataset(metric, config) for metric in config.metrics_to_scrape],
| {key: exp[key] for key in ("params", "code", "stack", "results") if key in exp}
| ({SCRAPE_LABEL: label} if label else {}),
"datasets": [metric_dataset(metric, config) for metric in config.metrics_to_scrape],
"panels": [],
"publish": False,
}
Expand Down Expand Up @@ -105,18 +121,19 @@ def unpublishable(exp: dict) -> Optional[str]:
return None


def publish_run_metrics(exp: dict, run_name: str) -> Optional[str]:
def publish_run_metrics(exp: dict, run: Path) -> Optional[str]:
"""Publish what `scrape_run_metrics` scraped for this run. Logs failures, never raises."""
reason = unpublishable(exp)
if reason:
logger.warning(f"`{run_name}` is not on the dashboard: {reason}")
logger.warning(f"`{run.name}` is not on the dashboard: {reason}")
return None
token = os.environ.get(TOKEN_ENV)
if not token:
logger.warning(f"{TOKEN_ENV} is not set; `{run_name}` is not on the dashboard")
logger.warning(f"{TOKEN_ENV} is not set; `{run.name}` is not on the dashboard")
return None
try:
return publish(experiment_body(exp, run_scrape_config(exp, name=run_name), run_name), token)
config = run_scrape_config(exp, name=run.name)
return publish(experiment_body(exp, config, run.name, scrape_label(run)), token)
except Exception:
logger.exception(f"Publishing `{run_name}` to the dashboard failed")
logger.exception(f"Publishing `{run.name}` to the dashboard failed")
return None
137 changes: 137 additions & 0 deletions src/analysis/post_run/tests/test_fetch.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
import json
from datetime import datetime, timezone

import pandas as pd
import pytest

from src.analysis.data.data_request_handler import DataRequestHandler
from src.analysis.post_run import fetch
from src.analysis.post_run.metrics import run_scrape_config
from src.analysis.post_run.publish import SCRAPE_LABEL, metric_dataset

EXP = {
"experiment": "nimlibp2p",
"stack": {"namespace": "zerotesting"},
"results": {"stable": {"start": "2026-09-16T10:10:00", "end": "2026-09-16T10:11:00"}},
}
STREAMS = "libp2p-open-streams"
T0 = datetime(2026, 9, 16, 10, 10, tzinfo=timezone.utc).timestamp()
VM_RESPONSE = {
"data": {
"result": [
{
"metric": {"instance": "10.0.0.2:8008", "type": "Yamux", "dir": "in", "job": "j"},
"values": [[T0, "3"], [T0 + 60, "4"]],
},
{
"metric": {"instance": "10.0.0.1:8008", "type": "Yamux", "dir": "out", "job": "j"},
"values": [[T0, "5"], [T0 + 60, "6.5"]],
},
]
}
}


class _Response:
def __init__(self, body):
self._body = body

def json(self):
return self._body

def raise_for_status(self):
pass


def _stored_rows(response):
"""What the dashboard keeps of a VM response: one row per sample, schema labels only."""
return [
{
"timestamp": pd.to_datetime(ts, unit="s").isoformat(),
"value": float(value),
**{k: series["metric"][k] for k in ("instance", "type", "dir")},
}
for series in response["data"]["result"]
for ts, value in series["values"]
]


def _published(monkeypatch, run, query=None, empty=()):
"""A dashboard holding `run` twice, from two windows; the right one has `empty` datasets too."""
(run).mkdir()
(run / "metadata.json").write_text(json.dumps(EXP))
config = run_scrape_config(EXP, name=run.name)
queries = {
m.folder_name.strip("/"): metric_dataset(m, config)["query"]
for m in config.metrics_to_scrape
}
stored = [{"name": STREAMS, "query": query or queries[STREAMS]}]
stored += [{"name": name, "query": queries[name]} for name in empty]
experiments = [
{"id": "e0", "title": "run 2026-01-01 00:00", "metadata": {"run": "run"}},
{"id": "e1", "title": "run 2026-09-16 10:10", "datasets": stored}
| {"metadata": {**EXP, "run": "run", SCRAPE_LABEL: "yamux"}},
]

def get(url, **kwargs):
if url.endswith("/experiments"):
return _Response(experiments)
assert url.endswith("/experiments/e1/datasets"), url
rows = {STREAMS: _stored_rows(VM_RESPONSE)} | dict.fromkeys(empty, [])
return _Response({"datasets": [{"name": n, "data": r} for n, r in rows.items()]})

monkeypatch.setattr(fetch.requests, "get", get)


def test_a_fetched_run_matches_its_local_scrape(tmp_path, monkeypatch):
_published(monkeypatch, tmp_path / "run")
dump = fetch.run_metrics(tmp_path / "run", url="http://dash")

scraped = DataRequestHandler(VM_RESPONSE)
scraped.create_dataframe_from_request("instance-type-dir", None, None)
scraped.dump_dataframe(str(tmp_path / "scraped.csv"))
read = lambda path: pd.read_csv(path, index_col="Time", parse_dates=["Time"])
pd.testing.assert_frame_equal(
read(dump / STREAMS / "yamux.csv"), read(tmp_path / "scraped.csv"), check_dtype=False
)


@pytest.mark.parametrize(
"query, empty, error, match",
[
(None, ["mesh-peers"], LookupError, "no stored rows for mesh-peers"),
({"expr": "other", "step": "60s"}, (), ValueError, "not as today's"),
],
)
def test_a_run_stored_empty_or_under_another_query_is_not_fetched(
tmp_path, monkeypatch, query, empty, error, match
):
_published(monkeypatch, tmp_path / "run", query, empty)
with pytest.raises(error, match=match):
fetch.run_metrics(tmp_path / "run", url="http://dash")
assert not (tmp_path / "run" / "metrics").exists()


def test_a_local_scrape_is_used_without_the_dashboard(tmp_path, monkeypatch):
local = tmp_path / "run" / "metrics" / "libp2p-in"
local.mkdir(parents=True)
(local / "quic.csv").write_text("Time,pod-0\n")

def get(*args, **kwargs):
raise AssertionError("contacted the dashboard")

monkeypatch.setattr(fetch.requests, "get", get)
assert fetch.run_metrics(tmp_path / "run") == tmp_path / "run" / "metrics"


def test_a_metrics_folder_without_csvs_is_not_overwritten(tmp_path, monkeypatch):
local = tmp_path / "run" / "metrics" / "libp2p-in"
local.mkdir(parents=True)
(local / "quic").write_text("Time,pod-0\n")

def get(*args, **kwargs):
raise AssertionError("contacted the dashboard")

monkeypatch.setattr(fetch.requests, "get", get)
with pytest.raises(FileExistsError):
fetch.run_metrics(tmp_path / "run")
4 changes: 3 additions & 1 deletion src/analysis/post_run/tests/test_publish.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
from pathlib import Path

import pytest

from src.analysis.post_run import publish as publish_module
Expand Down Expand Up @@ -84,6 +86,6 @@ def test_only_runs_from_clean_pushed_code_are_published(monkeypatch, code, publi
monkeypatch.setenv(publish_module.TOKEN_ENV, "t")
monkeypatch.setattr(publish_module, "publish", lambda body, token: posted.append(body))

publish_module.publish_run_metrics({**EXP, "code": code}, "run")
publish_module.publish_run_metrics({**EXP, "code": code}, Path("run"))

assert bool(posted) is published
Loading
Loading