diff --git a/src/analysis/fetch_run.py b/src/analysis/fetch_run.py new file mode 100644 index 00000000..fef5d694 --- /dev/null +++ b/src/analysis/fetch_run.py @@ -0,0 +1,34 @@ +"""Put published runs' metrics in their run folders, unless they were scraped there.""" + +import argparse +import logging +import sys +from pathlib import Path + +from src.analysis.post_run.fetch import run_metrics +from src.analysis.utils.log_utils import init_logger + +logger = logging.getLogger(__name__) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("runs", nargs="+", help="Run folders, named as the run on the dashboard") + args = parser.parse_args() + init_logger(logging.getLogger(), verbosity=1) + + failed = [] + for arg in args.runs: + run = Path(arg).resolve() + try: + print(run_metrics(run)) + except Exception: + logger.exception(f"Fetching `{run.name}` failed") + failed.append(run.name) + + if failed: + sys.exit(f"Not fetched: {failed}") + + +if __name__ == "__main__": + main() diff --git a/src/analysis/post_run/fetch.py b/src/analysis/post_run/fetch.py new file mode 100644 index 00000000..70f41d30 --- /dev/null +++ b/src/analysis/post_run/fetch.py @@ -0,0 +1,106 @@ +"""Read a published run's metrics back from the DST dashboard into the local scrape layout.""" + +import json +import logging +import shutil +from datetime import datetime, timezone +from pathlib import Path +from typing import List, Optional + +import requests + +from src.analysis.data.data_request_handler import DataRequestHandler +from src.analysis.post_run.metrics import run_scrape_config +from src.analysis.post_run.publish import ( + DASHBOARD_URL, + SCRAPE_LABEL, + TIMEOUT_S, + metric_dataset, + run_title, +) + +logger = logging.getLogger(__name__) + + +def run_metrics(run: Path, url: str = DASHBOARD_URL) -> Path: + """The run's metric dump: `/metrics` as scraped, else fetched there from the dashboard.""" + dump = run / "metrics" + if any(dump.glob("*/*.csv")): + return dump + if dump.exists(): + raise FileExistsError(f"`{dump}` holds no .csv files; rename or move it to fetch the run") + title = None + if (run / "metadata.json").exists(): + exp = json.loads((run / "metadata.json").read_text()) + title = run_title(run.name, run_scrape_config(exp, name=run.name)) + return fetch_run_metrics(run.name, dump, url, title) + + +def find_experiment(run_name: str, url: str = DASHBOARD_URL, title: Optional[str] = None) -> dict: + """The experiment titled `title`, else the only one published from a folder named `run_name`.""" + response = requests.get(f"{url}/experiments", timeout=TIMEOUT_S) + response.raise_for_status() + if title: + found = [e for e in response.json() if e["title"] == title] + else: + found = [e for e in response.json() if (e.get("metadata") or {}).get("run") == run_name] + if len(found) != 1: + titles = [e["title"] for e in found] + raise LookupError( + f"Expected one dashboard experiment for `{title or run_name}`, found {titles}" + ) + return found[0] + + +def _utc_seconds(timestamp: str) -> float: + dt = datetime.fromisoformat(timestamp) + return (dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)).timestamp() + + +def _query_response(rows: List[dict]) -> dict: + """The stored rows as the VM query_range response they came from.""" + series = {} + for row in rows: + labels = {k: v for k, v in row.items() if k not in ("timestamp", "value")} + key = tuple(sorted(labels.items())) + series.setdefault(key, (labels, []))[1].append( + [_utc_seconds(row["timestamp"]), row["value"]] + ) + return {"data": {"result": [{"metric": m, "values": v} for m, v in series.values()]}} + + +def fetch_run_metrics( + run_name: str, dump: Path, url: str = DASHBOARD_URL, title: Optional[str] = None +) -> Path: + """Write the run's stored datasets to `dump` with the scraper's own CSV writer.""" + experiment = find_experiment(run_name, url, title) + queries = {d["name"]: d["query"] for d in experiment.get("datasets", [])} + response = requests.get(f"{url}/experiments/{experiment['id']}/datasets", timeout=TIMEOUT_S) + response.raise_for_status() + stored = {d["name"]: d["data"] for d in response.json()["datasets"]} + + partial = dump.with_name(f"{dump.name}.partial") + shutil.rmtree(partial, ignore_errors=True) + config = run_scrape_config(experiment["metadata"], partial, run_name) + label = experiment["metadata"].get(SCRAPE_LABEL, run_name) + for metric in config.metrics_to_scrape: + name = metric.folder_name.strip("/") + if name not in queries: + logger.warning(f"`{run_name}` was published without {metric.name}") + continue + query, current = queries[name], metric_dataset(metric, config)["query"] + if (query.get("expr"), query.get("step")) != (current["expr"], current["step"]): + raise ValueError(f"`{run_name}` stored {name} as {query}, not as today's {current}") + if not stored.get(name): + raise LookupError(f"`{run_name}` has no stored rows for {name}") + handler = DataRequestHandler(_query_response(stored[name])) + handler.create_dataframe_from_request( + metric.extract_field, metric.container, metric.metrics_path + ) + handler.dump_dataframe((partial / metric.folder_name / f"{label}.csv").as_posix()) + + if not partial.exists(): + raise LookupError(f"`{run_name}` has no stored metrics on the dashboard") + partial.rename(dump) + logger.info(f"Fetched `{run_name}` from the dashboard ({experiment['id']}) into `{dump}`") + return dump diff --git a/src/analysis/post_run/logos_messaging.py b/src/analysis/post_run/logos_messaging.py index 00279a9c..19d73f58 100644 --- a/src/analysis/post_run/logos_messaging.py +++ b/src/analysis/post_run/logos_messaging.py @@ -400,7 +400,7 @@ def scrape_and_plot(experiment: "LogosMessagingExperiment") -> None: label = f"{experiment.config.num_relay_nodes} nodes" dump = scrape_run_metrics(experiment.metadata, out / "metrics", name=label) plot_run_metrics({label: dump}, out / "plots", xlabel=experiment.namespace) - publish_run_metrics(experiment.metadata, out.name) + publish_run_metrics(experiment.metadata, out) def _safe(label: str, query, default): diff --git a/src/analysis/post_run/nimlibp2p.py b/src/analysis/post_run/nimlibp2p.py index 8c10266f..de53cdbb 100644 --- a/src/analysis/post_run/nimlibp2p.py +++ b/src/analysis/post_run/nimlibp2p.py @@ -134,4 +134,4 @@ def _scrape_and_plot(experiment: "NimLibp2pExperiment") -> None: experiment.output_folder / "plots", xlabel=f"{experiment.namespace}, {experiment.config.num_relay_nodes} nodes", ) - publish_run_metrics(exp, experiment.output_folder.name) + publish_run_metrics(exp, experiment.output_folder) diff --git a/src/analysis/post_run/publish.py b/src/analysis/post_run/publish.py index a8f91937..6b5f1c96 100644 --- a/src/analysis/post_run/publish.py +++ b/src/analysis/post_run/publish.py @@ -3,6 +3,7 @@ import logging import os from datetime import datetime, timezone +from pathlib import Path from typing import List, Optional import requests @@ -16,6 +17,8 @@ TOKEN_ENV = "DST_DASHBOARD_TOKEN" DATASOURCE = "victoria-metrics" TIMEOUT_S = 600 +# The name the run's scraped CSVs share, kept so a fetch writes the same files. +SCRAPE_LABEL = "scrape_label" class PublishError(Exception): @@ -26,7 +29,7 @@ def _utc(dt: datetime) -> str: return (dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt).isoformat() -def _dataset(metric: MetricToScrape, config: ScrapeConfig) -> dict: +def metric_dataset(metric: MetricToScrape, config: ScrapeConfig) -> dict: labels = metric.extract_field.split("-") labels += [label for label in ("container", "metrics_path") if getattr(metric, label)] return { @@ -42,16 +45,29 @@ def _dataset(metric: MetricToScrape, config: ScrapeConfig) -> dict: } -def experiment_body(exp: dict, config: ScrapeConfig, run_name: str) -> dict: +def run_title(run_name: str, config: ScrapeConfig) -> str: + return f"{run_name} {config.start:%Y-%m-%d %H:%M}" + + +def scrape_label(run: Path) -> Optional[str]: + """The name the run's scraped CSVs share (`metrics//