From b85ab1d779d6c40487d8478247eeb73c99c5b03c Mon Sep 17 00:00:00 2001 From: radiken Date: Fri, 6 Nov 2026 14:46:56 +0000 Subject: [PATCH 1/3] Publish each run's metrics to the DST dashboard After the post-run scrape, post the same metrics and window to the dashboard, which keeps them past the lab's six-month retention. A dataset that comes back empty fails the publish and the experiment is removed again. publish_run publishes existing run folders. Co-Authored-By: Claude Opus 5.5 --- src/analysis/post_run/logos_messaging.py | 2 + src/analysis/post_run/metrics.py | 18 +++- src/analysis/post_run/nimlibp2p.py | 2 + src/analysis/post_run/publish.py | 97 +++++++++++++++++++++ src/analysis/post_run/tests/test_publish.py | 73 ++++++++++++++++ src/analysis/publish_run.py | 51 +++++++++++ 6 files changed, 239 insertions(+), 4 deletions(-) create mode 100644 src/analysis/post_run/publish.py create mode 100644 src/analysis/post_run/tests/test_publish.py create mode 100644 src/analysis/publish_run.py diff --git a/src/analysis/post_run/logos_messaging.py b/src/analysis/post_run/logos_messaging.py index 009c4113..00279a9c 100644 --- a/src/analysis/post_run/logos_messaging.py +++ b/src/analysis/post_run/logos_messaging.py @@ -18,6 +18,7 @@ from src.analysis.mesh_analysis.readers.victoria_reader import VictoriaReader from src.analysis.metrics import scrape_utils from src.analysis.post_run.metrics import plot_run_metrics, scrape_run_metrics +from src.analysis.post_run.publish import publish_run_metrics from src.deployments.core.event_window_bridge import EventNotFound if TYPE_CHECKING: @@ -399,6 +400,7 @@ def scrape_and_plot(experiment: "LogosMessagingExperiment") -> None: label = f"{experiment.config.num_relay_nodes} nodes" dump = scrape_run_metrics(experiment.metadata, out / "metrics", name=label) plot_run_metrics({label: dump}, out / "plots", xlabel=experiment.namespace) + publish_run_metrics(experiment.metadata, out.name) def _safe(label: str, query, default): diff --git a/src/analysis/post_run/metrics.py b/src/analysis/post_run/metrics.py index 9d780d1b..7bfc637e 100644 --- a/src/analysis/post_run/metrics.py +++ b/src/analysis/post_run/metrics.py @@ -10,6 +10,7 @@ from typing import Dict, List, Optional from src.analysis.data.data_file_handler import DataPath +from src.analysis.metrics.config import ScrapeConfig from src.analysis.metrics.libp2p.scrape import Nimlibp2pScrapeBuilder from src.analysis.metrics.scrapper import Scrapper from src.analysis.plotting.config import DataGroup, PlotConfig @@ -47,14 +48,23 @@ """Templates: `name` is the figure's file stem, the run fills in groups and output path.""" +def run_scrape_config( + exp: dict, dump_location: Optional[Path] = None, name: Optional[str] = None +) -> ScrapeConfig: + """The standard metrics over one run's stable window.""" + builder = Nimlibp2pScrapeBuilder().with_exp(exp, extract_name=name is None) + if name is not None: + builder.name = name + if dump_location is not None: + builder.with_dump_location(str(dump_location)) + return builder.with_libp2p_metrics().build() + + def scrape_run_metrics( exp: dict, dump_location: Path, kube_config: Optional[str] = None, name: Optional[str] = None ) -> Path: """Dump the per-metric CSVs for one run's stable window.""" - builder = Nimlibp2pScrapeBuilder().with_exp(exp, extract_name=name is None) - if name is not None: - builder.name = name - config = builder.with_dump_location(str(dump_location)).with_libp2p_metrics().build() + config = run_scrape_config(exp, dump_location, name) logger.info(f"Scraping `{config.name}` metrics: {config.start} -> {config.end}") Scrapper(kube_config, config).query_and_dump_metrics() return dump_location diff --git a/src/analysis/post_run/nimlibp2p.py b/src/analysis/post_run/nimlibp2p.py index 43f10931..8c10266f 100644 --- a/src/analysis/post_run/nimlibp2p.py +++ b/src/analysis/post_run/nimlibp2p.py @@ -9,6 +9,7 @@ from src.analysis.plotting.latency_plotter import plot_dump_latency from src.analysis.post_run.delivery_cross_check import counter_deliveries, cross_check, report from src.analysis.post_run.metrics import plot_run_metrics, scrape_run_metrics +from src.analysis.post_run.publish import publish_run_metrics if TYPE_CHECKING: from src.deployments.experiments.libp2p.nimlibp2p import NimLibp2pExperiment @@ -133,3 +134,4 @@ def _scrape_and_plot(experiment: "NimLibp2pExperiment") -> None: experiment.output_folder / "plots", xlabel=f"{experiment.namespace}, {experiment.config.num_relay_nodes} nodes", ) + publish_run_metrics(exp, experiment.output_folder.name) diff --git a/src/analysis/post_run/publish.py b/src/analysis/post_run/publish.py new file mode 100644 index 00000000..f7f2ba2d --- /dev/null +++ b/src/analysis/post_run/publish.py @@ -0,0 +1,97 @@ +"""Publish a run's metrics to the DST dashboard, which keeps them past the lab's retention.""" + +import logging +import os +from datetime import datetime, timezone +from typing import List, Optional + +import requests + +from src.analysis.metrics.config import MetricToScrape, ScrapeConfig +from src.analysis.post_run.metrics import run_scrape_config + +logger = logging.getLogger(__name__) + +DASHBOARD_URL = "https://api.dashboard.lab.vac.dev" +TOKEN_ENV = "DST_DASHBOARD_TOKEN" +DATASOURCE = "victoria-metrics" +TIMEOUT_S = 600 + + +class PublishError(Exception): + pass + + +def _utc(dt: datetime) -> str: + return (dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt).isoformat() + + +def _dataset(metric: MetricToScrape, config: ScrapeConfig) -> dict: + labels = metric.extract_field.split("-") + labels += [label for label in ("container", "metrics_path") if getattr(metric, label)] + return { + "name": metric.folder_name.strip("/"), + "datasource": DATASOURCE, + "timeRange": {"start": _utc(config.start), "end": _utc(config.end)}, + "query": { + "expr": metric.query.replace("$__rate_interval", config.rate_interval), + "step": config.step, + }, + "schema": [{"name": "timestamp", "type": "datetime"}, {"name": "value", "type": "float"}] + + [{"name": label, "type": "string"} for label in labels], + } + + +def experiment_body(exp: dict, config: ScrapeConfig, run_name: str) -> dict: + """One dataset per scraped metric, over the same window and step as the local CSVs.""" + return { + "title": f"{run_name} {config.start:%Y-%m-%d %H:%M}", + "family": exp["experiment"]["name"], + "date": f"{config.start:%Y-%m-%d}", + "docker_image": exp.get("params", {}).get("image"), + "metadata": {"run": run_name, "experiment": exp["experiment"]["name"]} + | {key: exp[key] for key in ("params", "stack", "results") if key in exp}, + "datasets": [_dataset(metric, config) for metric in config.metrics_to_scrape], + "panels": [], + "publish": False, + } + + +def _empty_datasets(datasets: dict) -> List[str]: + return [d["name"] for d in datasets["datasets"] if not d.get("rowCount")] + + +def publish(body: dict, token: str, url: str = DASHBOARD_URL) -> Optional[str]: + """Create the experiment and check every dataset got rows; returns None if already there.""" + headers = {"Authorization": f"Bearer {token}"} + response = requests.post(f"{url}/experiments", json=body, headers=headers, timeout=TIMEOUT_S) + if response.status_code == 409: + logger.info(f"`{body['title']}` is already on the dashboard") + return None + response.raise_for_status() + experiment_id = response.json()["id"] + + response = requests.get(f"{url}/experiments/{experiment_id}/datasets", timeout=TIMEOUT_S) + response.raise_for_status() + empty = _empty_datasets(response.json()) + if empty: + requests.delete( + f"{url}/experiments/{experiment_id}", headers=headers, timeout=TIMEOUT_S + ).raise_for_status() + raise PublishError(f"`{body['title']}` got no rows for {empty}; removed it") + + logger.info(f"Published `{body['title']}` to the dashboard as {experiment_id}") + return experiment_id + + +def publish_run_metrics(exp: dict, run_name: str) -> Optional[str]: + """Publish what `scrape_run_metrics` scraped for this run. Logs failures, never raises.""" + token = os.environ.get(TOKEN_ENV) + if not token: + logger.warning(f"{TOKEN_ENV} is not set; `{run_name}` is not on the dashboard") + return None + try: + return publish(experiment_body(exp, run_scrape_config(exp, name=run_name), run_name), token) + except Exception: + logger.exception(f"Publishing `{run_name}` to the dashboard failed") + return None diff --git a/src/analysis/post_run/tests/test_publish.py b/src/analysis/post_run/tests/test_publish.py new file mode 100644 index 00000000..e91a4616 --- /dev/null +++ b/src/analysis/post_run/tests/test_publish.py @@ -0,0 +1,73 @@ +import pytest + +from src.analysis.post_run import publish as publish_module +from src.analysis.post_run.metrics import run_scrape_config +from src.analysis.post_run.publish import PublishError, experiment_body, publish + +EXP = { + "experiment": {"name": "nimlibp2p"}, + "params": {"muxer": "yamux", "image": "regression-node:v2.4.0"}, + "stack": {"namespace": "zerotesting"}, + "results": { + "complete": {"start": "2026-09-16T10:03:45", "end": "2026-09-16T10:26:32"}, + "stable": {"start": "2026-09-16T10:10:00", "end": "2026-09-16T10:16:30"}, + }, +} + + +class _Response: + def __init__(self, status_code, body=None): + self.status_code = status_code + self._body = body + + def json(self): + return self._body + + def raise_for_status(self): + if self.status_code >= 400: + raise RuntimeError(self.status_code) + + +def test_each_scraped_metric_is_a_dataset_over_the_stable_window(): + config = run_scrape_config(EXP, name="run") + body = experiment_body(EXP, config, "run") + + datasets = {d["name"]: d for d in body["datasets"]} + assert set(datasets) == {m.folder_name.strip("/") for m in config.metrics_to_scrape} + for dataset in datasets.values(): + assert dataset["timeRange"] == { + "start": "2026-09-16T10:10:00+00:00", + "end": "2026-09-16T10:16:30+00:00", + } + assert "$__rate_interval" not in dataset["query"]["expr"] + assert "[121s]" in datasets["libp2p-in"]["query"]["expr"] + # The scrape names CSV columns after these labels, so they must be kept. + open_streams = [field["name"] for field in datasets["libp2p-open-streams"]["schema"]] + assert {"instance", "type", "dir"} <= set(open_streams) + + +def test_a_dataset_without_rows_fails_the_publish_and_is_removed(monkeypatch): + deleted = [] + monkeypatch.setattr( + publish_module.requests, "post", lambda *a, **k: _Response(201, {"id": "e1"}) + ) + monkeypatch.setattr( + publish_module.requests, + "get", + lambda *a, **k: _Response( + 200, + { + "datasets": [ + {"name": "libp2p-in", "rowCount": 7}, + {"name": "mesh-peers", "rowCount": 0}, + ] + }, + ), + ) + monkeypatch.setattr( + publish_module.requests, "delete", lambda url, **k: deleted.append(url) or _Response(204) + ) + + with pytest.raises(PublishError, match="mesh-peers"): + publish({"title": "run"}, token="t", url="http://dash") + assert deleted == ["http://dash/experiments/e1"] diff --git a/src/analysis/publish_run.py b/src/analysis/publish_run.py new file mode 100644 index 00000000..78153c04 --- /dev/null +++ b/src/analysis/publish_run.py @@ -0,0 +1,51 @@ +"""Publish finished runs' metrics to the DST dashboard from their folders.""" + +import argparse +import json +import logging +import os +import sys +from pathlib import Path + +from src.analysis.post_run.metrics import run_scrape_config +from src.analysis.post_run.publish import DASHBOARD_URL, TOKEN_ENV, experiment_body, publish +from src.analysis.utils.log_utils import init_logger + +logger = logging.getLogger(__name__) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("runs", nargs="+", help="Run folders holding a metadata.json") + parser.add_argument("--dry-run", action="store_true", help="Print the experiments instead") + args = parser.parse_args() + init_logger(logging.getLogger(), verbosity=1) + token = os.environ.get(TOKEN_ENV) + if not token and not args.dry_run: + sys.exit(f"Set {TOKEN_ENV} to a token from {DASHBOARD_URL}/admin/token") + + failed = [] + for arg in args.runs: + run = Path(arg).resolve() + exp = json.loads((run / "metadata.json").read_text()) + # Same rule as the post-run scrape, so the dashboard holds what the CSVs hold. + if not exp.get("results", {}).get("stable", {}).get("duration"): + logger.error(f"`{run.name}` has no stable window; skipping it") + failed.append(run.name) + continue + body = experiment_body(exp, run_scrape_config(exp, name=run.name), run.name) + if args.dry_run: + print(json.dumps(body, indent=2)) + continue + try: + publish(body, token) + except Exception: + logger.exception(f"Publishing `{run.name}` failed") + failed.append(run.name) + + if failed: + sys.exit(f"Not published: {failed}") + + +if __name__ == "__main__": + main() From 118e2572cc05b0a39dcf25a934fdb459e710480c Mon Sep 17 00:00:00 2001 From: radiken Date: Mon, 9 Nov 2026 14:18:49 +0000 Subject: [PATCH 2/3] Record the 10ksim commit with every run; publish only clean, pushed runs Every run records the repo, commit and branch in metadata.json, and whether the code was clean and pushed. Runs are never refused. Only runs from clean, pushed code go to the dashboard; the others are skipped with the reason. Co-Authored-By: Claude Opus 5.5 --- src/analysis/post_run/publish.py | 18 +++++++++- src/analysis/post_run/tests/test_publish.py | 19 +++++++++++ src/analysis/publish_run.py | 13 +++++++- src/deployments/core/base_bridge.py | 7 ++-- src/deployments/core/code_version.py | 33 +++++++++++++++++++ .../core/tests/test_code_version.py | 33 +++++++++++++++++++ .../experiments/base_experiment.py | 2 ++ 7 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 src/deployments/core/code_version.py create mode 100644 src/deployments/core/tests/test_code_version.py diff --git a/src/analysis/post_run/publish.py b/src/analysis/post_run/publish.py index f7f2ba2d..1fc2ca3d 100644 --- a/src/analysis/post_run/publish.py +++ b/src/analysis/post_run/publish.py @@ -50,7 +50,7 @@ def experiment_body(exp: dict, config: ScrapeConfig, run_name: str) -> dict: "date": f"{config.start:%Y-%m-%d}", "docker_image": exp.get("params", {}).get("image"), "metadata": {"run": run_name, "experiment": exp["experiment"]["name"]} - | {key: exp[key] for key in ("params", "stack", "results") if key in exp}, + | {key: exp[key] for key in ("params", "code", "stack", "results") if key in exp}, "datasets": [_dataset(metric, config) for metric in config.metrics_to_scrape], "panels": [], "publish": False, @@ -84,8 +84,24 @@ def publish(body: dict, token: str, url: str = DASHBOARD_URL) -> Optional[str]: return experiment_id +def unpublishable(exp: dict) -> Optional[str]: + """Why a run stays off the dashboard, or None if its 10ksim code was clean and pushed.""" + code = exp.get("code") + if not code: + return "no 10ksim commit was recorded" + if not code.get("clean"): + return f"10ksim had uncommitted changes on top of {code['commit'][:7]}" + if not code.get("pushed"): + return f"10ksim {code['commit'][:7]} was not pushed" + return None + + def publish_run_metrics(exp: dict, run_name: str) -> Optional[str]: """Publish what `scrape_run_metrics` scraped for this run. Logs failures, never raises.""" + reason = unpublishable(exp) + if reason: + logger.warning(f"`{run_name}` is not on the dashboard: {reason}") + return None token = os.environ.get(TOKEN_ENV) if not token: logger.warning(f"{TOKEN_ENV} is not set; `{run_name}` is not on the dashboard") diff --git a/src/analysis/post_run/tests/test_publish.py b/src/analysis/post_run/tests/test_publish.py index e91a4616..487f9767 100644 --- a/src/analysis/post_run/tests/test_publish.py +++ b/src/analysis/post_run/tests/test_publish.py @@ -71,3 +71,22 @@ def test_a_dataset_without_rows_fails_the_publish_and_is_removed(monkeypatch): with pytest.raises(PublishError, match="mesh-peers"): publish({"title": "run"}, token="t", url="http://dash") assert deleted == ["http://dash/experiments/e1"] + + +@pytest.mark.parametrize( + "code, published", + [ + (None, False), + ({"commit": "abc1234", "clean": False, "pushed": True}, False), + ({"commit": "abc1234", "clean": True, "pushed": False}, False), + ({"commit": "abc1234", "clean": True, "pushed": True}, True), + ], +) +def test_only_runs_from_clean_pushed_code_are_published(monkeypatch, code, published): + posted = [] + monkeypatch.setenv(publish_module.TOKEN_ENV, "t") + monkeypatch.setattr(publish_module, "publish", lambda body, token: posted.append(body)) + + publish_module.publish_run_metrics({**EXP, "code": code}, "run") + + assert bool(posted) is published diff --git a/src/analysis/publish_run.py b/src/analysis/publish_run.py index 78153c04..eec091be 100644 --- a/src/analysis/publish_run.py +++ b/src/analysis/publish_run.py @@ -8,7 +8,13 @@ from pathlib import Path from src.analysis.post_run.metrics import run_scrape_config -from src.analysis.post_run.publish import DASHBOARD_URL, TOKEN_ENV, experiment_body, publish +from src.analysis.post_run.publish import ( + DASHBOARD_URL, + TOKEN_ENV, + experiment_body, + publish, + unpublishable, +) from src.analysis.utils.log_utils import init_logger logger = logging.getLogger(__name__) @@ -33,6 +39,11 @@ def main() -> None: logger.error(f"`{run.name}` has no stable window; skipping it") failed.append(run.name) continue + reason = unpublishable(exp) + if reason: + logger.error(f"`{run.name}` is not published: {reason}") + failed.append(run.name) + continue body = experiment_body(exp, run_scrape_config(exp, name=run.name), run.name) if args.dry_run: print(json.dumps(body, indent=2)) diff --git a/src/deployments/core/base_bridge.py b/src/deployments/core/base_bridge.py index 89cc9f2c..fa44b7ef 100644 --- a/src/deployments/core/base_bridge.py +++ b/src/deployments/core/base_bridge.py @@ -78,10 +78,9 @@ def get_metadata(self, events_log: Path) -> dict: except ValueError as e: logger.info(e) metadata["stack"]["name"] = self._get_name(metadata) - try: - metadata["params"] = all_metadata["params"] - except KeyError: - pass + for key in ("params", "code"): + if key in all_metadata: + metadata[key] = all_metadata[key] return dict(metadata) def _get_name(self, metadata: dict): diff --git a/src/deployments/core/code_version.py b/src/deployments/core/code_version.py new file mode 100644 index 00000000..aed4c595 --- /dev/null +++ b/src/deployments/core/code_version.py @@ -0,0 +1,33 @@ +"""The 10ksim commit a run deploys from.""" + +import logging +import subprocess +from pathlib import Path +from typing import Optional + +logger = logging.getLogger(__name__) + +REPO = Path(__file__).parent + + +def _git(repo: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(repo), *args], check=True, capture_output=True, text=True + ) + return result.stdout.strip() + + +def code_version(repo: Path = REPO) -> Optional[dict]: + """Commit and branch of `repo`, and whether it was clean and pushed; None without git.""" + try: + commit = _git(repo, "rev-parse", "HEAD") + return { + "repo": _git(repo, "remote", "get-url", "origin"), + "commit": commit, + "branch": _git(repo, "rev-parse", "--abbrev-ref", "HEAD"), + "clean": not _git(repo, "status", "--porcelain", "--untracked-files=no"), + "pushed": bool(_git(repo, "branch", "--remotes", "--contains", commit)), + } + except (OSError, subprocess.CalledProcessError) as e: + logger.warning(f"Could not read the 10ksim commit: {e}") + return None diff --git a/src/deployments/core/tests/test_code_version.py b/src/deployments/core/tests/test_code_version.py new file mode 100644 index 00000000..66053813 --- /dev/null +++ b/src/deployments/core/tests/test_code_version.py @@ -0,0 +1,33 @@ +import subprocess + +from src.deployments.core.code_version import code_version + + +def _git(repo, *args): + subprocess.run( + ["git", "-C", str(repo), "-c", "user.name=t", "-c", "user.email=t@t"] + + ["-c", "commit.gpgsign=false", *args], + check=True, + capture_output=True, + ) + + +def test_records_whether_the_code_was_clean_and_pushed(tmp_path): + remote, repo = tmp_path / "remote.git", tmp_path / "repo" + _git(tmp_path, "init", "--bare", "-b", "main", str(remote)) + _git(tmp_path, "init", "-b", "main", str(repo)) + _git(repo, "remote", "add", "origin", str(remote)) + (repo / "a.py").write_text("x = 1\n") + _git(repo, "add", "a.py") + _git(repo, "commit", "-m", "a") + + assert code_version(repo)["pushed"] is False + + _git(repo, "push", "-u", "origin", "main") + (repo / "a.py").write_text("x = 2\n") + assert code_version(repo)["clean"] is False + + _git(repo, "checkout", "a.py") + (repo / "notes.txt").write_text("untracked\n") + code = code_version(repo) + assert (code["clean"], code["pushed"], code["branch"]) == (True, True, "main") diff --git a/src/deployments/experiments/base_experiment.py b/src/deployments/experiments/base_experiment.py index 992ac575..3e4c3ce5 100644 --- a/src/deployments/experiments/base_experiment.py +++ b/src/deployments/experiments/base_experiment.py @@ -36,6 +36,7 @@ from src.analysis.post_run_analysis import run_post_analysis from src.analysis.utils.log_utils import log_to_path from src.deployments.core.base_bridge import BaseBridge +from src.deployments.core.code_version import code_version from src.deployments.core.k8s_cleanup import ( get_cleanup, poll_namespace_has_objects, @@ -351,6 +352,7 @@ async def run(self, *, run_post_analysis: bool = True): self._deployed.clear() self._setup_log_paths() self._dump_initial_metadata() + self.log_metadata({"code": code_version()}) with log_to_path(self.out_log_path): with ExitStack() as self._stack: From 8bb1bdffceada6743c467d9f5b68fa4b110d223d Mon Sep 17 00:00:00 2001 From: radiken Date: Thu, 12 Nov 2026 20:44:37 +0000 Subject: [PATCH 3/3] Publish: check the existing experiment on a 409 A 409 used to count as published, so a run whose row check never ran (the check failed after the create) stayed unverified. Now a 409 finds the experiment by title and runs the same check; an incomplete one is removed so the next retry recreates it. Co-Authored-By: Claude Opus 5.5 --- src/analysis/post_run/publish.py | 33 +++++++++++++-------- src/analysis/post_run/tests/test_publish.py | 31 +++++++++---------- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/src/analysis/post_run/publish.py b/src/analysis/post_run/publish.py index 1fc2ca3d..a8f91937 100644 --- a/src/analysis/post_run/publish.py +++ b/src/analysis/post_run/publish.py @@ -57,30 +57,39 @@ def experiment_body(exp: dict, config: ScrapeConfig, run_name: str) -> dict: } -def _empty_datasets(datasets: dict) -> List[str]: - return [d["name"] for d in datasets["datasets"] if not d.get("rowCount")] +def _existing_id(title: str, url: str) -> str: + response = requests.get(f"{url}/experiments", timeout=TIMEOUT_S) + response.raise_for_status() + ids = [e["id"] for e in response.json() if e["title"] == title] + if not ids: + raise PublishError(f"`{title}` got a 409 but is not on the dashboard") + return ids[0] + + +def _empty_datasets(experiment_id: str, url: str) -> List[str]: + response = requests.get(f"{url}/experiments/{experiment_id}/datasets", timeout=TIMEOUT_S) + response.raise_for_status() + return [d["name"] for d in response.json()["datasets"] if not d.get("rowCount")] -def publish(body: dict, token: str, url: str = DASHBOARD_URL) -> Optional[str]: - """Create the experiment and check every dataset got rows; returns None if already there.""" +def publish(body: dict, token: str, url: str = DASHBOARD_URL) -> str: + """Create the experiment, or find the one already there, and check every dataset has rows.""" headers = {"Authorization": f"Bearer {token}"} response = requests.post(f"{url}/experiments", json=body, headers=headers, timeout=TIMEOUT_S) if response.status_code == 409: - logger.info(f"`{body['title']}` is already on the dashboard") - return None - response.raise_for_status() - experiment_id = response.json()["id"] + experiment_id = _existing_id(body["title"], url) + else: + response.raise_for_status() + experiment_id = response.json()["id"] - response = requests.get(f"{url}/experiments/{experiment_id}/datasets", timeout=TIMEOUT_S) - response.raise_for_status() - empty = _empty_datasets(response.json()) + empty = _empty_datasets(experiment_id, url) if empty: requests.delete( f"{url}/experiments/{experiment_id}", headers=headers, timeout=TIMEOUT_S ).raise_for_status() raise PublishError(f"`{body['title']}` got no rows for {empty}; removed it") - logger.info(f"Published `{body['title']}` to the dashboard as {experiment_id}") + logger.info(f"`{body['title']}` is on the dashboard as {experiment_id}") return experiment_id diff --git a/src/analysis/post_run/tests/test_publish.py b/src/analysis/post_run/tests/test_publish.py index 487f9767..18a25537 100644 --- a/src/analysis/post_run/tests/test_publish.py +++ b/src/analysis/post_run/tests/test_publish.py @@ -46,24 +46,21 @@ def test_each_scraped_metric_is_a_dataset_over_the_stable_window(): assert {"instance", "type", "dir"} <= set(open_streams) -def test_a_dataset_without_rows_fails_the_publish_and_is_removed(monkeypatch): +@pytest.mark.parametrize("created", [True, False]) +def test_a_dataset_without_rows_fails_the_publish_and_is_removed(monkeypatch, created): deleted = [] - monkeypatch.setattr( - publish_module.requests, "post", lambda *a, **k: _Response(201, {"id": "e1"}) - ) - monkeypatch.setattr( - publish_module.requests, - "get", - lambda *a, **k: _Response( - 200, - { - "datasets": [ - {"name": "libp2p-in", "rowCount": 7}, - {"name": "mesh-peers", "rowCount": 0}, - ] - }, - ), - ) + gets = { + "http://dash/experiments": [{"id": "e0", "title": "other"}, {"id": "e1", "title": "run"}], + "http://dash/experiments/e1/datasets": { + "datasets": [ + {"name": "libp2p-in", "rowCount": 7}, + {"name": "mesh-peers", "rowCount": 0}, + ] + }, + } + post = _Response(201, {"id": "e1"}) if created else _Response(409) + monkeypatch.setattr(publish_module.requests, "post", lambda *a, **k: post) + monkeypatch.setattr(publish_module.requests, "get", lambda url, **k: _Response(200, gets[url])) monkeypatch.setattr( publish_module.requests, "delete", lambda url, **k: deleted.append(url) or _Response(204) )