From 8acf0e8f5e334628175f4814ecf575637bb91897 Mon Sep 17 00:00:00 2001 From: radiken Date: Thu, 12 Nov 2026 18:58:34 +0000 Subject: [PATCH 1/2] Dashboard: fail a create on empty datasets, guard clears, raise the VM timeout A create whose datasets fetched no rows now returns 502 and removes the experiment. process_experiment always returned the id, so the old rollback never ran. The clear-all endpoint is gone, and a single clear is refused once the dataset's window is past the 6-month retention, since no reprocess could fetch it again. The dashboard's VM queries wait 120 s instead of 30 s; cold queries on 1k-node runs took longer than 30 s. Co-Authored-By: Claude Opus 5.5 --- dst_dashboard/api/admin.py | 40 +++++------ dst_dashboard/api/experiments.py | 21 +++--- dst_dashboard/api/tests/__init__.py | 0 dst_dashboard/api/tests/test_stored_data.py | 67 +++++++++++++++++++ dst_dashboard/config/constants.py | 4 ++ dst_dashboard/processors/dataset_processor.py | 4 +- dst_dashboard/storage/db.py | 7 -- src/analysis/metrics/scrape_utils.py | 4 +- 8 files changed, 110 insertions(+), 37 deletions(-) create mode 100644 dst_dashboard/api/tests/__init__.py create mode 100644 dst_dashboard/api/tests/test_stored_data.py diff --git a/dst_dashboard/api/admin.py b/dst_dashboard/api/admin.py index 783c0d0b..b41a5415 100644 --- a/dst_dashboard/api/admin.py +++ b/dst_dashboard/api/admin.py @@ -1,12 +1,15 @@ """Admin API routes for configuration and data management.""" import logging +from datetime import datetime, timezone from pathlib import Path from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import HTMLResponse from dst_dashboard.auth import create_admin_token, require_admin_token +from dst_dashboard.config.constants import VICTORIA_RETENTION +from dst_dashboard.config.data_structures import ExperimentConfig from dst_dashboard.config.utils import LoadConfig from dst_dashboard.processors.experiment_processor import ExperimentProcessor from dst_dashboard.storage.db import DSTDatabase @@ -52,12 +55,29 @@ def reload_config(request: Request, _: None = Depends(require_admin_token)): raise HTTPException(status_code=500, detail=f"Failed to reload configuration: {str(e)}") +def _past_retention(start: datetime) -> bool: + if start.tzinfo is None: + start = start.replace(tzinfo=timezone.utc) + return datetime.now(timezone.utc) - start > VICTORIA_RETENTION + + @router.delete("/datasets/{experiment_id}/{dataset_name}") def clear_dataset(experiment_id: str, dataset_name: str, _: None = Depends(require_admin_token)): - """Clear cached dataset data to force re-fetch on next request.""" + """Clear cached dataset data to force re-fetch on next reprocess.""" try: db = DSTDatabase() + experiment_data = db.get_experiment(experiment_id) + if experiment_data is None: + raise HTTPException(status_code=404, detail="Experiment not found") + datasets = ExperimentConfig(**experiment_data).datasets + dataset = next((d for d in datasets if d.name == dataset_name), None) + if dataset is not None and _past_retention(dataset.timeRange.start): + raise HTTPException( + status_code=409, + detail=f"Dataset '{dataset_name}' is past retention and could not be re-fetched", + ) + if db.delete_dataset(experiment_id, dataset_name): logger.info(f"Cleared dataset: {experiment_id}:{dataset_name}") return { @@ -74,22 +94,6 @@ def clear_dataset(experiment_id: str, dataset_name: str, _: None = Depends(requi raise HTTPException(status_code=500, detail=f"Failed to clear dataset: {str(e)}") -@router.delete("/datasets") -def clear_all_datasets(_: None = Depends(require_admin_token)): - """Clear all cached dataset data to force re-fetch.""" - try: - db = DSTDatabase() - count = db.clear_all_dataset_cache() - - logger.info(f"Cleared {count} datasets") - - return {"status": "success", "message": f"Cleared {count} datasets successfully"} - - except Exception as e: - logger.error(f"Failed to clear datasets: {e}") - raise HTTPException(status_code=500, detail=f"Failed to clear datasets: {str(e)}") - - @router.post("/experiments/{experiment_id}/reprocess") def reprocess_experiment( experiment_id: str, request: Request, _: None = Depends(require_admin_token) @@ -107,8 +111,6 @@ def reprocess_experiment( config = request.app.state.config # Initialize processor - from dst_dashboard.config.data_structures import ExperimentConfig - processor = ExperimentProcessor(config, db) experiment = ExperimentConfig(**experiment_data) diff --git a/dst_dashboard/api/experiments.py b/dst_dashboard/api/experiments.py index cc60445a..cdc649bb 100644 --- a/dst_dashboard/api/experiments.py +++ b/dst_dashboard/api/experiments.py @@ -117,15 +117,20 @@ def create_experiment( # Process the experiment (fetch datasets, generate panels) processor = get_processor(request) - success = processor.process_experiment(experiment) - - if not success: - # Rollback - delete from DB if processing failed + try: + processor.process_experiment(experiment) + empty = [ + d.name + for d in experiment.datasets + if not (db.get_dataset_metadata(experiment.id, d.name) or {}).get("row_count") + ] + if empty: + raise HTTPException( + status_code=502, detail=f"No rows fetched for datasets {empty}; nothing was stored" + ) + except Exception: db.delete_experiment(experiment.id) - raise HTTPException( - status_code=500, - detail="Failed to process experiment. Check that all datasets and datasources are valid.", - ) + raise return experiment diff --git a/dst_dashboard/api/tests/__init__.py b/dst_dashboard/api/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dst_dashboard/api/tests/test_stored_data.py b/dst_dashboard/api/tests/test_stored_data.py new file mode 100644 index 00000000..f8c752fe --- /dev/null +++ b/dst_dashboard/api/tests/test_stored_data.py @@ -0,0 +1,67 @@ +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock + +import pytest +from fastapi import HTTPException + +from dst_dashboard.api import admin, experiments +from dst_dashboard.config.data_structures import ExperimentConfig + + +def _experiment(start: datetime) -> dict: + dataset = { + "datasource": "victoria-metrics", + "timeRange": { + "start": start.isoformat(), + "end": (start + timedelta(minutes=1)).isoformat(), + }, + "query": {"expr": "up", "step": "15s"}, + "schema": [], + } + return { + "id": "e1", + "title": "run", + "family": "nimlibp2p", + "metadata": {}, + "datasets": [{**dataset, "name": "full"}, {**dataset, "name": "empty"}], + "panels": [], + "publish": False, + } + + +@pytest.mark.parametrize("rows, kept", [(5, True), (0, False)]) +def test_a_create_is_kept_only_if_every_dataset_has_rows(monkeypatch, rows, kept): + db = MagicMock() + db.experiments.find_one.return_value = None + db.get_dataset_metadata.side_effect = lambda _, name: { + "row_count": rows if name == "empty" else 5 + } + monkeypatch.setattr(experiments, "DSTDatabase", lambda: db) + monkeypatch.setattr(experiments, "get_processor", lambda request: MagicMock()) + experiment = ExperimentConfig(**_experiment(datetime(2026, 9, 16, 10, 10))) + + if kept: + assert experiments.create_experiment(experiment, request=None) is experiment + db.delete_experiment.assert_not_called() + else: + with pytest.raises(HTTPException) as error: + experiments.create_experiment(experiment, request=None) + assert error.value.status_code == 502 and "empty" in error.value.detail + db.delete_experiment.assert_called_once_with(experiment.id) + + +@pytest.mark.parametrize("age_days, cleared", [(30, True), (200, False)]) +def test_a_dataset_past_retention_is_not_cleared(monkeypatch, age_days, cleared): + start = (datetime.now(timezone.utc) - timedelta(days=age_days)).replace(tzinfo=None) + db = MagicMock() + db.get_experiment.return_value = _experiment(start) + monkeypatch.setattr(admin, "DSTDatabase", lambda: db) + + if cleared: + assert admin.clear_dataset("e1", "full")["status"] == "success" + db.delete_dataset.assert_called_once_with("e1", "full") + else: + with pytest.raises(HTTPException) as error: + admin.clear_dataset("e1", "full") + assert error.value.status_code == 409 + db.delete_dataset.assert_not_called() diff --git a/dst_dashboard/config/constants.py b/dst_dashboard/config/constants.py index 3091f4f1..66a4060f 100644 --- a/dst_dashboard/config/constants.py +++ b/dst_dashboard/config/constants.py @@ -1,7 +1,11 @@ import os +from datetime import timedelta from enum import StrEnum from pathlib import Path +# VictoriaLogs and VictoriaMetrics keep 6 months (retentionPeriod=6). +VICTORIA_RETENTION = timedelta(days=180) + DEFAULT_CONFIG_PATH = Path.home() / ".cache" / "dst_dashboard" / "config.yaml" DEFAULT_DATA_PATH = Path.home() / ".cache" / "dst_dashboard" / "data" DEFAULT_MONGO_URI = "mongodb://localhost:27017" diff --git a/dst_dashboard/processors/dataset_processor.py b/dst_dashboard/processors/dataset_processor.py index ec06c4c9..cdb9546a 100644 --- a/dst_dashboard/processors/dataset_processor.py +++ b/dst_dashboard/processors/dataset_processor.py @@ -20,6 +20,8 @@ logger = logging.getLogger(__name__) +VM_QUERY_TIMEOUT_S = 120 + _victorialogs_log_init_lock = threading.Lock() _victorialogs_log_initialized = False @@ -329,7 +331,7 @@ def _fetch_from_prometheus( try: # Execute query - result = scrape_utils.get_query_data(promql_url) + result = scrape_utils.get_query_data(promql_url, timeout=VM_QUERY_TIMEOUT_S) match result: case Ok(data): diff --git a/dst_dashboard/storage/db.py b/dst_dashboard/storage/db.py index bd471dfc..45626fb3 100644 --- a/dst_dashboard/storage/db.py +++ b/dst_dashboard/storage/db.py @@ -199,13 +199,6 @@ def delete_panel(self, experiment_id: str, panel_name: str) -> bool: panel_id = f"{experiment_id}:{panel_name}" return self._delete_gridfs_file(self.panel_fs, panel_id) - def clear_all_dataset_cache(self) -> int: - """Delete all cached dataset data across every experiment. Returns count removed.""" - count = self.db["datasets.files"].count_documents({}) - self.db["datasets.chunks"].delete_many({}) - self.db["datasets.files"].delete_many({}) - return count - @staticmethod def _delete_gridfs_file(bucket: GridFSBucket, filename: str) -> bool: """Delete a GridFS file by filename. Returns True if a file was found and deleted.""" diff --git a/src/analysis/metrics/scrape_utils.py b/src/analysis/metrics/scrape_utils.py index a14cb834..363dae4b 100644 --- a/src/analysis/metrics/scrape_utils.py +++ b/src/analysis/metrics/scrape_utils.py @@ -27,8 +27,8 @@ def create_promql( return promql -def get_query_data(request: str) -> Result[Dict, str]: - response = urllib.request.urlopen(request, timeout=30) +def get_query_data(request: str, timeout: float = 30) -> Result[Dict, str]: + response = urllib.request.urlopen(request, timeout=timeout) logger.debug(f"Response: {response.status}") if response.status != 200: From 5acb937e7a5f95037653c9899d2008360d0defee Mon Sep 17 00:00:00 2001 From: radiken Date: Tue, 17 Nov 2026 11:27:12 +0000 Subject: [PATCH 2/2] Dashboard: leave failed fetches unstored, check rows on update and reprocess A fetch that fails or returns nothing no longer stores [], which counted as fetched and was never retried; the next reprocess fetches it again. Update and reprocess now fail with the datasets that have no rows, as create does (update's rollback never ran). VM queries refuse partial answers, since stored rows are final. The 120 s VM timeout is dropped: vmselect caps queries at its default 30 s, so it had no effect. Co-Authored-By: Claude Opus 5.5 --- dst_dashboard/api/admin.py | 4 ++ dst_dashboard/api/experiments.py | 36 +++++++++------- dst_dashboard/api/tests/test_stored_data.py | 43 ++++++++++++++++++- dst_dashboard/config/constants.py | 2 +- dst_dashboard/processors/dataset_processor.py | 6 +-- .../processors/experiment_processor.py | 10 ++--- src/analysis/metrics/scrape_utils.py | 4 +- 7 files changed, 76 insertions(+), 29 deletions(-) diff --git a/dst_dashboard/api/admin.py b/dst_dashboard/api/admin.py index b41a5415..200b2087 100644 --- a/dst_dashboard/api/admin.py +++ b/dst_dashboard/api/admin.py @@ -7,6 +7,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import HTMLResponse +from dst_dashboard.api.experiments import missing_datasets, retry_detail from dst_dashboard.auth import create_admin_token, require_admin_token from dst_dashboard.config.constants import VICTORIA_RETENTION from dst_dashboard.config.data_structures import ExperimentConfig @@ -120,6 +121,9 @@ def reprocess_experiment( # Process the experiment (this will fetch datasets and transform panels). # process_experiment always returns the experiment ID or raises an exception on failure. processor.process_experiment(experiment) + missing = missing_datasets(db, experiment) + if missing: + raise HTTPException(status_code=502, detail=retry_detail(experiment_id, missing)) return { "status": "success", diff --git a/dst_dashboard/api/experiments.py b/dst_dashboard/api/experiments.py index cdc649bb..d82ce1db 100644 --- a/dst_dashboard/api/experiments.py +++ b/dst_dashboard/api/experiments.py @@ -11,6 +11,22 @@ router = APIRouter(prefix="/experiments", tags=["experiments"]) +def missing_datasets(db: DSTDatabase, experiment: ExperimentConfig) -> List[str]: + """Datasets of `experiment` with no stored rows.""" + return [ + d.name + for d in experiment.datasets + if not (db.get_dataset_metadata(experiment.id, d.name) or {}).get("row_count") + ] + + +def retry_detail(experiment_id: str, missing: List[str]) -> str: + return ( + f"No rows fetched for datasets {missing}; " + f"POST /admin/experiments/{experiment_id}/reprocess fetches them again" + ) + + @router.get("/families") def list_families(request: Request) -> Dict[str, Any]: """ @@ -119,11 +135,7 @@ def create_experiment( processor = get_processor(request) try: processor.process_experiment(experiment) - empty = [ - d.name - for d in experiment.datasets - if not (db.get_dataset_metadata(experiment.id, d.name) or {}).get("row_count") - ] + empty = missing_datasets(db, experiment) if empty: raise HTTPException( status_code=502, detail=f"No rows fetched for datasets {empty}; nothing was stored" @@ -185,16 +197,10 @@ def update_experiment( # Reprocess if configuration changed if needs_reprocessing: - processor = get_processor(request) - success = processor.process_experiment(experiment) - - if not success: - # Rollback - restore old experiment - db.store_experiment(existing.model_dump()) - raise HTTPException( - status_code=500, - detail="Failed to reprocess experiment. Rolled back to previous configuration.", - ) + get_processor(request).process_experiment(experiment) + missing = missing_datasets(db, experiment) + if missing: + raise HTTPException(status_code=502, detail=retry_detail(experiment_id, missing)) return experiment diff --git a/dst_dashboard/api/tests/test_stored_data.py b/dst_dashboard/api/tests/test_stored_data.py index f8c752fe..ea165bf0 100644 --- a/dst_dashboard/api/tests/test_stored_data.py +++ b/dst_dashboard/api/tests/test_stored_data.py @@ -5,7 +5,12 @@ from fastapi import HTTPException from dst_dashboard.api import admin, experiments -from dst_dashboard.config.data_structures import ExperimentConfig +from dst_dashboard.config.data_structures import ( + DashboardFullConfig, + DatasetConfig, + ExperimentConfig, +) +from dst_dashboard.processors.experiment_processor import ExperimentProcessor def _experiment(start: datetime) -> dict: @@ -65,3 +70,39 @@ def test_a_dataset_past_retention_is_not_cleared(monkeypatch, age_days, cleared) admin.clear_dataset("e1", "full") assert error.value.status_code == 409 db.delete_dataset.assert_not_called() + + +@pytest.mark.parametrize("fetched", [[], TimeoutError("timed out")]) +def test_a_fetch_with_no_rows_is_left_unstored_for_the_next_reprocess(monkeypatch, fetched): + db = MagicMock() + db.get_dataset.return_value = None + processor = ExperimentProcessor(DashboardFullConfig(datasources=[]), db) + + def fetch(*args): + if isinstance(fetched, Exception): + raise fetched + return fetched + + monkeypatch.setattr(processor, "fetch_dataset", fetch) + dataset = DatasetConfig(**_experiment(datetime(2026, 9, 16, 10, 10))["datasets"][0]) + + assert processor.process_dataset("e1", dataset) is False + db.store_dataset.assert_not_called() + + +@pytest.mark.parametrize("rows", [5, 0]) +def test_a_reprocess_that_leaves_a_dataset_empty_fails(monkeypatch, rows): + db = MagicMock() + db.get_experiment.return_value = _experiment(datetime(2026, 9, 16, 10, 10)) + db.get_dataset_metadata.side_effect = lambda _, name: { + "row_count": rows if name == "empty" else 5 + } + monkeypatch.setattr(admin, "DSTDatabase", lambda: db) + monkeypatch.setattr(admin, "ExperimentProcessor", lambda config, db: MagicMock()) + + if rows: + assert admin.reprocess_experiment("e1", request=MagicMock())["status"] == "success" + return + with pytest.raises(HTTPException) as error: + admin.reprocess_experiment("e1", request=MagicMock()) + assert error.value.status_code == 502 and "'empty'" in error.value.detail diff --git a/dst_dashboard/config/constants.py b/dst_dashboard/config/constants.py index 66a4060f..18e03272 100644 --- a/dst_dashboard/config/constants.py +++ b/dst_dashboard/config/constants.py @@ -3,7 +3,7 @@ from enum import StrEnum from pathlib import Path -# VictoriaLogs and VictoriaMetrics keep 6 months (retentionPeriod=6). +# Under the lab's VictoriaLogs/VictoriaMetrics retention: retentionPeriod=6 months of 31 days. VICTORIA_RETENTION = timedelta(days=180) DEFAULT_CONFIG_PATH = Path.home() / ".cache" / "dst_dashboard" / "config.yaml" diff --git a/dst_dashboard/processors/dataset_processor.py b/dst_dashboard/processors/dataset_processor.py index cdb9546a..096ef87f 100644 --- a/dst_dashboard/processors/dataset_processor.py +++ b/dst_dashboard/processors/dataset_processor.py @@ -20,8 +20,6 @@ logger = logging.getLogger(__name__) -VM_QUERY_TIMEOUT_S = 120 - _victorialogs_log_init_lock = threading.Lock() _victorialogs_log_initialized = False @@ -326,12 +324,14 @@ def _fetch_from_prometheus( dataset_config.timeRange.end, step_value, ) + # Stored rows are final, so a partial answer must fail instead. + promql_url += "&deny_partial_response=1" logger.debug(f"Prometheus query URL: {promql_url}") try: # Execute query - result = scrape_utils.get_query_data(promql_url, timeout=VM_QUERY_TIMEOUT_S) + result = scrape_utils.get_query_data(promql_url) match result: case Ok(data): diff --git a/dst_dashboard/processors/experiment_processor.py b/dst_dashboard/processors/experiment_processor.py index e5f278a4..e1d6d150 100644 --- a/dst_dashboard/processors/experiment_processor.py +++ b/dst_dashboard/processors/experiment_processor.py @@ -55,16 +55,12 @@ def process_dataset(self, experiment_id: str, dataset_config: DatasetConfig) -> self.db.store_dataset(experiment_id, dataset_config.name, data) logger.info(f"Stored {len(data)} rows for dataset '{dataset_config.name}'") return True - else: - logger.warning(f"No data fetched for dataset '{dataset_config.name}'") - # Store empty dataset to mark it as attempted - self.db.store_dataset(experiment_id, dataset_config.name, []) - return False + # Left unstored, so the next reprocess fetches it again. + logger.warning(f"No data fetched for dataset '{dataset_config.name}'") + return False except Exception as e: logger.error(f"Failed to fetch dataset '{dataset_config.name}': {e}") - # Store empty dataset to mark it as processed but failed - self.db.store_dataset(experiment_id, dataset_config.name, []) return False def process_experiment_datasets( diff --git a/src/analysis/metrics/scrape_utils.py b/src/analysis/metrics/scrape_utils.py index 363dae4b..a14cb834 100644 --- a/src/analysis/metrics/scrape_utils.py +++ b/src/analysis/metrics/scrape_utils.py @@ -27,8 +27,8 @@ def create_promql( return promql -def get_query_data(request: str, timeout: float = 30) -> Result[Dict, str]: - response = urllib.request.urlopen(request, timeout=timeout) +def get_query_data(request: str) -> Result[Dict, str]: + response = urllib.request.urlopen(request, timeout=30) logger.debug(f"Response: {response.status}") if response.status != 200: