diff --git a/dst_dashboard/api/admin.py b/dst_dashboard/api/admin.py index 783c0d0b..200b2087 100644 --- a/dst_dashboard/api/admin.py +++ b/dst_dashboard/api/admin.py @@ -1,12 +1,16 @@ """Admin API routes for configuration and data management.""" import logging +from datetime import datetime, timezone from pathlib import Path from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import HTMLResponse +from dst_dashboard.api.experiments import missing_datasets, retry_detail from dst_dashboard.auth import create_admin_token, require_admin_token +from dst_dashboard.config.constants import VICTORIA_RETENTION +from dst_dashboard.config.data_structures import ExperimentConfig from dst_dashboard.config.utils import LoadConfig from dst_dashboard.processors.experiment_processor import ExperimentProcessor from dst_dashboard.storage.db import DSTDatabase @@ -52,12 +56,29 @@ def reload_config(request: Request, _: None = Depends(require_admin_token)): raise HTTPException(status_code=500, detail=f"Failed to reload configuration: {str(e)}") +def _past_retention(start: datetime) -> bool: + if start.tzinfo is None: + start = start.replace(tzinfo=timezone.utc) + return datetime.now(timezone.utc) - start > VICTORIA_RETENTION + + @router.delete("/datasets/{experiment_id}/{dataset_name}") def clear_dataset(experiment_id: str, dataset_name: str, _: None = Depends(require_admin_token)): - """Clear cached dataset data to force re-fetch on next request.""" + """Clear cached dataset data to force re-fetch on next reprocess.""" try: db = DSTDatabase() + experiment_data = db.get_experiment(experiment_id) + if experiment_data is None: + raise HTTPException(status_code=404, detail="Experiment not found") + datasets = ExperimentConfig(**experiment_data).datasets + dataset = next((d for d in datasets if d.name == dataset_name), None) + if dataset is not None and _past_retention(dataset.timeRange.start): + raise HTTPException( + status_code=409, + detail=f"Dataset '{dataset_name}' is past retention and could not be re-fetched", + ) + if db.delete_dataset(experiment_id, dataset_name): logger.info(f"Cleared dataset: {experiment_id}:{dataset_name}") return { @@ -74,22 +95,6 @@ def clear_dataset(experiment_id: str, dataset_name: str, _: None = Depends(requi raise HTTPException(status_code=500, detail=f"Failed to clear dataset: {str(e)}") -@router.delete("/datasets") -def clear_all_datasets(_: None = Depends(require_admin_token)): - """Clear all cached dataset data to force re-fetch.""" - try: - db = DSTDatabase() - count = db.clear_all_dataset_cache() - - logger.info(f"Cleared {count} datasets") - - return {"status": "success", "message": f"Cleared {count} datasets successfully"} - - except Exception as e: - logger.error(f"Failed to clear datasets: {e}") - raise HTTPException(status_code=500, detail=f"Failed to clear datasets: {str(e)}") - - @router.post("/experiments/{experiment_id}/reprocess") def reprocess_experiment( experiment_id: str, request: Request, _: None = Depends(require_admin_token) @@ -107,8 +112,6 @@ def reprocess_experiment( config = request.app.state.config # Initialize processor - from dst_dashboard.config.data_structures import ExperimentConfig - processor = ExperimentProcessor(config, db) experiment = ExperimentConfig(**experiment_data) @@ -118,6 +121,9 @@ def reprocess_experiment( # Process the experiment (this will fetch datasets and transform panels). # process_experiment always returns the experiment ID or raises an exception on failure. processor.process_experiment(experiment) + missing = missing_datasets(db, experiment) + if missing: + raise HTTPException(status_code=502, detail=retry_detail(experiment_id, missing)) return { "status": "success", diff --git a/dst_dashboard/api/experiments.py b/dst_dashboard/api/experiments.py index cc60445a..d82ce1db 100644 --- a/dst_dashboard/api/experiments.py +++ b/dst_dashboard/api/experiments.py @@ -11,6 +11,22 @@ router = APIRouter(prefix="/experiments", tags=["experiments"]) +def missing_datasets(db: DSTDatabase, experiment: ExperimentConfig) -> List[str]: + """Datasets of `experiment` with no stored rows.""" + return [ + d.name + for d in experiment.datasets + if not (db.get_dataset_metadata(experiment.id, d.name) or {}).get("row_count") + ] + + +def retry_detail(experiment_id: str, missing: List[str]) -> str: + return ( + f"No rows fetched for datasets {missing}; " + f"POST /admin/experiments/{experiment_id}/reprocess fetches them again" + ) + + @router.get("/families") def list_families(request: Request) -> Dict[str, Any]: """ @@ -117,15 +133,16 @@ def create_experiment( # Process the experiment (fetch datasets, generate panels) processor = get_processor(request) - success = processor.process_experiment(experiment) - - if not success: - # Rollback - delete from DB if processing failed + try: + processor.process_experiment(experiment) + empty = missing_datasets(db, experiment) + if empty: + raise HTTPException( + status_code=502, detail=f"No rows fetched for datasets {empty}; nothing was stored" + ) + except Exception: db.delete_experiment(experiment.id) - raise HTTPException( - status_code=500, - detail="Failed to process experiment. Check that all datasets and datasources are valid.", - ) + raise return experiment @@ -180,16 +197,10 @@ def update_experiment( # Reprocess if configuration changed if needs_reprocessing: - processor = get_processor(request) - success = processor.process_experiment(experiment) - - if not success: - # Rollback - restore old experiment - db.store_experiment(existing.model_dump()) - raise HTTPException( - status_code=500, - detail="Failed to reprocess experiment. Rolled back to previous configuration.", - ) + get_processor(request).process_experiment(experiment) + missing = missing_datasets(db, experiment) + if missing: + raise HTTPException(status_code=502, detail=retry_detail(experiment_id, missing)) return experiment diff --git a/dst_dashboard/api/tests/__init__.py b/dst_dashboard/api/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dst_dashboard/api/tests/test_stored_data.py b/dst_dashboard/api/tests/test_stored_data.py new file mode 100644 index 00000000..ea165bf0 --- /dev/null +++ b/dst_dashboard/api/tests/test_stored_data.py @@ -0,0 +1,108 @@ +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock + +import pytest +from fastapi import HTTPException + +from dst_dashboard.api import admin, experiments +from dst_dashboard.config.data_structures import ( + DashboardFullConfig, + DatasetConfig, + ExperimentConfig, +) +from dst_dashboard.processors.experiment_processor import ExperimentProcessor + + +def _experiment(start: datetime) -> dict: + dataset = { + "datasource": "victoria-metrics", + "timeRange": { + "start": start.isoformat(), + "end": (start + timedelta(minutes=1)).isoformat(), + }, + "query": {"expr": "up", "step": "15s"}, + "schema": [], + } + return { + "id": "e1", + "title": "run", + "family": "nimlibp2p", + "metadata": {}, + "datasets": [{**dataset, "name": "full"}, {**dataset, "name": "empty"}], + "panels": [], + "publish": False, + } + + +@pytest.mark.parametrize("rows, kept", [(5, True), (0, False)]) +def test_a_create_is_kept_only_if_every_dataset_has_rows(monkeypatch, rows, kept): + db = MagicMock() + db.experiments.find_one.return_value = None + db.get_dataset_metadata.side_effect = lambda _, name: { + "row_count": rows if name == "empty" else 5 + } + monkeypatch.setattr(experiments, "DSTDatabase", lambda: db) + monkeypatch.setattr(experiments, "get_processor", lambda request: MagicMock()) + experiment = ExperimentConfig(**_experiment(datetime(2026, 9, 16, 10, 10))) + + if kept: + assert experiments.create_experiment(experiment, request=None) is experiment + db.delete_experiment.assert_not_called() + else: + with pytest.raises(HTTPException) as error: + experiments.create_experiment(experiment, request=None) + assert error.value.status_code == 502 and "empty" in error.value.detail + db.delete_experiment.assert_called_once_with(experiment.id) + + +@pytest.mark.parametrize("age_days, cleared", [(30, True), (200, False)]) +def test_a_dataset_past_retention_is_not_cleared(monkeypatch, age_days, cleared): + start = (datetime.now(timezone.utc) - timedelta(days=age_days)).replace(tzinfo=None) + db = MagicMock() + db.get_experiment.return_value = _experiment(start) + monkeypatch.setattr(admin, "DSTDatabase", lambda: db) + + if cleared: + assert admin.clear_dataset("e1", "full")["status"] == "success" + db.delete_dataset.assert_called_once_with("e1", "full") + else: + with pytest.raises(HTTPException) as error: + admin.clear_dataset("e1", "full") + assert error.value.status_code == 409 + db.delete_dataset.assert_not_called() + + +@pytest.mark.parametrize("fetched", [[], TimeoutError("timed out")]) +def test_a_fetch_with_no_rows_is_left_unstored_for_the_next_reprocess(monkeypatch, fetched): + db = MagicMock() + db.get_dataset.return_value = None + processor = ExperimentProcessor(DashboardFullConfig(datasources=[]), db) + + def fetch(*args): + if isinstance(fetched, Exception): + raise fetched + return fetched + + monkeypatch.setattr(processor, "fetch_dataset", fetch) + dataset = DatasetConfig(**_experiment(datetime(2026, 9, 16, 10, 10))["datasets"][0]) + + assert processor.process_dataset("e1", dataset) is False + db.store_dataset.assert_not_called() + + +@pytest.mark.parametrize("rows", [5, 0]) +def test_a_reprocess_that_leaves_a_dataset_empty_fails(monkeypatch, rows): + db = MagicMock() + db.get_experiment.return_value = _experiment(datetime(2026, 9, 16, 10, 10)) + db.get_dataset_metadata.side_effect = lambda _, name: { + "row_count": rows if name == "empty" else 5 + } + monkeypatch.setattr(admin, "DSTDatabase", lambda: db) + monkeypatch.setattr(admin, "ExperimentProcessor", lambda config, db: MagicMock()) + + if rows: + assert admin.reprocess_experiment("e1", request=MagicMock())["status"] == "success" + return + with pytest.raises(HTTPException) as error: + admin.reprocess_experiment("e1", request=MagicMock()) + assert error.value.status_code == 502 and "'empty'" in error.value.detail diff --git a/dst_dashboard/config/constants.py b/dst_dashboard/config/constants.py index 3091f4f1..18e03272 100644 --- a/dst_dashboard/config/constants.py +++ b/dst_dashboard/config/constants.py @@ -1,7 +1,11 @@ import os +from datetime import timedelta from enum import StrEnum from pathlib import Path +# Under the lab's VictoriaLogs/VictoriaMetrics retention: retentionPeriod=6 months of 31 days. +VICTORIA_RETENTION = timedelta(days=180) + DEFAULT_CONFIG_PATH = Path.home() / ".cache" / "dst_dashboard" / "config.yaml" DEFAULT_DATA_PATH = Path.home() / ".cache" / "dst_dashboard" / "data" DEFAULT_MONGO_URI = "mongodb://localhost:27017" diff --git a/dst_dashboard/processors/dataset_processor.py b/dst_dashboard/processors/dataset_processor.py index ec06c4c9..096ef87f 100644 --- a/dst_dashboard/processors/dataset_processor.py +++ b/dst_dashboard/processors/dataset_processor.py @@ -324,6 +324,8 @@ def _fetch_from_prometheus( dataset_config.timeRange.end, step_value, ) + # Stored rows are final, so a partial answer must fail instead. + promql_url += "&deny_partial_response=1" logger.debug(f"Prometheus query URL: {promql_url}") diff --git a/dst_dashboard/processors/experiment_processor.py b/dst_dashboard/processors/experiment_processor.py index e5f278a4..e1d6d150 100644 --- a/dst_dashboard/processors/experiment_processor.py +++ b/dst_dashboard/processors/experiment_processor.py @@ -55,16 +55,12 @@ def process_dataset(self, experiment_id: str, dataset_config: DatasetConfig) -> self.db.store_dataset(experiment_id, dataset_config.name, data) logger.info(f"Stored {len(data)} rows for dataset '{dataset_config.name}'") return True - else: - logger.warning(f"No data fetched for dataset '{dataset_config.name}'") - # Store empty dataset to mark it as attempted - self.db.store_dataset(experiment_id, dataset_config.name, []) - return False + # Left unstored, so the next reprocess fetches it again. + logger.warning(f"No data fetched for dataset '{dataset_config.name}'") + return False except Exception as e: logger.error(f"Failed to fetch dataset '{dataset_config.name}': {e}") - # Store empty dataset to mark it as processed but failed - self.db.store_dataset(experiment_id, dataset_config.name, []) return False def process_experiment_datasets( diff --git a/dst_dashboard/storage/db.py b/dst_dashboard/storage/db.py index bd471dfc..45626fb3 100644 --- a/dst_dashboard/storage/db.py +++ b/dst_dashboard/storage/db.py @@ -199,13 +199,6 @@ def delete_panel(self, experiment_id: str, panel_name: str) -> bool: panel_id = f"{experiment_id}:{panel_name}" return self._delete_gridfs_file(self.panel_fs, panel_id) - def clear_all_dataset_cache(self) -> int: - """Delete all cached dataset data across every experiment. Returns count removed.""" - count = self.db["datasets.files"].count_documents({}) - self.db["datasets.chunks"].delete_many({}) - self.db["datasets.files"].delete_many({}) - return count - @staticmethod def _delete_gridfs_file(bucket: GridFSBucket, filename: str) -> bool: """Delete a GridFS file by filename. Returns True if a file was found and deleted."""