Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 25 additions & 19 deletions dst_dashboard/api/admin.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
"""Admin API routes for configuration and data management."""

import logging
from datetime import datetime, timezone
from pathlib import Path

from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi.responses import HTMLResponse

from dst_dashboard.api.experiments import missing_datasets, retry_detail
from dst_dashboard.auth import create_admin_token, require_admin_token
from dst_dashboard.config.constants import VICTORIA_RETENTION
from dst_dashboard.config.data_structures import ExperimentConfig
from dst_dashboard.config.utils import LoadConfig
from dst_dashboard.processors.experiment_processor import ExperimentProcessor
from dst_dashboard.storage.db import DSTDatabase
Expand Down Expand Up @@ -52,12 +56,29 @@ def reload_config(request: Request, _: None = Depends(require_admin_token)):
raise HTTPException(status_code=500, detail=f"Failed to reload configuration: {str(e)}")


def _past_retention(start: datetime) -> bool:
if start.tzinfo is None:
start = start.replace(tzinfo=timezone.utc)
return datetime.now(timezone.utc) - start > VICTORIA_RETENTION


@router.delete("/datasets/{experiment_id}/{dataset_name}")
def clear_dataset(experiment_id: str, dataset_name: str, _: None = Depends(require_admin_token)):
"""Clear cached dataset data to force re-fetch on next request."""
"""Clear cached dataset data to force re-fetch on next reprocess."""
try:
db = DSTDatabase()

experiment_data = db.get_experiment(experiment_id)
if experiment_data is None:
raise HTTPException(status_code=404, detail="Experiment not found")
datasets = ExperimentConfig(**experiment_data).datasets
dataset = next((d for d in datasets if d.name == dataset_name), None)
if dataset is not None and _past_retention(dataset.timeRange.start):
raise HTTPException(
status_code=409,
detail=f"Dataset '{dataset_name}' is past retention and could not be re-fetched",
)

if db.delete_dataset(experiment_id, dataset_name):
logger.info(f"Cleared dataset: {experiment_id}:{dataset_name}")
return {
Expand All @@ -74,22 +95,6 @@ def clear_dataset(experiment_id: str, dataset_name: str, _: None = Depends(requi
raise HTTPException(status_code=500, detail=f"Failed to clear dataset: {str(e)}")


@router.delete("/datasets")
def clear_all_datasets(_: None = Depends(require_admin_token)):
"""Clear all cached dataset data to force re-fetch."""
try:
db = DSTDatabase()
count = db.clear_all_dataset_cache()

logger.info(f"Cleared {count} datasets")

return {"status": "success", "message": f"Cleared {count} datasets successfully"}

except Exception as e:
logger.error(f"Failed to clear datasets: {e}")
raise HTTPException(status_code=500, detail=f"Failed to clear datasets: {str(e)}")


@router.post("/experiments/{experiment_id}/reprocess")
def reprocess_experiment(
experiment_id: str, request: Request, _: None = Depends(require_admin_token)
Expand All @@ -107,8 +112,6 @@ def reprocess_experiment(
config = request.app.state.config

# Initialize processor
from dst_dashboard.config.data_structures import ExperimentConfig

processor = ExperimentProcessor(config, db)

experiment = ExperimentConfig(**experiment_data)
Expand All @@ -118,6 +121,9 @@ def reprocess_experiment(
# Process the experiment (this will fetch datasets and transform panels).
# process_experiment always returns the experiment ID or raises an exception on failure.
processor.process_experiment(experiment)
missing = missing_datasets(db, experiment)
if missing:
raise HTTPException(status_code=502, detail=retry_detail(experiment_id, missing))

return {
"status": "success",
Expand Down
47 changes: 29 additions & 18 deletions dst_dashboard/api/experiments.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,22 @@
router = APIRouter(prefix="/experiments", tags=["experiments"])


def missing_datasets(db: DSTDatabase, experiment: ExperimentConfig) -> List[str]:
"""Datasets of `experiment` with no stored rows."""
return [
d.name
for d in experiment.datasets
if not (db.get_dataset_metadata(experiment.id, d.name) or {}).get("row_count")
]


def retry_detail(experiment_id: str, missing: List[str]) -> str:
return (
f"No rows fetched for datasets {missing}; "
f"POST /admin/experiments/{experiment_id}/reprocess fetches them again"
)


@router.get("/families")
def list_families(request: Request) -> Dict[str, Any]:
"""
Expand Down Expand Up @@ -117,15 +133,16 @@ def create_experiment(

# Process the experiment (fetch datasets, generate panels)
processor = get_processor(request)
success = processor.process_experiment(experiment)

if not success:
# Rollback - delete from DB if processing failed
try:
processor.process_experiment(experiment)
empty = missing_datasets(db, experiment)
if empty:
raise HTTPException(
status_code=502, detail=f"No rows fetched for datasets {empty}; nothing was stored"
)
except Exception:
db.delete_experiment(experiment.id)
raise HTTPException(
status_code=500,
detail="Failed to process experiment. Check that all datasets and datasources are valid.",
)
raise

return experiment

Expand Down Expand Up @@ -180,16 +197,10 @@ def update_experiment(

# Reprocess if configuration changed
if needs_reprocessing:
processor = get_processor(request)
success = processor.process_experiment(experiment)

if not success:
# Rollback - restore old experiment
db.store_experiment(existing.model_dump())
raise HTTPException(
status_code=500,
detail="Failed to reprocess experiment. Rolled back to previous configuration.",
)
get_processor(request).process_experiment(experiment)
missing = missing_datasets(db, experiment)
if missing:
raise HTTPException(status_code=502, detail=retry_detail(experiment_id, missing))

return experiment

Expand Down
Empty file.
108 changes: 108 additions & 0 deletions dst_dashboard/api/tests/test_stored_data.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
from datetime import datetime, timedelta, timezone
from unittest.mock import MagicMock

import pytest
from fastapi import HTTPException

from dst_dashboard.api import admin, experiments
from dst_dashboard.config.data_structures import (
DashboardFullConfig,
DatasetConfig,
ExperimentConfig,
)
from dst_dashboard.processors.experiment_processor import ExperimentProcessor


def _experiment(start: datetime) -> dict:
dataset = {
"datasource": "victoria-metrics",
"timeRange": {
"start": start.isoformat(),
"end": (start + timedelta(minutes=1)).isoformat(),
},
"query": {"expr": "up", "step": "15s"},
"schema": [],
}
return {
"id": "e1",
"title": "run",
"family": "nimlibp2p",
"metadata": {},
"datasets": [{**dataset, "name": "full"}, {**dataset, "name": "empty"}],
"panels": [],
"publish": False,
}


@pytest.mark.parametrize("rows, kept", [(5, True), (0, False)])
def test_a_create_is_kept_only_if_every_dataset_has_rows(monkeypatch, rows, kept):
db = MagicMock()
db.experiments.find_one.return_value = None
db.get_dataset_metadata.side_effect = lambda _, name: {
"row_count": rows if name == "empty" else 5
}
monkeypatch.setattr(experiments, "DSTDatabase", lambda: db)
monkeypatch.setattr(experiments, "get_processor", lambda request: MagicMock())
experiment = ExperimentConfig(**_experiment(datetime(2026, 9, 16, 10, 10)))

if kept:
assert experiments.create_experiment(experiment, request=None) is experiment
db.delete_experiment.assert_not_called()
else:
with pytest.raises(HTTPException) as error:
experiments.create_experiment(experiment, request=None)
assert error.value.status_code == 502 and "empty" in error.value.detail
db.delete_experiment.assert_called_once_with(experiment.id)


@pytest.mark.parametrize("age_days, cleared", [(30, True), (200, False)])
def test_a_dataset_past_retention_is_not_cleared(monkeypatch, age_days, cleared):
start = (datetime.now(timezone.utc) - timedelta(days=age_days)).replace(tzinfo=None)
db = MagicMock()
db.get_experiment.return_value = _experiment(start)
monkeypatch.setattr(admin, "DSTDatabase", lambda: db)

if cleared:
assert admin.clear_dataset("e1", "full")["status"] == "success"
db.delete_dataset.assert_called_once_with("e1", "full")
else:
with pytest.raises(HTTPException) as error:
admin.clear_dataset("e1", "full")
assert error.value.status_code == 409
db.delete_dataset.assert_not_called()


@pytest.mark.parametrize("fetched", [[], TimeoutError("timed out")])
def test_a_fetch_with_no_rows_is_left_unstored_for_the_next_reprocess(monkeypatch, fetched):
db = MagicMock()
db.get_dataset.return_value = None
processor = ExperimentProcessor(DashboardFullConfig(datasources=[]), db)

def fetch(*args):
if isinstance(fetched, Exception):
raise fetched
return fetched

monkeypatch.setattr(processor, "fetch_dataset", fetch)
dataset = DatasetConfig(**_experiment(datetime(2026, 9, 16, 10, 10))["datasets"][0])

assert processor.process_dataset("e1", dataset) is False
db.store_dataset.assert_not_called()


@pytest.mark.parametrize("rows", [5, 0])
def test_a_reprocess_that_leaves_a_dataset_empty_fails(monkeypatch, rows):
db = MagicMock()
db.get_experiment.return_value = _experiment(datetime(2026, 9, 16, 10, 10))
db.get_dataset_metadata.side_effect = lambda _, name: {
"row_count": rows if name == "empty" else 5
}
monkeypatch.setattr(admin, "DSTDatabase", lambda: db)
monkeypatch.setattr(admin, "ExperimentProcessor", lambda config, db: MagicMock())

if rows:
assert admin.reprocess_experiment("e1", request=MagicMock())["status"] == "success"
return
with pytest.raises(HTTPException) as error:
admin.reprocess_experiment("e1", request=MagicMock())
assert error.value.status_code == 502 and "'empty'" in error.value.detail
4 changes: 4 additions & 0 deletions dst_dashboard/config/constants.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
import os
from datetime import timedelta
from enum import StrEnum
from pathlib import Path

# Under the lab's VictoriaLogs/VictoriaMetrics retention: retentionPeriod=6 months of 31 days.
VICTORIA_RETENTION = timedelta(days=180)

DEFAULT_CONFIG_PATH = Path.home() / ".cache" / "dst_dashboard" / "config.yaml"
DEFAULT_DATA_PATH = Path.home() / ".cache" / "dst_dashboard" / "data"
DEFAULT_MONGO_URI = "mongodb://localhost:27017"
Expand Down
2 changes: 2 additions & 0 deletions dst_dashboard/processors/dataset_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,8 @@ def _fetch_from_prometheus(
dataset_config.timeRange.end,
step_value,
)
# Stored rows are final, so a partial answer must fail instead.
promql_url += "&deny_partial_response=1"

logger.debug(f"Prometheus query URL: {promql_url}")

Expand Down
10 changes: 3 additions & 7 deletions dst_dashboard/processors/experiment_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,16 +55,12 @@ def process_dataset(self, experiment_id: str, dataset_config: DatasetConfig) ->
self.db.store_dataset(experiment_id, dataset_config.name, data)
logger.info(f"Stored {len(data)} rows for dataset '{dataset_config.name}'")
return True
else:
logger.warning(f"No data fetched for dataset '{dataset_config.name}'")
# Store empty dataset to mark it as attempted
self.db.store_dataset(experiment_id, dataset_config.name, [])
return False
# Left unstored, so the next reprocess fetches it again.
logger.warning(f"No data fetched for dataset '{dataset_config.name}'")
return False

except Exception as e:
logger.error(f"Failed to fetch dataset '{dataset_config.name}': {e}")
# Store empty dataset to mark it as processed but failed
self.db.store_dataset(experiment_id, dataset_config.name, [])
return False

def process_experiment_datasets(
Expand Down
7 changes: 0 additions & 7 deletions dst_dashboard/storage/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,13 +199,6 @@ def delete_panel(self, experiment_id: str, panel_name: str) -> bool:
panel_id = f"{experiment_id}:{panel_name}"
return self._delete_gridfs_file(self.panel_fs, panel_id)

def clear_all_dataset_cache(self) -> int:
"""Delete all cached dataset data across every experiment. Returns count removed."""
count = self.db["datasets.files"].count_documents({})
self.db["datasets.chunks"].delete_many({})
self.db["datasets.files"].delete_many({})
return count

@staticmethod
def _delete_gridfs_file(bucket: GridFSBucket, filename: str) -> bool:
"""Delete a GridFS file by filename. Returns True if a file was found and deleted."""
Expand Down
Loading