added endpoint for data retrieval

.gitignore

@@ -21,8 +21,7 @@ test_manifest.yml
 /htmlcov
 /test-reports/
 /tests/cli/test.cfg
-/tests/cli/*.yaml
-.eggs
+/tests/cli/*.yaml.eggs
 .coverage
 *.xml
 simdb-coverage-report

pyproject.toml

@@ -44,6 +44,7 @@ dependencies = [
     "numpy>=1.14",
     "pydantic>=2.10.6",
     "python-dateutil>=2.6",
+    "plotext>=5.3.2",
     "pyyaml>=3.13",
     "requests>=2.27.0",
     "semantic-version>=2.8",

src/simdb/cli/commands/simulation.py

@@ -16,7 +16,12 @@
 from simdb.validation import ValidationError, Validator
 
 from . import check_meta_args, pass_config
-from .utils import print_simulations
+from .utils import (
+    is_numeric_1d,
+    print_quantity,
+    print_simulations,
+    show_quantity_textual_plot,
+)
 from .validators import validate_non_negative
 
 
@@ -353,6 +358,69 @@ def simulation_query(
     )
 
 
+@simulation.command("data", cls=n_required_args_adaptor(2))
+@pass_config
+@click.argument("remote", required=False)
+@click.argument("sim_id")
+@click.argument("ids_path")
+@click.option("--username", help="Username used to authenticate with the remote.")
+@click.option("--password", help="Password used to authenticate with the remote.")
+def simulation_data(
+    config: Config,
+    remote: Optional[str],
+    sim_id: str,
+    ids_path: str,
+    username: Optional[str],
+    password: Optional[str],
+):
+    """Fetch IDS field data for simulation SIM_ID (UUID or alias) from REMOTE.
+
+    \b
+    IDS_PATH format:
+        ids_name[:<occurrence>]/path/to/field
+
+    \b
+    Examples:
+        simdb sim data iter 4dd781b... profiles_1d[0]/grid/rho_tor_norm
+        simdb sim data 4dd781b... equilibrium:0/time_slice[0]/profiles_1d/psi
+    """
+    api = RemoteAPI(remote, username, password, config)
+
+    try:
+        result = api.get_simulation_data(sim_id, ids_path)
+    except Exception as err:
+        raise click.ClickException(str(err)) from err
+
+    click.echo(f"simulation : {result['simulation']}")
+    click.echo(f"path       : {result['path']}  (occurrence {result['occurrence']})")
+
+    coordinates = result.get("coordinates") or []
+    plot_coordinate = next(
+        (
+            coord
+            for coord in coordinates
+            if isinstance(coord.get("data"), list)
+            and isinstance(result["field"].get("data"), list)
+            and len(coord["data"]) == len(result["field"]["data"])
+        ),
+        None,
+    )
+    field_is_1d = is_numeric_1d(result["field"].get("data"))
+    if field_is_1d:
+        show_quantity_textual_plot(
+            result["field"], label="field", x_quantity=plot_coordinate
+        )
+    else:
+        print_quantity(result["field"], label="field")
+
+    if config.verbose and coordinates:
+        for coord in coordinates:
+            if field_is_1d and is_numeric_1d(coord.get("data")):
+                continue
+            if isinstance(coord.get("data"), list):
+                print_quantity(coord, label=f"coord  {coord['name']}", show_stats=False)
+
+
 @simulation.command("validate", cls=n_required_args_adaptor(1))
 @pass_config
 @click.argument("remote", required=False)

src/simdb/cli/commands/utils.py

@@ -1,7 +1,12 @@
+import importlib
 from collections import OrderedDict
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, TypeVar
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple, TypeVar
 
 import click
+from rich.console import Console, Group
+from rich.panel import Panel
+from rich.table import Table
+from rich.text import Text
 
 if TYPE_CHECKING:
     # Only importing these for type checking and documentation generation in order to
@@ -10,6 +15,231 @@
 else:
     Config = TypeVar("Config")
 
+_RICH_CONSOLE = Console()
+
+
+def _get_shape(data: Any) -> Tuple[int, ...]:
+    """Recursively compute shape of a nested list"""
+    if not isinstance(data, list):
+        return ()
+    if not data:
+        return (0,)
+    return (len(data), *_get_shape(data[0]))
+
+
+def _fmt_val(v: Any) -> str:
+    if isinstance(v, float):
+        return f"{v:.6g}"
+    return str(v)
+
+
+def _fmt_row(row: list) -> str:
+    """Format a 1-D list with numpy-style head/tail truncation."""
+    if len(row) <= 8:
+        return " ".join(_fmt_val(v) for v in row)
+    head = " ".join(_fmt_val(v) for v in row[:3])
+    tail = " ".join(_fmt_val(v) for v in row[-3:])
+    return f"{head} ... {tail}"
+
+
+def _is_numeric(v: Any) -> bool:
+    return isinstance(v, (int, float)) and not isinstance(v, bool)
+
+
+def is_numeric_1d(data: Any) -> bool:
+    return isinstance(data, list) and bool(data) and all(_is_numeric(v) for v in data)
+
+
+def _quantity_axis_label(q: dict, fallback: str = "") -> str:
+    name = q.get("name") or fallback
+    units = q.get("units") or "-"
+    label = str(name).rsplit("/", 1)[-1] or str(name)
+    return f"{label} [{units}]"
+
+
+def _build_array_body(data: list, shape: Tuple[int, ...]) -> str:
+    """Build string for 1-D or 2-D arrays."""
+    if len(shape) == 1:
+        return f"[{_fmt_row(data)}]"
+
+    if len(shape) == 2:
+        rows = data if len(data) <= 8 else [*data[:3], ..., *data[-3:]]
+        lines = []
+        for row in rows:
+            if row is ...:
+                lines.append(" ...")
+            else:
+                lines.append(f" [{_fmt_row(row)}]")
+        inner = "\n".join(lines)
+        return f"[\n{inner}\n]"
+
+    return f"<{len(shape)}-D array, shape {shape}>"
+
+
+def _iter_numeric(data: Any) -> Iterable[float]:
+    """Yield all numeric leaf values from a nested list, skipping None."""
+    if isinstance(data, list):
+        for item in data:
+            yield from _iter_numeric(item)
+    elif isinstance(data, (int, float)) and data is not None:
+        yield float(data)
+
+
+def _compute_stats(data: Any) -> Optional[Dict[str, float]]:
+    """Return basic statistics for numeric data, or None if not applicable."""
+    values = list(_iter_numeric(data))
+    if len(values) < 2:
+        return None
+    n = len(values)
+    vmin = min(values)
+    vmax = max(values)
+    mean = sum(values) / n
+    std = (sum((x - mean) ** 2 for x in values) / n) ** 0.5
+    sorted_v = sorted(values)
+    mid = n // 2
+    median = sorted_v[mid] if n % 2 else (sorted_v[mid - 1] + sorted_v[mid]) / 2
+    return {
+        "n": n,
+        "min": vmin,
+        "max": vmax,
+        "mean": mean,
+        "std": std,
+        "median": median,
+    }
+
+
+def _stats_table(stats: Dict[str, float]) -> Table:
+    table = Table(show_header=True, header_style="bold", box=None, padding=(0, 2))
+    for key in ("n", "min", "max", "mean", "std", "median"):
+        table.add_column(key, justify="right")
+    table.add_row(
+        str(int(stats["n"])),
+        _fmt_val(stats["min"]),
+        _fmt_val(stats["max"]),
+        _fmt_val(stats["mean"]),
+        _fmt_val(stats["std"]),
+        _fmt_val(stats["median"]),
+    )
+    return table
+
+
+def _plot_stats_table(stats: Dict[str, float], shape: Tuple[int, ...]) -> Table:
+    table = Table(show_header=True, header_style="bold", box=None, padding=(0, 2))
+    for key in ("n", "min", "max", "mean", "std", "median"):
+        table.add_column(key, justify="right")
+    table.add_row(
+        str(int(stats["n"])),
+        _fmt_val(stats["min"]),
+        _fmt_val(stats["max"]),
+        _fmt_val(stats["mean"]),
+        _fmt_val(stats["std"]),
+        _fmt_val(stats["median"]),
+    )
+    return table
+
+
+def _plot_panel(
+    *,
+    plot: Text,
+    title: str,
+    units: str,
+    stats: Optional[Dict[str, float]],
+    shape: Tuple[int, ...],
+) -> None:
+    content = plot
+    if stats:
+        content = Group(plot, _plot_stats_table(stats, shape))
+
+    _RICH_CONSOLE.print(
+        Panel(
+            content,
+            title=f"[bold]{title}[/bold]  [dim]\\[{units}][/dim]",
+            subtitle=f"shape {shape}",
+        )
+    )
+
+
+def show_quantity_textual_plot(
+    q: dict,
+    label: str = "",
+    x_quantity: Optional[dict] = None,
+) -> None:
+    """Print line plot for a 1-D numeric QuantityData dict."""
+    name = q["name"]
+    units = q["units"] or "-"
+    data = q["data"]
+    if not is_numeric_1d(data):
+        print_quantity(q, label=label)
+        return
+
+    try:
+        plotext = importlib.import_module("plotext")
+    except ImportError:
+        print_quantity(q, label=label)
+        return
+
+    y_values = [float(value) for value in data]
+    shape = _get_shape(data)
+    x_values = None
+    xlabel = "index [-]"
+    if (
+        x_quantity
+        and is_numeric_1d(x_quantity.get("data"))
+        and len(x_quantity["data"]) == len(y_values)
+    ):
+        x_values = [float(value) for value in x_quantity["data"]]
+        xlabel = _quantity_axis_label(x_quantity, fallback="x")
+
+    title = label or name
+    if x_values is None:
+        x_values = [float(index) for index in range(len(y_values))]
+
+    console_width = _RICH_CONSOLE.size.width
+    plot_width = max(48, min(70, console_width - 12))
+
+    plotext.clear_figure()
+    plotext.theme("clear")
+    plotext.plotsize(plot_width, 18)
+    plotext.xlabel(xlabel)
+    plotext.ylabel(_quantity_axis_label(q, fallback=label or "field"))
+    plotext.plot(x_values, y_values, marker="braille", color="cyan")
+    plot = Text.from_ansi(plotext.build())
+    stats = _compute_stats(y_values)
+    _plot_panel(
+        plot=plot,
+        title=title,
+        units=units,
+        stats=stats,
+        shape=shape,
+    )
+
+
+def print_quantity(q: dict, label: str = "", show_stats: bool = True) -> None:
+    """Print a QuantityData dict with array display and stats."""
+    name = q["name"]
+    units = q["units"] or "-"
+    data = q["data"]
+    title = f"[bold]{label or name}[/bold]  [dim]\\[{units}][/dim]"
+
+    if not isinstance(data, list):
+        _RICH_CONSOLE.print(Panel(f"{_fmt_val(data)}", title=title, subtitle="scalar"))
+        return
+
+    shape = _get_shape(data)
+    stats = _compute_stats(data)
+    array_body = _build_array_body(data, shape)
+    subtitle = f"shape ({shape[0]},)" if len(shape) == 1 else f"shape {shape}"
+    if show_stats and stats:
+        _RICH_CONSOLE.print(
+            Panel(
+                Group(array_body, _stats_table(stats)),
+                title=title,
+                subtitle=subtitle,
+            )
+        )
+    else:
+        _RICH_CONSOLE.print(Panel(array_body, title=title, subtitle=subtitle))
+
 
 def _flatten_dict(values: Dict) -> List[Tuple[str, str]]:
     items = []

src/simdb/cli/remote_api.py

@@ -663,6 +663,11 @@ def delete_metadata(self, sim_id: str, key: str) -> List[str]:
         res = self.delete("simulation/metadata/" + sim_id, {"key": key})
         return [data["value"] for data in res.json()]
 
+    @try_request
+    def get_simulation_data(self, sim_id: str, path: str) -> Dict[str, Any]:
+        res = self.get(f"simulation/{sim_id}/data", params={"path": path})
+        return res.json()
+
     @try_request
     def get_directory(self) -> str:
         res = self.get("staging_dir")

src/simdb/database/models/simulation.py

@@ -361,14 +361,19 @@ def from_data(cls, data: Dict[str, Union[str, Dict, List]]) -> "Simulation":
             outputs = checked_get(data, "outputs", list)
             simulation.outputs = [File.from_data(el) for el in outputs]
         if "metadata" in data:
-            metadata = checked_get(data, "metadata", list)
-            meta_dict = {}
-            for el in metadata:
-                if not isinstance(el, dict):
-                    raise Exception("corrupted metadata element - expected dictionary")
-                if "element" in el and "value" in el:
-                    meta_dict[el["element"]] = el["value"]
-            simulation._set_metadata_dict(meta_dict)
+            metadata = data.get("metadata")
+            if isinstance(metadata, list):
+                meta_dict = {}
+                for el in metadata:
+                    if not isinstance(el, dict):
+                        raise Exception(
+                            "corrupted metadata element - expected dictionary"
+                        )
+                    if "element" in el and "value" in el:
+                        meta_dict[el["element"]] = el["value"]
+                simulation._set_metadata_dict(meta_dict)
+            elif isinstance(metadata, dict):
+                simulation._set_metadata_dict(metadata)
         return simulation
 
     @classmethod