From 2eee82cb34c3ade8f689939866eadbbdd715c594 Mon Sep 17 00:00:00 2001 From: stewit <> Date: Mon, 26 Oct 2026 21:35:34 +0100 Subject: [PATCH 1/8] rework to strict conventions --- README.md | 8 +- pyproject.toml | 12 +- run | 1 - sphinx/source/first_steps.rst | 106 +- src/hdhelpers/__init__.py | 43 +- src/hdhelpers/metadata/__init__.py | 25 +- src/hdhelpers/metadata/_current_convention.py | 223 -- src/hdhelpers/metadata/_helpers.py | 343 ---- src/hdhelpers/metadata/_reading.py | 433 ++++ src/hdhelpers/metadata/_recording.py | 169 -- src/hdhelpers/metadata/_specs.py | 201 -- src/hdhelpers/metadata/attached.py | 192 ++ src/hdhelpers/metadata/conventions.py | 1792 +++++++++++++++++ src/hdhelpers/metadata/helpers.py | 1460 +++----------- tests/benchmarks/benchmark_metadata.py | 104 + tests/conftest.py | 9 - tests/data/convention_schemas.json | 1727 ++++++++++++++++ tests/data/fixtures/metadata.py | 74 - tests/data/json_templates/mts_attrs.json | 78 - .../json_templates/mts_attrs_old_real.json | 81 - tests/data/json_templates/old_mts_attrs.json | 27 - .../data/json_templates/old_series_attrs.json | 43 - tests/data/json_templates/series_attrs.json | 16 - .../json_templates/series_attrs_old_real.json | 49 - .../json_templates/singletsframe_attrs.json | 29 - tests/metadata/test_attached.py | 159 ++ tests/metadata/test_conventions.py | 1039 ++++++++++ tests/metadata/test_conventions_immutable.py | 171 ++ tests/metadata/test_conventions_rules.py | 233 +++ tests/metadata/test_conventions_schema.py | 35 + tests/metadata/test_helpers.py | 476 +++++ tests/metadata/test_metadata.py | 625 ------ .../test_metadata_current_convention.py | 179 -- .../test_metadata_default_timezone.py | 169 -- tests/metadata/test_metadata_fallbacks.py | 341 ---- tests/metadata/test_metadata_migration.py | 195 -- tests/metadata/test_metadata_recording.py | 183 -- tests/metadata/test_reading.py | 412 ++++ tests/test_optional_plotting.py | 40 + uv.lock | 57 +- 40 files changed, 7279 insertions(+), 4280 deletions(-) delete mode 100644 src/hdhelpers/metadata/_current_convention.py delete mode 100644 src/hdhelpers/metadata/_helpers.py create mode 100644 src/hdhelpers/metadata/_reading.py delete mode 100644 src/hdhelpers/metadata/_recording.py delete mode 100644 src/hdhelpers/metadata/_specs.py create mode 100644 src/hdhelpers/metadata/attached.py create mode 100644 src/hdhelpers/metadata/conventions.py create mode 100644 tests/benchmarks/benchmark_metadata.py create mode 100644 tests/data/convention_schemas.json delete mode 100644 tests/data/fixtures/metadata.py delete mode 100644 tests/data/json_templates/mts_attrs.json delete mode 100644 tests/data/json_templates/mts_attrs_old_real.json delete mode 100644 tests/data/json_templates/old_mts_attrs.json delete mode 100644 tests/data/json_templates/old_series_attrs.json delete mode 100644 tests/data/json_templates/series_attrs.json delete mode 100644 tests/data/json_templates/series_attrs_old_real.json delete mode 100644 tests/data/json_templates/singletsframe_attrs.json create mode 100644 tests/metadata/test_attached.py create mode 100644 tests/metadata/test_conventions.py create mode 100644 tests/metadata/test_conventions_immutable.py create mode 100644 tests/metadata/test_conventions_rules.py create mode 100644 tests/metadata/test_conventions_schema.py create mode 100644 tests/metadata/test_helpers.py delete mode 100644 tests/metadata/test_metadata.py delete mode 100644 tests/metadata/test_metadata_current_convention.py delete mode 100644 tests/metadata/test_metadata_default_timezone.py delete mode 100644 tests/metadata/test_metadata_fallbacks.py delete mode 100644 tests/metadata/test_metadata_migration.py delete mode 100644 tests/metadata/test_metadata_recording.py create mode 100644 tests/metadata/test_reading.py create mode 100644 tests/test_optional_plotting.py diff --git a/README.md b/README.md index 83de197..1a8cc36 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,9 @@ The documentation of the package is a [GitHub page](https://hetida.github.io/hdh ## Getting Started with hdhelpers Since version >0.13.10 the hetida designer runtime comes with an installed version of hdhelpers. To start the hetida designer we recommend following the [hetida designer setup guide](https://github.com/hetida/hetida-designer/blob/release/README.md#getting-started-with-hetida-designer). +The plotting helpers (`hdhelpers.plotting` and their names in `hdhelpers`) need plotly, which is the optional extra +`plotting`: `pip install "hdhelpers[plotting]"`. Everything else, e.g. the metadata models, works without plotly. + An example is given in [GitHub page](https://hetida.github.io/hdhelpers/first_steps.html#how-to-get-metadata-with-hdhelpers) of how to use functionalities of hdhelpers inside a hetida designer component. Furthermore the base component [Single Time Series Plot](https://github.com/hetida/hetida-designer/blob/release/runtime/transformations/components/visualization) uses hdhelpers to demonstrate the usage (since version >0.13.10). ## Developing for hdhelpers @@ -27,7 +30,8 @@ For dependency management and venv setup, building and publishing, [uv](https:// #### Python environment 1) Create a virtual environment with `uv venv`. This will create a hidden `.venv` directory. 2) Activate the virtual environment via `source .venv/bin/activate` -3) Run `uv sync` to install all dependencies given in pyproject.toml. +3) Run `uv sync --all-extras` to install all dependencies given in pyproject.toml, including the optional + ones of the plotting helpers. 4) In case you need to add a new dependency, do so via `uv add `. That way, uv finds versions of all dependencies that are compatible with each other. 5) In case you need a new requirement for development purposes please use `uv add --dev ` @@ -43,7 +47,7 @@ FROM hetida/designer-backend:<> USER root -RUN pip install . +RUN pip install ".[plotting]" USER hd_app diff --git a/pyproject.toml b/pyproject.toml index f9f52e0..07440df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,9 +30,7 @@ maintainers = [ requires-python = ">=3.12" dependencies = [ "pandas>=3", - "plotly", "pydantic", - "glom>25,<26", "numpy", "pytz", ] @@ -55,6 +53,12 @@ classifiers = [ "License :: OSI Approved :: MIT License", ] +[project.optional-dependencies] +# the plotting helpers (hdhelpers.plotting and its names in hdhelpers) +plotting = [ + "plotly", +] + [project.urls] homepage = "https://fuseki.com/data-science/hetida-designer/" documentation = "https://hetida.github.io/hdhelpers/" @@ -102,10 +106,6 @@ dev = [ "/**/tests/**/*.py" = ["S101", "PT023", "T201", "INP001", "PT001", "ARG001", "S113", "S603", "PLR0912"] "tests/conftest.py" = ["F401"] -[[tool.mypy.overrides]] -module = ["glom"] -ignore_missing_imports = true - [[tool.mypy.overrides]] module = ["plotly", "plotly.*"] ignore_missing_imports = true diff --git a/run b/run index 344e5ff..6908e6e 100755 --- a/run +++ b/run @@ -39,7 +39,6 @@ build_docs(){ uv pip install . uv run sphinx-build -M html sphinx/source sphinx/build uv run sphinx-build -M doctest sphinx/source sphinx/build - uv run pytest --doctest-modules src/hdhelpers/metadata/_specs.py cp -r sphinx/build/html/* docs/ } diff --git a/sphinx/source/first_steps.rst b/sphinx/source/first_steps.rst index 24dab44..dc5324e 100644 --- a/sphinx/source/first_steps.rst +++ b/sphinx/source/first_steps.rst @@ -17,16 +17,18 @@ In hetida designer this series can be represented as json for *direct provisioni "dataset_metadata": { "metric_key": "external_id", "single_metric": "water.north.level", + "queried_metrics": ["water.north.level"], "ref_interval_start_timestamp": "2020-01-01T08:10:00+00:00", "ref_interval_end_timestamp": "2020-01-01T08:20:00+00:00", "ref_interval_type": "closed" }, "metrics": [ { + "id": "4c1f0e2a-7b3d-4e5f-9a8b-1c2d3e4f5a6b", "external_id": "water.north.level", "name": "Water level north", "short_display_name": "Water Level", - "value_dimensions": [{"column": "value", "unit": "cm"}], + "value_dimensions": [{"column": "value", "unit": "cm", "measurement": "level"}], "inherited": {"latitude": 51.43, "longitude": 7.58} } ] @@ -55,8 +57,65 @@ We can retrieve the name, unit and latitude of the series with the following cod ... +or, equivalently, via the metadata as model: + +.. code-block:: python + + from hdhelpers.metadata import read_metadata + + def main(*, series): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + + meta = read_metadata(series) # None if the series has no valid metadata + if meta is not None: + name = meta.name() # "Water level north" + unit = meta.unit() # "cm" + latitude = meta.metric_info("latitude") # 51.43 + + ... + .. _hetida designer metadata conventions: https://hetida.github.io/hetida-designer/user_guide/attached_metadata/ +Metadata as model +================= + +``hdhelpers.metadata.conventions`` holds the models of the metadata conventions, one per data +type: ``SeriesMetadata``, ``SingleTSFrameMetadata`` and ``MultiTSFrameMetadata``. They are the +contract of the conventions: Parsing fails for metadata which does not follow them. +``check_metadata_conventions`` reports the issues of such metadata at their locations. + +``read_metadata(obj)`` gives the metadata of a Series or DataFrame as model, or None if there is +no valid metadata. With ``required=True`` it raises a ``MetadataConventionError`` listing the +issues instead, for components which cannot work without metadata. The methods of the models +resolve the fallbacks described below, e.g. ``unit()``, ``display_name()``, ``metric_info()``, +``default_timezone()``, ``recording()`` and ``queried_interval()``. For a SERIES or +SINGLETSFRAME, the metric can be omitted. For a MULTITSFRAME, the methods take the metric, and +methods in plural like ``units()`` return the information of all metrics. + +The hetida designer runtime parses the metadata of the workflow inputs once and attaches the +model to them (``attach_metadata``). The models are immutable, so that all objects pandas +derives from an input share the model instead of copying the metadata, which saves a lot of +time for metadata of many metrics. To change metadata, create a changed version, which shares +all unchanged parts, and set it on your result: + +.. code-block:: python + + from hdhelpers.metadata import read_metadata, set_metadata + + def main(*, series): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + + result = series * 100 + meta = read_metadata(series, required=True) + set_metadata(result, meta.with_value_dimension(None, "value", unit="m")) + + return {"result": result} + +Since the model is no JSON value, ``json.dumps(obj.attrs)`` or ``obj.to_parquet()`` fail for +objects with attached metadata. Use ``attrs_to_json(obj.attrs)`` to get attrs in their JSON form. + Where does the information come from? ===================================== @@ -91,43 +150,36 @@ In contrast, the functions for dedicated information do not use them by default, inherited property with the same label usually does not describe the metric itself, e.g. a property "unit" of a plant. Pass ``include_inherited=True`` to use them as last resort anyway. -Entries which do not adhere to the conventions are skipped: metrics which are no objects or -whose field named by the metric key is no string, and value dimensions which are no objects or -whose column is no string. ``"value_dimensions"`` and ``"value_dimensions_shared"`` which are no -lists count as empty, a metric key which is no string as missing, i.e. ``"id"`` is used. +Metadata which does not follow the conventions provides no information at all: The functions +return None or empty results for it. How to query nested information? ================================ The information requested from ``get_metric_info``, ``get_singlets_metric_info``, -``get_series_info`` and ``get_singlets_info`` is interpreted as a `glom`_ spec. A string is a -path in which dots separate the keys, so it also reaches into sub dictionaries. Let's say the -metric of the series above additionally provides comments and information on its sensor (the -conventions allow additional fields at every level), and the label of one of its inherited -properties contains a dot: +``get_series_info`` and ``get_singlets_info`` (and from the corresponding methods of the models) +is a field or a path of fields separated by dots, which reaches into sub objects. Let's say the +metric of the series above additionally provides information on its sensor (the conventions +allow additional fields at every level): .. code-block:: json { + "id": "4c1f0e2a-7b3d-4e5f-9a8b-1c2d3e4f5a6b", "external_id": "water.north.level", "name": "Water level north", "sensor": { "manufacturer": "Acme", "calibration": {"date": "2026-03-01", "offset": 0.02} }, - "comments": [ - {"content": "Gauge maintenance"}, - {"content": "Sensor replaced"} - ], - "value_dimensions": [{"column": "value", "unit": "cm"}], - "inherited": {"latitude": 51.43, "operator.name": "Ruhrverband"} + "value_dimensions": [{"column": "value", "unit": "cm", "measurement": "level"}], + "inherited": {"latitude": 51.43} } We can retrieve the nested information with the following code .. code-block:: python - from glom import Path from hdhelpers.metadata import get_series_info def main(*, series): @@ -139,13 +191,6 @@ We can retrieve the nested information with the following code latitude = get_series_info(series, "inherited.latitude") # 51.43, never the metric's own firmware = get_series_info(series, "sensor.firmware") # None, not provided - # any other glom spec, e.g. the content of each comment - comments = get_series_info(series, ("comments", ["content"])) - # ["Gauge maintenance", "Sensor replaced"] - - # a key which itself contains dots requires a glom Path - operator = get_series_info(series, Path("operator.name")) # "Ruhrverband" - ... For a MULTITSFRAME, ``get_metric_info(multitsframe, "sensor.manufacturer")`` provides the @@ -153,16 +198,9 @@ manufacturer keyed by metric accordingly. The fallbacks described above apply to the whole path, e.g. ``"sensor.manufacturer"`` is looked up in the metric first, then in its value dimension ``"value"`` and so on. A path which is not -present yields None instead of an error. Note that a plain string like ``"operator.name"`` is -always split at its dots, so it does not find the inherited property ``"operator.name"``. - -A spec may also transform the information, e.g. ``("name", str.upper)``. Since information -provided as null counts as missing, such a spec may receive null: on the metric itself and, if -the metric does not provide the information, on the levels it falls back to. Errors raised by -the spec are not caught. So either let it handle None, or rather transform the information -after retrieving it. - -.. _glom: https://glom.readthedocs.io/ +present yields None instead of an error. Paths cannot address list items or keys which contain +dots themselves. Such information, e.g. the comments of a metric, is available via the models: +``read_metadata(series).metric().comments``. Which time zone to compute in? ============================== diff --git a/src/hdhelpers/__init__.py b/src/hdhelpers/__init__.py index 40f1073..7208cb4 100644 --- a/src/hdhelpers/__init__.py +++ b/src/hdhelpers/__init__.py @@ -1,17 +1,52 @@ """helper functions for writing components in hetida designer""" -from hdhelpers import exceptions, helpers, metadata, plotting +import importlib +from typing import TYPE_CHECKING, Any + +from hdhelpers import exceptions, helpers, metadata from hdhelpers.exceptions import HelperException, InsufficientPlottingData from hdhelpers.helpers.locale import get_locale from hdhelpers.helpers.timezone_handling import modify_timezone from hdhelpers.plot_target_settings import StatusColors -from hdhelpers.plotting.color import FUSEKI_COLOR_CYCLE, resolve_color -from hdhelpers.plotting.plotly_theme import set_agnostic_theme -from hdhelpers.plotting.timestamp_ticks import TICKFORMATSTOPS, set_dt_ticks + +if TYPE_CHECKING: + from hdhelpers import plotting + from hdhelpers.plotting.color import FUSEKI_COLOR_CYCLE, resolve_color + from hdhelpers.plotting.plotly_theme import set_agnostic_theme + from hdhelpers.plotting.timestamp_ticks import TICKFORMATSTOPS, set_dt_ticks # do not edit line of __version__ as it is automatically modified by running ./run build_package __version__ = "0.0.9" +# The plotting helpers need plotly, which comes with the optional extra hdhelpers[plotting]. They +# are imported on first access, so that everything else works without plotly. +_PLOTTING_ATTRIBUTES: dict[str, tuple[str, str | None]] = { + "plotting": ("hdhelpers.plotting", None), + "FUSEKI_COLOR_CYCLE": ("hdhelpers.plotting.color", "FUSEKI_COLOR_CYCLE"), + "resolve_color": ("hdhelpers.plotting.color", "resolve_color"), + "set_agnostic_theme": ("hdhelpers.plotting.plotly_theme", "set_agnostic_theme"), + "TICKFORMATSTOPS": ("hdhelpers.plotting.timestamp_ticks", "TICKFORMATSTOPS"), + "set_dt_ticks": ("hdhelpers.plotting.timestamp_ticks", "set_dt_ticks"), +} + + +def __getattr__(name: str) -> Any: + if name not in _PLOTTING_ATTRIBUTES: + raise AttributeError(f"module 'hdhelpers' has no attribute {name!r}") + module_name, attribute = _PLOTTING_ATTRIBUTES[name] + try: + module = importlib.import_module(module_name) + except ImportError as e: + if (e.name or "").partition(".")[0] != "plotly": + raise + raise ImportError( + f"hdhelpers.{name} needs plotly, which is installed with the extra hdhelpers[plotting]" + ) from e + value = module if attribute is None else getattr(module, attribute) + globals()[name] = value + return value + + # import can be done by using `from hdhelpers import *`` __all__ = [ "HelperException", diff --git a/src/hdhelpers/metadata/__init__.py b/src/hdhelpers/metadata/__init__.py index 969d034..22c20df 100644 --- a/src/hdhelpers/metadata/__init__.py +++ b/src/hdhelpers/metadata/__init__.py @@ -1,7 +1,19 @@ -"""Collection of functions to access metadata information from timeseries objects. -Metadata information can follow varying conventions as the package glom is used to -extract the requested information.""" +"""Access to the metadata of SERIES, SINGLETSFRAMEs and MULTITSFRAMEs +The metadata follows the hetida designer metadata conventions, see +hdhelpers.metadata.conventions for its models. read_metadata gives the metadata of an object as +model, whose methods resolve the fallbacks of the conventions. The get_* functions are thin +wrappers over these methods. +""" + +from hdhelpers.metadata.attached import ( + META_KEY, + attach_metadata, + attrs_to_json, + read_metadata, + set_metadata, +) +from hdhelpers.metadata.conventions import MetadataConventionError, check_metadata_conventions from hdhelpers.metadata.helpers import ( Recording, get_default_aggregation_methods, @@ -37,6 +49,13 @@ ) __all__ = [ + "META_KEY", + "MetadataConventionError", + "attach_metadata", + "attrs_to_json", + "check_metadata_conventions", + "read_metadata", + "set_metadata", "get_queried_interval", "get_series_display_name", "get_series_measurement", diff --git a/src/hdhelpers/metadata/_current_convention.py b/src/hdhelpers/metadata/_current_convention.py deleted file mode 100644 index 3f589b5..0000000 --- a/src/hdhelpers/metadata/_current_convention.py +++ /dev/null @@ -1,223 +0,0 @@ -"""Extraction of metadata following the current metadata convention - -The metadata is gathered in a single pass in plain Python: the object of each metric with its -value dimensions by column, and the value dimensions shared by all metrics -("value_dimensions_shared"). The fallback chains are resolved on these objects, applying the -spec of the requested information only to the objects of the levels actually needed. Field -names and dotted paths of them are looked up without glom, since evaluating glom specs costs a -lot of time per object, which adds up for frames with many metrics. - -Entries not adhering to the conventions are skipped: metrics which are no objects or whose -field named by the metric key is no string, and value dimensions which are no objects or whose -column is no string. "value_dimensions" and "value_dimensions_shared" which are no lists count -as empty, a metric key which is no string as missing, i.e. the default "id" is used. - -Older metadata structures are handled by the glom specs in _specs. -""" - -from collections import defaultdict -from collections.abc import Callable, Sequence -from dataclasses import dataclass, replace -from typing import Any - -from glom import Coalesce, Spec, glom - -from hdhelpers.metadata._specs import INHERITED - -Getter = Callable[[Any], Any] - -_MISSING = object() -_DEFAULT_COALESCE_SKIP = Coalesce("any").skip - - -@dataclass(frozen=True) -class GatheredMetric: - metric: dict - value_dimensions: dict[str, dict] # by column - - -@dataclass(frozen=True) -class GatheredMetadata: - metrics: dict[str, GatheredMetric] # by value of the field named by the metric key - value_dimensions_shared: dict[str, dict] # by column - - -def follows_current_convention(attrs: Any) -> bool: - """Whether metadata follows the current metadata convention, i.e. has a list of metrics""" - return isinstance(attrs, dict) and isinstance(attrs.get("metrics"), list) - - -def _metric_key(attrs: dict) -> str: - dataset_metadata = attrs.get("dataset_metadata") - metric_key = dataset_metadata.get("metric_key") if isinstance(dataset_metadata, dict) else None - return metric_key if isinstance(metric_key, str) else "id" - - -def _by_column(value_dimensions: Any) -> dict[str, dict]: - """Value dimension objects by column, the last one winning, skipping malformed ones""" - if not isinstance(value_dimensions, list): - return {} - return { - value_dimension["column"]: value_dimension - for value_dimension in value_dimensions - if isinstance(value_dimension, dict) and isinstance(value_dimension.get("column"), str) - } - - -def gather(attrs: Any) -> GatheredMetadata | None: - """Gather the objects of all levels, None if not following the current convention""" - if not follows_current_convention(attrs): - return None - - metric_key = _metric_key(attrs) - return GatheredMetadata( - metrics={ - metric[metric_key]: GatheredMetric(metric, _by_column(metric.get("value_dimensions"))) - for metric in attrs["metrics"] - if isinstance(metric, dict) and isinstance(metric.get(metric_key), str) - }, - value_dimensions_shared=_by_column(attrs.get("value_dimensions_shared")), - ) - - -def restricted_to(gathered: GatheredMetadata, metric: str) -> GatheredMetadata: - """The gathered metadata of one of the metrics only, so that only its information is resolved""" - return replace(gathered, metrics={metric: gathered.metrics[metric]}) - - -def _simple_path(spec: Any) -> tuple[str, ...] | None: - """The keys a spec looks up one after another, if it is just a path of dict keys - - glom splits string paths at dots, and only interprets stars besides. - """ - if isinstance(spec, str) and "*" not in spec and "\\" not in spec: - return tuple(spec.split(".")) - if ( - isinstance(spec, Coalesce) - and len(spec.subspecs) == 1 - and spec.default is None - and spec.skip is _DEFAULT_COALESCE_SKIP - ): - return _simple_path(spec.subspecs[0]) - return None - - -def getter(spec: str | Spec) -> Getter: - """Apply spec as Coalesce(spec, default=None), without glom for paths of dict keys""" - generic_spec = Coalesce(spec, default=None) - - def generic(obj: Any) -> Any: - return glom(obj, generic_spec) - - path = _simple_path(spec) - if path is None: - return generic - - def by_path(obj: Any) -> Any: - # glom accesses keys of dicts via item access only, so a missing key yields None. - # Other targets, e.g. lists accessed via indices, are left to glom. - target = obj - for key in path: - if type(target) is not dict: - return generic(obj) - if key not in target: - return None - target = target[key] - return target - - return by_path - - -def _inherited_info(metric: dict, get: Getter) -> Any: - # the spec is not applied if the metric does not provide inherited properties at all - return get(metric[INHERITED]) if INHERITED in metric else None - - -def value_dimension_info( - gathered: GatheredMetadata, specs: Sequence[str | Spec], include_inherited: bool -) -> defaultdict[str, defaultdict[str, Any]]: - """Information per value dimension per metric, falling back to further specs - - The value dimensions of a metric are its own ones, "value" and the shared ones. For each - of them, the first information which is not None is used: Each spec is applied on - - 1. the value dimension itself, - 2. only for the value dimension "value": the metric, - 3. "value_dimensions_shared" - - before the next spec is tried. If include_inherited, the properties the metric inherits - are used as last resort for the value dimension "value", again trying each spec in turn. - """ - getters = [getter(spec) for spec in specs] - shared_info_cache: list[dict[str, Any]] = [{} for _ in getters] - - def shared_info(index: int, column: str) -> Any: - cache = shared_info_cache[index] - if column not in cache: - shared = gathered.value_dimensions_shared.get(column) - cache[column] = None if shared is None else getters[index](shared) - return cache[column] - - result: defaultdict[str, defaultdict[str, Any]] = defaultdict(lambda: defaultdict(lambda: None)) - for metric, gathered_metric in gathered.metrics.items(): - info_by_column: defaultdict[str, Any] = defaultdict(lambda: None) - columns = dict.fromkeys( - [*gathered.value_dimensions_shared, "value", *gathered_metric.value_dimensions] - ) - for column in columns: - value_dimension = gathered_metric.value_dimensions.get(column) - info = None - for index, get in enumerate(getters): - info = get(value_dimension) if value_dimension is not None else None - if info is None and column == "value": - info = get(gathered_metric.metric) - if info is None: - info = shared_info(index, column) - if info is not None: - break - if info is None and include_inherited and column == "value": - for get in getters: - info = _inherited_info(gathered_metric.metric, get) - if info is not None: - break - info_by_column[column] = info - result[metric] = info_by_column - return result - - -def metric_info( - gathered: GatheredMetadata, spec: str | Spec, include_inherited: bool -) -> dict[str, Any]: - """Information per metric, the first one which is not None of - - 1. the metric itself, - 2. its value dimension "value", - 3. the value dimension "value" in "value_dimensions_shared", - 4. if include_inherited, the properties the metric inherits. - """ - get = getter(spec) - shared = gathered.value_dimensions_shared.get("value") - shared_info: Any = _MISSING - - result: dict[str, Any] = {} - for metric, gathered_metric in gathered.metrics.items(): - info = get(gathered_metric.metric) - if info is None: - value_dimension = gathered_metric.value_dimensions.get("value") - info = get(value_dimension) if value_dimension is not None else None - if info is None and shared is not None: - if shared_info is _MISSING: - shared_info = get(shared) - info = shared_info - if info is None and include_inherited: - info = _inherited_info(gathered_metric.metric, get) - result[metric] = info - return result - - -def own_metric_info(gathered: GatheredMetadata, key: str) -> dict[str, Any]: - """A field of each metric itself, without falling back to other levels""" - return { - metric: gathered_metric.metric.get(key) - for metric, gathered_metric in gathered.metrics.items() - } diff --git a/src/hdhelpers/metadata/_helpers.py b/src/hdhelpers/metadata/_helpers.py deleted file mode 100644 index 7af114d..0000000 --- a/src/hdhelpers/metadata/_helpers.py +++ /dev/null @@ -1,343 +0,0 @@ -from collections import defaultdict -from collections.abc import Callable, Collection -from typing import Any, cast - -import pandas as pd -from glom import Check, Coalesce, CoalesceError, Spec, glom - -import hdhelpers.metadata._current_convention as _current_convention -import hdhelpers.metadata._specs as _specs - - -def check_dataframe(data: pd.DataFrame): - if not isinstance(data, pd.DataFrame): - raise TypeError("Input is not a pd.DataFrame") - - -def check_series(series: pd.Series): - if not isinstance(series, pd.Series): - raise TypeError("Input is not a pd.Series.") - - -def check_series_or_dataframe(data: pd.Series | pd.DataFrame): - if not isinstance(data, pd.Series) and not isinstance(data, pd.DataFrame): - raise TypeError("Input is not a pd.Series or pd.DataFrame.") - - -def spec_not_none(spec: str | Spec) -> Spec: - """this entries must be given in the spec""" - # TODO: Given the suite of tools introduced with Match, the Check specifier type may be deprecated in a future release - return (spec, Check(validate=lambda x: x is not None)) - - -def extract_from_metadata(metadata: Any, key: str, default: str | None = None) -> Any: - return glom(metadata, Coalesce(f"dataset_metadata.{key}", default=default)) - - -def _legacy_value_dimension_info( - attrs: Any, value_dim_info: str | Spec -) -> defaultdict[str, defaultdict[str, Any]]: - """get_value_dimension_info for older metadata structures""" - return defaultdict( - lambda: defaultdict(lambda: None), - glom(attrs, _specs.by_metric_key_by_val_dimension(value_dim_info)), - ) - - -def get_value_dimension_info( - multitsframe: pd.DataFrame | pd.Series, - value_dim_info: str | Spec, - include_inherited: bool = False, -) -> defaultdict[str, defaultdict[str, Any]]: - """Obtain metadata info associated to the value dimensions of the metrics - - Returns a default dict whose values are the entries of the metrics metadata specified via - "metric_key" in "dataset_metadata". - - Its values are defaultdicts whose keys are the "column" entries of the value dimension - objects of that metric and whose values are extracted from the value_dimension object - using using value_dim_info as a glom Spec, typically just a subfield. - - For the default "value" value dimension, if no concrete / explicit information is available - for this value dimension, a corresponding entry in the metric object may be used. - - For all value dimensions, if no concrete explicit information is available for that value - dimension in the value_dimensions list under the metric, the global "value_dimensions_shared" - field of the attrs object is searched for corresponding information. - - If include_inherited, the properties a metric inherits are used as last resort for the - "value" value dimension, i.e. after "value_dimensions_shared". - - If no information is found, None is set as value and is the default value of the - inner default dict. - - For metadata following the current metadata convention, value_dim_info is only applied to - the levels needed and entries not adhering to the conventions are skipped, see - _current_convention. - - For examples we refer to the corresponding unit tests (/tests/helpers/test_metadata.py). - """ - gathered = _current_convention.gather(multitsframe.attrs) - if gathered is not None: - return _current_convention.value_dimension_info( - gathered, [value_dim_info], include_inherited - ) - return _legacy_value_dimension_info(multitsframe.attrs, value_dim_info) - - -def value_dimension_info_with_fallbacks( - data: pd.DataFrame | pd.Series, - fields: tuple[str | Spec, ...], - legacy_spec: Spec, - include_inherited: bool = False, -) -> defaultdict[str, defaultdict[str, Any]]: - """Obtain a field of the value dimensions, falling back to further fields - - For metadata following the current metadata convention, each field is looked up on all - levels described in get_value_dimension_info (the value dimension, the metric for the - value dimension "value" and "value_dimensions_shared") before the next field is tried. - So e.g. a display name of the metric wins over the name of its value dimension. Fields - provided as null are treated as missing. Inherited properties are only used if none of the - fields is provided by the metadata itself. - - Older metadata structures are handled via legacy_spec, keeping their established behaviour. - """ - gathered = _current_convention.gather(data.attrs) - if gathered is None: - return _legacy_value_dimension_info(data.attrs, legacy_spec) - return _current_convention.value_dimension_info(gathered, fields, include_inherited) - - -_SINGLE_METRIC_KEY = "single_metric" - - -def _single_metric_name(attrs: Any) -> str | None: - """The metric named by _SINGLE_METRIC_KEY in "dataset_metadata", if it is a string""" - metric = extract_from_metadata(attrs, key=_SINGLE_METRIC_KEY) - return metric if isinstance(metric, str) else None - - -def _single_metric(attrs: Any, metrics: Collection[str]) -> str | None: - """The single metric of a SingleTSFrame or Series among the metrics present - - The single metric is identified via _SINGLE_METRIC_KEY in "dataset_metadata", values which - are no strings count as missing. If it is missing but exactly one metric is present, that - one is used, since a SingleTSFrame or Series cannot be ambiguous in this respect. Otherwise - None is returned - also if the identified metric is not present, since the only metric - present is another one then. - """ - metric = _single_metric_name(attrs) - if metric is not None: - return metric if metric in metrics else None - - if len(metrics) == 1: - return next(iter(metrics)) - - return None - - -def _select_single_metric(info_by_metric: dict, attrs: Any, empty: Any) -> Any: - """Pick the entry of the single metric (see _single_metric) out of a by-metric mapping - - `empty` is returned if the single metric cannot be determined. - """ - metric = _single_metric(attrs, info_by_metric) - return empty if metric is None else info_by_metric[metric] - - -def _single_metric_info( - attrs: Any, - current: Callable[[_current_convention.GatheredMetadata], dict[str, Any]], - legacy: Callable[[], dict[str, Any]], - empty: Any, -) -> Any: - """Information of the single metric of a SingleTSFrame or Series (see _single_metric) - - For metadata following the current metadata convention, `current` gets the gathered metadata - of the single metric only, so that no information of the other metrics is resolved - of - which there may be many, e.g. for a Series taken from a MultiTSFrame. For older metadata - structures, the entry of the single metric is picked out of the result of `legacy`. - - `empty` is returned if the single metric cannot be determined. - """ - gathered = _current_convention.gather(attrs) - if gathered is None: - return _select_single_metric(legacy(), attrs, empty) - - metric = _single_metric(attrs, gathered.metrics) - if metric is None: - return empty - return current(_current_convention.restricted_to(gathered, metric))[metric] - - -def singlets_value_dimension_info( - singletsframe: pd.DataFrame, value_dim_info: str | Spec, include_inherited: bool = False -) -> defaultdict[str, Any]: - """Metadata info of the value dimensions of the single metric of a SingleTSFrame - - A SingleTSFrame holds exactly one metric but - like a MultiTSFrame - arbitrarily many - value dimensions. So in contrast to get_value_dimension_info (which is keyed by metric - first) this returns a defaultdict keyed by value dimension column name only. - - The single metric is identified via "dataset_metadata.single_metric". If that is missing - but the metadata contains exactly one metric, that metric is used, since a SingleTSFrame - cannot be ambiguous in this respect. - - What is extracted per value dimension is exactly what get_value_dimension_info extracts - - only the metric level is collapsed away. - """ - return singlets_value_dimension_info_with_fallbacks( - singletsframe, (value_dim_info,), value_dim_info, include_inherited - ) - - -def singlets_value_dimension_info_with_fallbacks( - singletsframe: pd.DataFrame, - fields: tuple[str | Spec, ...], - legacy_spec: Spec, - include_inherited: bool = False, -) -> defaultdict[str, Any]: - """Counterpart of value_dimension_info_with_fallbacks for SingleTSFrames - - Like singlets_value_dimension_info, the result is keyed by value dimension only. - """ - return _single_metric_info( - singletsframe.attrs, - lambda gathered: _current_convention.value_dimension_info( - gathered, fields, include_inherited - ), - lambda: _legacy_value_dimension_info(singletsframe.attrs, legacy_spec), - empty=defaultdict(lambda: None), - ) - - -# the metric of a Series in older metadata structures, used if "single_metric" is missing -_LEGACY_SERIES_METRIC = "series" - - -def series_value_info( - series: pd.Series, fields: tuple[str | Spec, ...], legacy_spec: Spec, include_inherited: bool -) -> Any: - """Info of the value dimension "value" of the single metric of a Series - - Each field is looked up as described in value_dimension_info_with_fallbacks. The single - metric is determined like for SingleTSFrames, see singlets_value_dimension_info. Only older - metadata structures name the metric of a Series "series", which is used if "single_metric" - is missing. - """ - - def legacy() -> dict[str, Any]: - info_by_metric = _legacy_value_dimension_info(series.attrs, legacy_spec) - if _single_metric_name(series.attrs) is None and _LEGACY_SERIES_METRIC in info_by_metric: - return {_LEGACY_SERIES_METRIC: info_by_metric[_LEGACY_SERIES_METRIC]} - return info_by_metric - - return _single_metric_info( - series.attrs, - lambda gathered: _current_convention.value_dimension_info( - gathered, fields, include_inherited - ), - legacy, - empty={}, - ).get("value") - - -def _legacy_metric_info(attrs: Any, metric_info: str | Spec) -> dict[str, Any]: - """metric_info_by_metric for older metadata structures""" - try: - return glom(attrs, _specs.spec_by_metric_key(metric_info)) # type: ignore[no-any-return] - except CoalesceError: # no metric metadata present at all - return {} - - -def metric_info_by_metric( - data: pd.DataFrame | pd.Series, metric_info: str | Spec, include_inherited: bool = True -) -> dict[str, Any]: - """Obtain metadata info associated to the metrics, keyed by metric - - For metadata following the current metadata convention, information the metric itself does - not provide is taken from its value dimension "value", then from "value_dimensions_shared" - and, if include_inherited, finally from the properties the metric inherits. Information - provided as null counts as missing. metric_info is only applied to the levels needed and - entries not adhering to the conventions are skipped, see _current_convention. - - Returns an empty dict if no metric metadata is present at all. Errors raised by applying - metric_info, e.g. a transformation failing on null, are not caught. - """ - gathered = _current_convention.gather(data.attrs) - if gathered is not None: - return _current_convention.metric_info(gathered, metric_info, include_inherited) - return _legacy_metric_info(data.attrs, metric_info) - - -def singlets_metric_info( - singletsframe: pd.DataFrame, metric_info: str | Spec, include_inherited: bool = True -) -> Any: - """Metadata of the single metric of a SingleTSFrame - - Counterpart of get_metric_info for SingleTSFrames: instead of a mapping keyed by metric - this directly returns the requested information for the one metric, or None if it cannot - be determined - which includes the case of absent metric metadata, since components should - not require metadata (see the metadata conventions documentation). - """ - return _single_metric_info( - singletsframe.attrs, - lambda gathered: _current_convention.metric_info(gathered, metric_info, include_inherited), - lambda: _legacy_metric_info(singletsframe.attrs, metric_info), - empty=None, - ) - - -def _string_or_none(value: Any) -> str | None: - return value if isinstance(value, str) and value != "" else None - - -def _dataset_default_timezone(attrs: Any) -> str | None: - return _string_or_none(extract_from_metadata(attrs, key=_specs.DEFAULT_TIMEZONE)) - - -def _default_timezones( - gathered: _current_convention.GatheredMetadata | None, dataset_timezone: str | None -) -> defaultdict[str, str | None]: - own_timezones = ( - {} - if gathered is None - else _current_convention.own_metric_info(gathered, _specs.DEFAULT_TIMEZONE) - ) - return defaultdict( - lambda: dataset_timezone, - { - metric: _string_or_none(timezone) or dataset_timezone - for metric, timezone in own_timezones.items() - }, - ) - - -def default_timezone_by_metric(data: pd.DataFrame | pd.Series) -> defaultdict[str, str | None]: - """The default time zone of each metric, falling back to the one of the dataset metadata - - The default time zone of a metric only falls back to the one of the dataset metadata, not to - its value dimensions or inherited properties. - The default of the returned defaultdict is the one of the dataset metadata, since it applies - to all metrics without their own. Values which are no (non-empty) strings count as missing. - """ - return _default_timezones( - _current_convention.gather(data.attrs), _dataset_default_timezone(data.attrs) - ) - - -def single_metric_default_timezone(data: pd.DataFrame | pd.Series) -> str | None: - """The default time zone of the single metric of a Series or SingleTSFrame - - Falls back to the one of the dataset metadata, also if the single metric is not found. - """ - dataset_timezone = _dataset_default_timezone(data.attrs) - return cast( - str | None, - _single_metric_info( - data.attrs, - lambda gathered: _default_timezones(gathered, dataset_timezone), - dict, # older metadata structures provide no default time zones of the metrics - empty=dataset_timezone, - ), - ) diff --git a/src/hdhelpers/metadata/_reading.py b/src/hdhelpers/metadata/_reading.py new file mode 100644 index 0000000..b2efe24 --- /dev/null +++ b/src/hdhelpers/metadata/_reading.py @@ -0,0 +1,433 @@ +"""Reading metadata of the conventions, with the fallbacks of the conventions + +The methods of MetadataReading are available on the root models of the conventions +(SeriesMetadata, SingleTSFrameMetadata, MultiTSFrameMetadata). They resolve the fallbacks: + +- Information of a value dimension, e.g. its unit: Each field of a chain, e.g. display_name and + then name, is looked up on the value dimension, for the value dimension "value" on the metric, + then on the shared value dimension of the column (value_dimensions_shared), before the next + field is tried. With include_inherited, the properties the metric inherits are the last resort + for the value dimension "value". +- Information of a metric: the metric, its value dimension "value", the shared value dimension + "value", with include_inherited finally the properties the metric inherits. +- Default time zone: the one of the metric, otherwise the one of the dataset metadata. +- Recording: the frequency of the metric together with its offset and time zone, otherwise the + ones of the dataset metadata (ref_data_frequency etc.), always as a whole. +- Default aggregation method: the one of the metric, with AVG as MEAN, otherwise LAST for + boolean and string values and MEAN for all others. + +Fields can be given as dotted paths, e.g. "inherited.site". Information which is null counts as +missing. For a metric without metadata, information is None, apart from the fallbacks to the +dataset metadata. + +For SERIES and SINGLETSFRAME metadata, the metric_id can be omitted, which refers to their single +metric. MULTITSFRAME metadata requires it. +""" + +import datetime +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING, Any, ClassVar, Literal + +from pydantic import BaseModel + +if TYPE_CHECKING: + from hdhelpers.metadata.conventions import ( + MetricMetadata, + SharedValueDimensionMetadata, + ValueDimensionMetadata, + _DatasetMetadataBase, + ) + +DISPLAY_NAME_FIELDS = ("display_name", "name") +SHORT_DISPLAY_NAME_FIELDS = ("short_display_name", *DISPLAY_NAME_FIELDS) + +# default aggregation method of metrics without one by the data type of their values +_DEFAULT_AGGREGATION_METHOD_BY_VALUE_DATA_TYPE = {"boolean": "LAST", "string": "LAST"} +_DEFAULT_AGGREGATION_METHOD = "MEAN" +_AGGREGATION_METHOD_ALIASES = {"AVG": "MEAN"} + +_MISSING = object() + + +@dataclass(frozen=True) +class Recording: + """How the values of a metric are recorded, as given in its metadata + + The fields mean the following: + + Attributes: + recording_mode (str | None): "periodic": one value per interval of data_frequency, timestamped with the start of the interval. A missing value is a gap. "on_change": a value is recorded when it changes by more than the deadband of its value dimension and holds until the next one. Missing timestamps are no gaps. "irregular": values at arbitrary points in time. None if unknown. + data_frequency (str | None): Frequency as Pandas date offset alias or timedelta string, e.g. "15min", "1D" or "1MS". For "on_change", the interval after which the value is recorded again at the latest, also without a change (heartbeat). + data_frequency_offset (str | None): Timedelta string by which the intervals of data_frequency are shifted, e.g. "5min". None means 0. + data_frequency_timezone (str | None): Time zone the intervals of data_frequency are aligned to, e.g. "Europe/Berlin". None means UTC. + recording_aggregation_method (str | None): Only for "periodic": the method each value aggregates its interval with, e.g. "MEAN" or "MAX". None if the values are samples, e.g. the value holding at the timestamp. + + data_frequency_offset and data_frequency_timezone belong to data_frequency: They are None + if no data_frequency applies. + """ # noqa: E501 + + recording_mode: str | None = None + data_frequency: str | None = None + data_frequency_offset: str | None = None + data_frequency_timezone: str | None = None + recording_aggregation_method: str | None = None + + +_Frequency = tuple[str | None, str | None, str | None] + + +def lookup(obj: Any, path: str) -> Any: + """A field, extra field or dotted path of fields of a model or mapping, None if missing + + Members of enums are given as their values, e.g. "MEAN" for AggregationMethod.MEAN, as in + the JSON form of the metadata. + """ + for key in path.split("."): + if isinstance(obj, BaseModel): + # __dict__ holds the fields, without warning about deprecated ones like getattr + value = obj.__dict__.get(key, _MISSING) + obj = (obj.__pydantic_extra__ or {}).get(key) if value is _MISSING else value + elif isinstance(obj, Mapping): + obj = obj.get(key) + else: + return None + if obj is None: + return None + return _plain(obj) + + +def _plain(value: Any) -> Any: + """The value of an enum member, e.g. "MEAN" instead of AggregationMethod.MEAN""" + return value.value if isinstance(value, Enum) else value + + +def _non_empty(value: Any) -> str | None: + value = _plain(value) + return value if isinstance(value, str) and value != "" else None + + +def _frequency(obj: Any, fields: tuple[str, str, str]) -> _Frequency: + """Frequency, offset and time zone of obj, all None if it has no frequency""" + frequency, offset, timezone = (_non_empty(lookup(obj, field)) for field in fields) + return (None, None, None) if frequency is None else (frequency, offset, timezone) + + +def _value_dimension_info( + fields: tuple[str, ...], + metric: "MetricMetadata", + value_dimensions: Mapping[str, "ValueDimensionMetadata"], + shared: Mapping[str, "SharedValueDimensionMetadata"], + column: str, + include_inherited: bool, +) -> Any: + value_dimension = value_dimensions.get(column) + shared_value_dimension = shared.get(column) + for field in fields: + info = lookup(value_dimension, field) + if info is None and column == "value": + info = lookup(metric, field) + if info is None: + info = lookup(shared_value_dimension, field) + if info is not None: + return info + if include_inherited and column == "value" and metric.inherited is not None: + for field in fields: + info = lookup(metric.inherited, field) + if info is not None: + return info + return None + + +def _by_column( + value_dimensions: "Sequence[ValueDimensionMetadata] | Sequence[SharedValueDimensionMetadata]", +) -> dict[str, Any]: + return {value_dimension.column: value_dimension for value_dimension in value_dimensions} + + +def _fields(fields: str | Sequence[str]) -> tuple[str, ...]: + return (fields,) if isinstance(fields, str) else tuple(fields) + + +class MetadataReading: + """Lookups with the fallbacks of the conventions, see the module docstring""" + + if TYPE_CHECKING: + dataset_metadata: _DatasetMetadataBase + metrics: tuple[MetricMetadata, ...] + value_dimensions_shared: tuple[SharedValueDimensionMetadata, ...] | None + _data_type: ClassVar[Literal["SERIES", "SINGLETSFRAME", "MULTITSFRAME"]] + _by_id: dict[str, MetricMetadata] | None + + # metrics + + def metric_ids(self) -> list[str]: + """The ids of the metrics, i.e. their values of the field named by the metric key""" + return list(self._metrics_by_id()) + + def metric(self, metric_id: str | None = None) -> "MetricMetadata": + """The metadata of a metric, KeyError if there is none""" + metric = self._metric_or_none(metric_id) + if metric is None: + raise KeyError(metric_id) + return metric + + def _metrics_by_id(self) -> "dict[str, MetricMetadata]": + if self._by_id is None: + metric_key = self.dataset_metadata.metric_key + self._by_id = {lookup(metric, metric_key): metric for metric in self.metrics} + return self._by_id + + def _metric_or_none(self, metric_id: str | None) -> "MetricMetadata | None": + if metric_id is None: + if self._data_type == "MULTITSFRAME": + raise TypeError("The metric_id is required for MULTITSFRAME metadata.") + return self.metrics[0] + return self._metrics_by_id().get(metric_id) + + def _shared_by_column(self) -> "dict[str, SharedValueDimensionMetadata]": + return _by_column(self.value_dimensions_shared or ()) + + # information of value dimensions + + def value_dimension_info( + self, + fields: str | Sequence[str], + metric_id: str | None = None, + column: str = "value", + *, + include_inherited: bool = False, + ) -> Any: + """Information of a value dimension, trying the fields in turn""" + metric = self._metric_or_none(metric_id) + if metric is None: + return None + return _value_dimension_info( + _fields(fields), + metric, + _by_column(metric.value_dimensions), + self._shared_by_column(), + column, + include_inherited, + ) + + def value_dimension_infos( + self, fields: str | Sequence[str], *, include_inherited: bool = False + ) -> dict[str, dict[str, Any]]: + """Information of the value dimensions by column by metric + + The value dimensions of a metric are its own ones, "value" and the shared ones. + """ + fields = _fields(fields) + shared = self._shared_by_column() + result = {} + for metric_id, metric in self._metrics_by_id().items(): + value_dimensions = _by_column(metric.value_dimensions) + columns = dict.fromkeys([*shared, "value", *value_dimensions]) + result[metric_id] = { + column: _value_dimension_info( + fields, metric, value_dimensions, shared, column, include_inherited + ) + for column in columns + } + return result + + def unit( + self, + metric_id: str | None = None, + column: str = "value", + *, + include_inherited: bool = False, + ) -> str | None: + return self.value_dimension_info( # type: ignore[no-any-return] + "unit", metric_id, column, include_inherited=include_inherited + ) + + def units(self, *, include_inherited: bool = False) -> dict[str, dict[str, str | None]]: + return self.value_dimension_infos("unit", include_inherited=include_inherited) + + def measurement( + self, + metric_id: str | None = None, + column: str = "value", + *, + include_inherited: bool = False, + ) -> str | None: + return self.value_dimension_info( # type: ignore[no-any-return] + "measurement", metric_id, column, include_inherited=include_inherited + ) + + def measurements(self, *, include_inherited: bool = False) -> dict[str, dict[str, str | None]]: + return self.value_dimension_infos("measurement", include_inherited=include_inherited) + + def name( + self, + metric_id: str | None = None, + column: str = "value", + *, + include_inherited: bool = False, + ) -> str | None: + return self.value_dimension_info( # type: ignore[no-any-return] + "name", metric_id, column, include_inherited=include_inherited + ) + + def names(self, *, include_inherited: bool = False) -> dict[str, dict[str, str | None]]: + return self.value_dimension_infos("name", include_inherited=include_inherited) + + def display_name( + self, + metric_id: str | None = None, + column: str = "value", + *, + include_inherited: bool = False, + ) -> str | None: + """The display name, falling back to the name""" + return self.value_dimension_info( # type: ignore[no-any-return] + DISPLAY_NAME_FIELDS, metric_id, column, include_inherited=include_inherited + ) + + def display_names(self, *, include_inherited: bool = False) -> dict[str, dict[str, str | None]]: + return self.value_dimension_infos(DISPLAY_NAME_FIELDS, include_inherited=include_inherited) + + def short_display_name( + self, + metric_id: str | None = None, + column: str = "value", + *, + include_inherited: bool = False, + ) -> str | None: + """The short display name, falling back to the display name and the name""" + return self.value_dimension_info( # type: ignore[no-any-return] + SHORT_DISPLAY_NAME_FIELDS, metric_id, column, include_inherited=include_inherited + ) + + def short_display_names( + self, *, include_inherited: bool = False + ) -> dict[str, dict[str, str | None]]: + return self.value_dimension_infos( + SHORT_DISPLAY_NAME_FIELDS, include_inherited=include_inherited + ) + + # information of metrics + + def metric_info( + self, field: str, metric_id: str | None = None, *, include_inherited: bool = True + ) -> Any: + """Information of a metric, see the module docstring for the fallbacks""" + metric = self._metric_or_none(metric_id) + if metric is None: + return None + return self._metric_info(field, metric, self._shared_by_column(), include_inherited) + + def metric_infos(self, field: str, *, include_inherited: bool = True) -> dict[str, Any]: + shared = self._shared_by_column() + return { + metric_id: self._metric_info(field, metric, shared, include_inherited) + for metric_id, metric in self._metrics_by_id().items() + } + + @staticmethod + def _metric_info( + field: str, + metric: "MetricMetadata", + shared: Mapping[str, "SharedValueDimensionMetadata"], + include_inherited: bool, + ) -> Any: + info = lookup(metric, field) + if info is None: + info = lookup(_by_column(metric.value_dimensions).get("value"), field) + if info is None: + info = lookup(shared.get("value"), field) + if info is None and include_inherited: + info = lookup(metric.inherited, field) + return info + + # time zones, recording, aggregation and the queried interval + + def default_timezone(self, metric_id: str | None = None) -> str | None: + """The default time zone of the metric, otherwise the one of the dataset metadata""" + metric = self._metric_or_none(metric_id) + own = None if metric is None else _non_empty(metric.default_timezone) + return own or dataset_default_timezone(self) + + def default_timezones(self) -> dict[str, str | None]: + return {metric_id: self.default_timezone(metric_id) for metric_id in self._metrics_by_id()} + + def recording(self, metric_id: str | None = None) -> Recording: + """How the values of the metric are recorded + + The frequency with its offset and time zone falls back to the ones of the dataset + metadata as a whole. + """ + dataset_frequency = _dataset_frequency(self) + metric = self._metric_or_none(metric_id) + if metric is None: + return Recording(None, *dataset_frequency, None) + own_frequency = _frequency( + metric, ("data_frequency", "data_frequency_offset", "data_frequency_timezone") + ) + return Recording( + _non_empty(metric.recording_mode), + *(own_frequency if own_frequency[0] is not None else dataset_frequency), + _non_empty(metric.recording_aggregation_method), + ) + + def recordings(self) -> dict[str, Recording]: + return {metric_id: self.recording(metric_id) for metric_id in self._metrics_by_id()} + + def default_aggregation_method(self, metric_id: str | None = None) -> str: + """The default aggregation method of the metric, see the module docstring""" + metric = self._metric_or_none(metric_id) + if metric is None: + return _DEFAULT_AGGREGATION_METHOD + own_method = _non_empty(metric.default_aggregation_method) + if own_method is not None: + return _AGGREGATION_METHOD_ALIASES.get(own_method, own_method) + value_data_type = _plain( + _value_dimension_info( + ("value_data_type",), + metric, + _by_column(metric.value_dimensions), + self._shared_by_column(), + "value", + include_inherited=False, + ) + ) + return _DEFAULT_AGGREGATION_METHOD_BY_VALUE_DATA_TYPE.get( + value_data_type, _DEFAULT_AGGREGATION_METHOD + ) + + def default_aggregation_methods(self) -> dict[str, str]: + return { + metric_id: self.default_aggregation_method(metric_id) + for metric_id in self._metrics_by_id() + } + + def queried_interval(self) -> tuple[datetime.datetime | None, datetime.datetime | None]: + """Start and end of the queried interval (the ref interval), None if missing""" + return ( + _datetime(self.dataset_metadata.ref_interval_start_timestamp), + _datetime(self.dataset_metadata.ref_interval_end_timestamp), + ) + + +def _datetime(timestamp: str | None) -> datetime.datetime | None: + return None if timestamp is None else datetime.datetime.fromisoformat(timestamp) + + +def _dataset_frequency(meta: MetadataReading) -> _Frequency: + return _frequency( + meta.dataset_metadata, + ("ref_data_frequency", "ref_data_frequency_offset", "ref_data_frequency_timezone"), + ) + + +def dataset_default_timezone(meta: MetadataReading) -> str | None: + """The default time zone of the dataset metadata, which applies to metrics without one""" + return _non_empty(meta.dataset_metadata.default_timezone) + + +def dataset_recording(meta: MetadataReading) -> Recording: + """The recording of a metric without metadata: only the frequency of the dataset metadata""" + return Recording(None, *_dataset_frequency(meta), None) diff --git a/src/hdhelpers/metadata/_recording.py b/src/hdhelpers/metadata/_recording.py deleted file mode 100644 index 5ae2dc3..0000000 --- a/src/hdhelpers/metadata/_recording.py +++ /dev/null @@ -1,169 +0,0 @@ -"""How the values of the metrics are recorded and how they should be aggregated by default - -The frequency of the recorded values of a metric falls back as a whole: If the metric has a -"data_frequency", it applies together with the metric's "data_frequency_offset" and -"data_frequency_timezone". Otherwise "ref_data_frequency" of the dataset metadata applies -together with "ref_data_frequency_offset" and "ref_data_frequency_timezone". -""" - -from collections import defaultdict -from dataclasses import dataclass -from typing import Any, cast - -import pandas as pd - -import hdhelpers.metadata._current_convention as _current_convention -from hdhelpers.metadata._helpers import _select_single_metric, _string_or_none - -_FREQUENCY_FIELDS = ("data_frequency", "data_frequency_offset", "data_frequency_timezone") -_DATASET_FREQUENCY_FIELDS = ( - "ref_data_frequency", - "ref_data_frequency_offset", - "ref_data_frequency_timezone", -) - -# default aggregation method of metrics without one by the data type of their values -_DEFAULT_AGGREGATION_METHOD_BY_VALUE_DATA_TYPE = {"boolean": "LAST", "string": "LAST"} -_DEFAULT_AGGREGATION_METHOD = "MEAN" -_AGGREGATION_METHOD_ALIASES = {"AVG": "MEAN"} - - -@dataclass(frozen=True) -class Recording: - """How the values of a metric are recorded, as given in its metadata - - The fields mean the following: - - Attributes: - recording_mode (str | None): "periodic": one value per interval of data_frequency, timestamped with the start of the interval. A missing value is a gap. "on_change": a value is recorded when it changes by more than the deadband of its value dimension and holds until the next one. Missing timestamps are no gaps. "irregular": values at arbitrary points in time. None if unknown. - data_frequency (str | None): Frequency as Pandas date offset alias or timedelta string, e.g. "15min", "1D" or "1MS". For "on_change", the interval after which the value is recorded again at the latest, also without a change (heartbeat). - data_frequency_offset (str | None): Timedelta string by which the intervals of data_frequency are shifted, e.g. "5min". None means 0. - data_frequency_timezone (str | None): Time zone the intervals of data_frequency are aligned to, e.g. "Europe/Berlin". None means UTC. - recording_aggregation_method (str | None): Only for "periodic": the method each value aggregates its interval with, e.g. "MEAN" or "MAX". None if the values are samples, e.g. the value holding at the timestamp. - - data_frequency_offset and data_frequency_timezone belong to data_frequency: They are None - if no data_frequency applies. - """ - - recording_mode: str | None = None - data_frequency: str | None = None - data_frequency_offset: str | None = None - data_frequency_timezone: str | None = None - recording_aggregation_method: str | None = None - - -_Frequency = tuple[str | None, str | None, str | None] -_NO_FREQUENCY: _Frequency = (None, None, None) - - -def _frequency(obj: Any, fields: tuple[str, str, str]) -> _Frequency: - """Frequency, offset and time zone of obj, all None if it has no frequency - - Values which are no (non-empty) strings count as missing. - """ - if not isinstance(obj, dict): - return _NO_FREQUENCY - frequency, offset, timezone = (_string_or_none(obj.get(field)) for field in fields) - if frequency is None: - return _NO_FREQUENCY - return frequency, offset, timezone - - -def _dataset_frequency(attrs: Any) -> _Frequency: - dataset_metadata = attrs.get("dataset_metadata") if isinstance(attrs, dict) else None - return _frequency(dataset_metadata, _DATASET_FREQUENCY_FIELDS) - - -def _recording(metric: dict, dataset_frequency: _Frequency) -> Recording: - own_frequency = _frequency(metric, _FREQUENCY_FIELDS) - frequency, offset, timezone = ( - own_frequency if own_frequency[0] is not None else dataset_frequency - ) - return Recording( - recording_mode=_string_or_none(metric.get("recording_mode")), - data_frequency=frequency, - data_frequency_offset=offset, - data_frequency_timezone=timezone, - recording_aggregation_method=_string_or_none(metric.get("recording_aggregation_method")), - ) - - -def recording_by_metric(data: pd.DataFrame | pd.Series) -> defaultdict[str, Recording]: - """The recording of each metric, falling back to the frequency of the dataset metadata - - The default of the returned defaultdict is the recording of a metric without metadata, i.e. - with the frequency of the dataset metadata. - """ - dataset_frequency = _dataset_frequency(data.attrs) - gathered = _current_convention.gather(data.attrs) - metrics = ( - {} - if gathered is None - else { - metric: gathered_metric.metric for metric, gathered_metric in gathered.metrics.items() - } - ) - return defaultdict( - lambda: Recording(None, *dataset_frequency, None), - {metric: _recording(obj, dataset_frequency) for metric, obj in metrics.items()}, - ) - - -def single_metric_recording(data: pd.DataFrame | pd.Series) -> Recording: - """The recording of the single metric of a Series or SingleTSFrame - - Falls back to the frequency of the dataset metadata, also if the single metric is not found. - """ - return cast( - Recording, - _select_single_metric( - recording_by_metric(data), - data.attrs, - empty=Recording(None, *_dataset_frequency(data.attrs), None), - ), - ) - - -def _default_aggregation_method(method: Any, value_data_type: Any) -> str: - own_method = _string_or_none(method) - if own_method is not None: - return _AGGREGATION_METHOD_ALIASES.get(own_method, own_method) - return _DEFAULT_AGGREGATION_METHOD_BY_VALUE_DATA_TYPE.get( - value_data_type, _DEFAULT_AGGREGATION_METHOD - ) - - -def default_aggregation_method_by_metric(data: pd.DataFrame | pd.Series) -> defaultdict[str, str]: - """The default aggregation method of each metric - - The method of the metric, with "AVG" normalized to its alias "MEAN". If the metric does not - provide one, "LAST" for values of the data type "boolean" or "string" (of the value - dimension "value", see value_dimension_info) and "MEAN" otherwise, which is also the default - of the returned defaultdict. - """ - gathered = _current_convention.gather(data.attrs) - if gathered is None: - return defaultdict(lambda: _DEFAULT_AGGREGATION_METHOD) - methods = _current_convention.own_metric_info(gathered, "default_aggregation_method") - value_data_types = _current_convention.value_dimension_info( - gathered, ["value_data_type"], include_inherited=False - ) - return defaultdict( - lambda: _DEFAULT_AGGREGATION_METHOD, - { - metric: _default_aggregation_method(method, value_data_types[metric]["value"]) - for metric, method in methods.items() - }, - ) - - -def single_metric_default_aggregation_method(data: pd.DataFrame | pd.Series) -> str: - """The default aggregation method of the single metric of a Series or SingleTSFrame""" - return cast( - str, - _select_single_metric( - default_aggregation_method_by_metric(data), - data.attrs, - empty=_DEFAULT_AGGREGATION_METHOD, - ), - ) diff --git a/src/hdhelpers/metadata/_specs.py b/src/hdhelpers/metadata/_specs.py deleted file mode 100644 index 021a638..0000000 --- a/src/hdhelpers/metadata/_specs.py +++ /dev/null @@ -1,201 +0,0 @@ -from glom import Check, Coalesce, Iter, Merge, Spec, T - -# The metadatum specs of the individual accessors, including their fallback chains. -# Defined once here and shared by the MultiTSFrame and the SingleTSFrame accessors, which -# only differ in whether the result is keyed by metric first - not in what they extract. -UNIT: Spec = "unit" -MEASUREMENT: Spec = "measurement" -NAME: Spec = Coalesce("name", default=None) - -# The fallback chains of the display names. For metadata following the current metadata -# convention, each field is looked up on all levels (value dimension, metric, shared value -# dimensions) before the next field is tried, see _current_convention.value_dimension_info. -# DISPLAY_NAME and SHORT_DISPLAY_NAME are the corresponding specs for older metadata structures. -DISPLAY_NAME_FIELDS = ("display_name", "name") -SHORT_DISPLAY_NAME_FIELDS = ("short_display_name", *DISPLAY_NAME_FIELDS) -DISPLAY_NAME: Spec = Coalesce(*DISPLAY_NAME_FIELDS, default=None) -SHORT_DISPLAY_NAME: Spec = Coalesce(*SHORT_DISPLAY_NAME_FIELDS, default=None) - -# Key of the dict in a metric object holding the properties the metric inherits from objects -# higher up in an asset hierarchy, e.g. the latitude of the plant a sensor belongs to. -INHERITED = "inherited" - -# Field of a metric and of the dataset metadata: the time zone the timeseries should be -# evaluated in by default, e.g. for bucketing days when aggregating. -DEFAULT_TIMEZONE = "default_timezone" - - -def by_metric_key_by_val_dimension(metadatum_key: str | Spec) -> Spec: - """Provides glom spec that extracts a metadatum by metric by value dimension - - Only for older metadata structures, metadata following the current metadata convention is - handled by _current_convention. - - The generated glom spec returns a defaultdict of defaultdicts: - {metric_key: {value_dimension_column_name: metadatum_value}} - defaulting to None in the inner default dict. - - Properly falls back to respective field in metric metadatum for the "value" - value_dimension if this value dimension is not explicitely included in the - metadata of the metric. - - Properly falls back to "value_dimensions_shared" metadata if a value_dimension - is not given for a metric if its available there. - - metadatum_key can also be any glom spec. - """ - return Coalesce( - _spec_platform_convention_series_metric(metadatum_key), - _spec_platform_convention_series_value_dim(metadatum_key), - _spec_older_convention1_metric(metadatum_key), - _spec_older_convention1_value_dim(metadatum_key), - _spec_older_convention2(metadatum_key), - _spec_older_convention4(metadatum_key), - default={}, - ) - - -def _spec_older_convention1_value_dim(metadatum_key: str | Spec) -> Spec: - return ( - "by_metric", - Check(instance_of=dict), - _glom_dict_with_keys_of_current_dict_and_values_something_deeper_nested( - ( - Coalesce("value_dimensions", default={}), - Check(instance_of=dict), - _glom_dict_with_keys_of_current_dict_and_values_something_deeper_nested( - Coalesce(metadatum_key) - ), - ) - ), - ) - - -def _spec_older_convention1_metric(metadatum_key: str | Spec) -> Spec: - return ( - "by_metric", - Check(instance_of=dict), - _glom_dict_with_keys_of_current_dict_and_values_something_deeper_nested( - ( - Coalesce("metric", default={}), - Check(instance_of=dict), - {"value": Coalesce(metadatum_key)}, - ) - ), - ) - - -def _spec_older_convention2(metadatum_key: str | Spec) -> Spec: - return ( - "metrics", - Check(instance_of=dict), - _glom_dict_with_keys_of_current_dict_and_values_something_deeper_nested( - {"value": Coalesce(metadatum_key, default=None)} # only SERIES / only value column. - ), - ) - - -def _spec_platform_convention_series_metric(metadatum_key: str | Spec) -> Spec: - return ( - "single_metric_metadata.structured_metadata.metric", - Check(instance_of=dict), - {"series": {"value": Coalesce(metadatum_key, default=None)}}, - ) - - -def _spec_platform_convention_series_value_dim(metadatum_key: str | Spec) -> Spec: - return ( - "single_metric_metadata.structured_metadata.value_dimensions.value", - Check(instance_of=dict), - {"series": {"value": Coalesce(metadatum_key, default=None)}}, - ) - - -def _spec_older_convention4(metadatum_key: str | Spec) -> Spec: - return ( - ( - "metric_metadata", - Check(instance_of=dict), - _glom_dict_with_keys_of_current_dict_and_values_something_deeper_nested( - {"value": Coalesce(metadatum_key, default=None)} # only SERIES / only value column. - ), - ), - ) - - -def spec_by_metric_key(metadatum_key: str | Spec) -> Spec: - """Provides glom spec that extracts a metadatum from the metric objects, keyed by metric - - Only for older metadata structures, metadata following the current metadata convention is - handled by _current_convention. - """ - return Coalesce( - ( # some older, simpler metadata structure - "by_metric", - _glom_dict_with_keys_of_current_dict_and_values_something_deeper_nested( - Coalesce(metadatum_key, default=None) - ), - ), - ( # another older / simplified metadata structure - "metrics", - _glom_dict_with_keys_of_current_dict_and_values_something_deeper_nested( - Coalesce(metadatum_key, default=None) - ), - ), - ) - - -def _glom_dict_with_keys_of_current_dict_and_values_something_deeper_nested( - deeper_glom_spec: Spec, -) -> Spec: - """Create dicts with keys from current dict and values from deeper in their value objects - - This function provides a glom spec to do this. - It uses https://glom.readthedocs.io/en/latest/tutorial.html#data-driven-assignment. - - deeper_glom_spec is the spec to get to the deeper values in each value object. - - .. doctest:: - - >>> from hdhelpers.metadata._specs import _glom_dict_with_keys_of_current_dict_and_values_something_deeper_nested - >>> from glom import glom - >>> data = { - ... 'some_other_field': 'value', - ... 'by_item': { - ... 'item1': { - ... 'metadata': { - ... 'properties': { - ... 'unit': 'kg' - ... } - ... } - ... }, - ... 'item2': { - ... 'info': { - ... 'details': { - ... 'unit': 'meters' - ... } - ... } - ... }, - ... 'item3': { - ... 'unit': 'liters' - ... } - ... } - ... } - >>> glom( - ... data, - ... ("by_item", - ... _glom_dict_with_keys_of_current_dict_and_values_something_deeper_nested( - ... Coalesce( - ... "metadata.properties.unit", "info.details.unit", "unit", default=None - ... ) - ... ), - ... ), - ... ) - {'item1': 'kg', 'item2': 'meters', 'item3': 'liters'} - """ - # T is a stand-in for the target: everything done to T is recorded and executed by glom - return ( - T.items(), # treat it as list of (key, value) tuples - Iter({T[0]: (T[1], deeper_glom_spec)}), - Merge(), - ) diff --git a/src/hdhelpers/metadata/attached.py b/src/hdhelpers/metadata/attached.py new file mode 100644 index 0000000..17f1fe1 --- /dev/null +++ b/src/hdhelpers/metadata/attached.py @@ -0,0 +1,192 @@ +"""Metadata of the conventions attached to pandas objects as model + +The runtime of the hetida designer parses the metadata of each workflow input once and attaches +the model under attrs["hd_metadata"] (META_KEY), in place of the keys of the conventions +("dataset_metadata", "metrics", "value_dimensions_shared"). Other keys of attrs stay as they are. + +pandas deep-copies attrs for (almost) every object derived from another one. The models are +immutable and return themselves when copied, so that all derived objects share the model +instead of copying the metadata. + +Component code reads the metadata via read_metadata and attaches a changed version (see the +with_* methods of the models) to its own result via set_metadata. Everything serializing attrs +to JSON needs attrs_to_json, since the model is no JSON value. +""" + +from collections.abc import Mapping +from typing import Any, Literal, overload + +import pandas as pd +from pydantic import ValidationError + +from hdhelpers.metadata.conventions import ( + METADATA_MODELS, + MetadataConventionError, + MetadataConventionIssue, + MetadataIssueSeverity, + MultiTSFrameMetadata, + SeriesMetadata, + SingleTSFrameMetadata, + TimeseriesDataType, + _TimeseriesMetadataBase, + check_metadata_conventions, +) + +META_KEY = "hd_metadata" + +# the top-level keys of attrs which belong to the conventions, i.e. the fields of the models +CONVENTION_KEYS = frozenset( + field for model in METADATA_MODELS.values() for field in model.model_fields +) + +AnyMetadata = SeriesMetadata | SingleTSFrameMetadata | MultiTSFrameMetadata + + +def infer_data_type(obj: pd.Series | pd.DataFrame) -> TimeseriesDataType: + """SERIES for a Series, MULTITSFRAME for a DataFrame with a "metric" column, else SINGLETSFRAME""" + if isinstance(obj, pd.Series): + return "SERIES" + return "MULTITSFRAME" if "metric" in obj.columns else "SINGLETSFRAME" + + +def _convention_attrs(attrs: Mapping[Any, Any]) -> dict[str, Any]: + return {key: value for key, value in attrs.items() if key in CONVENTION_KEYS} + + +def attach_metadata( + obj: pd.Series | pd.DataFrame, data_type: TimeseriesDataType | None = None +) -> AnyMetadata | None: + """Parse the metadata in obj.attrs and attach the model in place of the convention keys + + For the runtime, once per workflow input. Returns the attached model, None if attrs do not + contain valid metadata of the data type, which then stay untouched. The data type is + inferred from obj if not given, see infer_data_type. Metadata which is already attached + stays as it is. + """ + attached = obj.attrs.get(META_KEY) + if isinstance(attached, _TimeseriesMetadataBase): + return attached # type: ignore[return-value] + convention_attrs = _convention_attrs(obj.attrs) + if len(convention_attrs) == 0: + return None + model = METADATA_MODELS[data_type or infer_data_type(obj)] + try: + meta = model.model_validate(convention_attrs) + except ValidationError: + return None + obj.attrs = {key: value for key, value in obj.attrs.items() if key not in CONVENTION_KEYS} | { + META_KEY: meta + } + return meta # type: ignore[return-value] + + +@overload +def read_metadata( + obj: pd.Series | pd.DataFrame, data_type: Literal["SERIES"], *, required: Literal[True] +) -> SeriesMetadata: ... +@overload +def read_metadata( + obj: pd.Series | pd.DataFrame, data_type: Literal["SERIES"], *, required: bool = False +) -> SeriesMetadata | None: ... +@overload +def read_metadata( + obj: pd.Series | pd.DataFrame, data_type: Literal["SINGLETSFRAME"], *, required: Literal[True] +) -> SingleTSFrameMetadata: ... +@overload +def read_metadata( + obj: pd.Series | pd.DataFrame, data_type: Literal["SINGLETSFRAME"], *, required: bool = False +) -> SingleTSFrameMetadata | None: ... +@overload +def read_metadata( + obj: pd.Series | pd.DataFrame, data_type: Literal["MULTITSFRAME"], *, required: Literal[True] +) -> MultiTSFrameMetadata: ... +@overload +def read_metadata( + obj: pd.Series | pd.DataFrame, data_type: Literal["MULTITSFRAME"], *, required: bool = False +) -> MultiTSFrameMetadata | None: ... +@overload +def read_metadata( + obj: pd.Series | pd.DataFrame, data_type: None = None, *, required: Literal[True] +) -> AnyMetadata: ... +@overload +def read_metadata( + obj: pd.Series | pd.DataFrame, data_type: None = None, *, required: bool = False +) -> AnyMetadata | None: ... +def read_metadata( + obj: pd.Series | pd.DataFrame, + data_type: TimeseriesDataType | None = None, + *, + required: bool = False, +) -> AnyMetadata | None: + """The metadata of obj as model of the conventions + + Returns the attached model. It may belong to another data type than obj, e.g. the metadata + of a SINGLETSFRAME on a Series taken from it, unless data_type requests a specific one. + Without attached metadata, the convention keys of attrs are parsed, as data_type or as the + type inferred from obj (see infer_data_type), without attaching the model. + + Returns None if there is no valid metadata of the data type. With required=True, raises a + MetadataConventionError with the issues found instead. + """ + attached = obj.attrs.get(META_KEY) + if isinstance(attached, _TimeseriesMetadataBase): + if data_type is None or attached._data_type == data_type: + return attached # type: ignore[return-value] + _no_metadata( + required, + f"The attached metadata is {attached._data_type} metadata, not {data_type} metadata.", + ) + return None + + convention_attrs = _convention_attrs(obj.attrs) + if len(convention_attrs) == 0: + _no_metadata(required, "No metadata in attrs.") + return None + data_type = data_type or infer_data_type(obj) + try: + return METADATA_MODELS[data_type].model_validate(convention_attrs) # type: ignore[return-value] + except ValidationError as e: + if not required: + return None + issues = [ + issue + for issue in check_metadata_conventions(convention_attrs, data_type) + if issue.severity == MetadataIssueSeverity.ERROR + ] + raise MetadataConventionError(issues) from e + + +def _no_metadata(required: bool, message: str) -> None: + """None, or with required raise a MetadataConventionError""" + if required: + raise MetadataConventionError( + [ + MetadataConventionIssue( + severity=MetadataIssueSeverity.ERROR, location="", message=message + ) + ] + ) + return None + + +def set_metadata(obj: pd.Series | pd.DataFrame, meta: AnyMetadata) -> None: + """Attach meta to obj, e.g. a changed version to the result of a component + + Replaces attached metadata and convention keys in obj.attrs. Only set metadata on objects + you created: The runtime passes the same input object to all components using it. + """ + obj.attrs = {key: value for key, value in obj.attrs.items() if key not in CONVENTION_KEYS} | { + META_KEY: meta + } + + +def attrs_to_json(attrs: Mapping[Any, Any]) -> dict[Any, Any]: + """attrs with the attached model as convention keys, e.g. for json.dumps or to_parquet + + Optional fields which are null or have their default value are left out. + """ + attached = attrs.get(META_KEY) + if not isinstance(attached, _TimeseriesMetadataBase): + return dict(attrs) + rest = {key: value for key, value in attrs.items() if key != META_KEY} + return rest | attached.model_dump(mode="json") diff --git a/src/hdhelpers/metadata/conventions.py b/src/hdhelpers/metadata/conventions.py new file mode 100644 index 0000000..d2b6caa --- /dev/null +++ b/src/hdhelpers/metadata/conventions.py @@ -0,0 +1,1792 @@ +"""Metadata conventions for timeseries data + +The models in this module specify the hetida designer metadata conventions for the metadata +(attrs) of SERIES, SINGLETSFRAME and MULTITSFRAME objects, one model per data type. See the user +guide page on attached metadata for an introduction. + +The conventions are strict: Metadata adhering to them provides the required fields, possibly as +null if the information is unknown, e.g. the unit of a value dimension. Consumers like components +should nevertheless cope with missing metadata, since validation of metadata received from +adapters is optional. + +The models document the request body of the hetida designer backend endpoint for checking +metadata. Hence they occur as JSON Schemas in the openapi.json of the backend, so that third party +projects, e.g. adapters providing or receiving such metadata, can validate against them. +Additional fields are allowed at every level. The models are immutable, see ConventionModel. + +check_metadata_conventions additionally checks the rules which JSON Schema cannot express, +optionally against the actual data. +""" + +import datetime +import difflib +import json +import math +import re +from collections import Counter +from collections.abc import Mapping +from enum import StrEnum +from functools import cache, lru_cache +from typing import Annotated, Any, ClassVar, Literal, NoReturn, Self +from zoneinfo import available_timezones + +import pandas as pd +from pydantic import ( + AfterValidator, + BaseModel, + BeforeValidator, + ConfigDict, + Field, + GetJsonSchemaHandler, + ModelWrapValidatorHandler, + PrivateAttr, + StrictBool, + StrictFloat, + StrictInt, + StrictStr, + ValidationError, + ValidationInfo, + model_validator, +) +from pydantic.json_schema import JsonSchemaValue +from pydantic_core import CoreSchema, PydanticCustomError + +from hdhelpers.metadata._reading import MetadataReading, lookup + +TimeseriesDataType = Literal["SERIES", "SINGLETSFRAME", "MULTITSFRAME"] + + +class IntervalType(StrEnum): + """Which boundaries belong to a time interval + + closed: both, open: none, left_closed / right_open: only the start, + right_closed / left_open: only the end. + """ + + CLOSED = "closed" + LEFT_CLOSED = "left_closed" + RIGHT_CLOSED = "right_closed" + OPEN = "open" + LEFT_OPEN = "left_open" + RIGHT_OPEN = "right_open" + + +# [0-9] instead of \d, since \d also matches non-ASCII digits in some regex dialects +UTC_TIMESTAMP_PATTERN = ( + r"^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])" + r"T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\.[0-9]+)?(Z|\+00:00)$" +) + + +def _check_point_in_time(timestamp: str) -> str: + """Reject timestamps of the pattern which are no actual points in time, e.g. February 30""" + try: + datetime.datetime.fromisoformat(timestamp) + except ValueError as e: + raise ValueError(f"{timestamp!r} is not an actual point in time: {e}.") from e + return timestamp + + +def _utc_isoformat(value: Any) -> Any: + """A datetime with UTC offset 0 as isoformat string, so that producers can pass datetimes""" + if not isinstance(value, datetime.datetime): + return value + if value.utcoffset() is None: + raise ValueError(f"{value!r} is naive, but needs to be in UTC.") + if value.utcoffset() != datetime.timedelta(0): + raise ValueError(f"{value!r} is not in UTC.") + return value.isoformat() + + +# The pattern specifies the format in the JSON Schema, which cannot express that the date and +# time must exist. Pydantic's datetime type is not used, since it accepts further formats and +# offsets, so that the check would be more lenient than the published JSON Schema. +UtcTimestamp = Annotated[ + str, + Field(pattern=UTC_TIMESTAMP_PATTERN, json_schema_extra={"format": "date-time"}), + AfterValidator(_check_point_in_time), + # last, so that it runs first and the JSON Schema keeps the pattern + BeforeValidator(_utc_isoformat), +] + + +class AggregationMethod(StrEnum): + """How a timeseries should be aggregated by default, e.g. in coarser intervals + + AVG is an alias of MEAN. + """ + + MEAN = "MEAN" + AVG = "AVG" + MEDIAN = "MEDIAN" + SUM = "SUM" + MIN = "MIN" + MAX = "MAX" + FIRST = "FIRST" + LAST = "LAST" + COUNT = "COUNT" + + +class ValueDataType(StrEnum): + """Data type of the values of a value dimension""" + + FLOAT = "float" + INTEGER = "integer" + BOOLEAN = "boolean" + STRING = "string" + + +class RecordingMode(StrEnum): + """When the values of a timeseries are recorded + + periodic: one value per interval of data_frequency, timestamped with the start of the + interval. A missing value is a gap. + on_change: a value is recorded when it changes (by more than the deadband of its value + dimension) and holds until the next one. Missing timestamps are no gaps, as long as + data_frequency (the heartbeat) is not exceeded. + irregular: values at arbitrary points in time. + """ + + PERIODIC = "periodic" + ON_CHANGE = "on_change" + IRREGULAR = "irregular" + + +_FREQUENCY_FORMAT = 'as Pandas date offset alias or timedelta string, e.g. "15min" or "1D"' +_OFFSET_DESCRIPTION = ( + "Offset of the timestamps against the starts of the intervals of {frequency} (see" + ' {timezone}) as timedelta string, e.g. "4min": not negative and smaller than {frequency}. 0' + " if missing or null. Requires {frequency}." +) + +_TIMEZONE_FORMAT = ( + 'as IANA time zone name like "Europe/Berlin" or "UTC" (preferred, since it respects daylight' + ' saving time) or as fixed UTC offset like "+02:00"' +) + +_ALIGNMENT_DESCRIPTION = ( + "Time zone the intervals of {frequency} are aligned to, " + + _TIMEZONE_FORMAT + + ". Without it, the intervals start at the multiples of {frequency} since" + " 1970-01-01T00:00:00Z. With it, for a {frequency} shorter than a day, the intervals restart" + " at each local midnight: They start at local midnight plus multiples of {frequency}, and the" + " last interval of a day ends at the next local midnight, so it may be shorter, e.g. on the" + " day of a daylight saving time change. For a {frequency} of a day or longer, the intervals" + " follow the calendar of the time zone, e.g. days from local midnight to local midnight. The" + " offset shifts the intervals in both cases. Requires {frequency}." +) +_TIMEZONE_PURPOSE = ( + " Components evaluating the timeseries in calendar intervals use it, e.g. a report" + " aggregating days, which start at midnight of this time zone, so the results differ between" + " time zones. This is not the time zone timestamps are displayed in: That is the plot target" + " timezone of the execution context, which depends on the viewer, while this time zone belongs" + " to the data, e.g. the time zone of the plant." +) + + +def _immutable(self: Any, *args: Any, **kwargs: Any) -> NoReturn: + raise TypeError(f"{type(self).__name__} is immutable") + + +class FrozenDict(dict[str, Any]): + """A dict which cannot be changed + + Holds dict values of extra fields of the convention models, since the models are shared + between pandas objects instead of being copied. Use it as a read-only mapping, not as a dict: + The builtin frozendict of Python 3.15, which is planned to replace it, is no dict subclass. + """ + + __slots__ = () + __setitem__ = __delitem__ = __ior__ = _immutable # type: ignore[assignment] + clear = pop = popitem = setdefault = update = _immutable # type: ignore[assignment] + + def __copy__(self) -> Self: + return self + + def __deepcopy__(self, memo: dict[int, Any]) -> Self: + return self + + def __reduce__(self) -> tuple[Any, ...]: + return (FrozenDict, (dict(self),)) + + +def _freeze(value: Any) -> Any: + """A JSON value with dicts and lists as FrozenDicts and tuples, recursively""" + if isinstance(value, dict): + if not all(isinstance(key, str) for key in value): + raise ValueError("Keys of JSON objects must be strings.") + return FrozenDict({key: _freeze(item) for key, item in value.items()}) + if isinstance(value, list | tuple): + return tuple(_freeze(item) for item in value) + if ( + value is None + or isinstance(value, str | int) + or (isinstance(value, float) and math.isfinite(value)) + ): + return value + raise ValueError(f"{value!r} is not a JSON value.") + + +_JsonFloat = Annotated[StrictFloat, Field(allow_inf_nan=False)] + +# Scalars are validated within pydantic-core. Only dicts and lists need Python for freezing them. +_FrozenValue = Annotated[ + StrictStr + | StrictInt + | _JsonFloat + | StrictBool + | None + | Annotated[Any, AfterValidator(_freeze)], + Field(union_mode="left_to_right"), +] + + +def optional(default: Any = None, **kwargs: Any) -> Any: + """Field for an optional value, left out of dumps while it is None or the default""" + return Field(default, exclude_if=lambda value: value is None or value == default, **kwargs) + + +def _item_count(min_length: int, max_length: int | None = None) -> BeforeValidator: + """Check the number of items before validating them + + Pydantic counts the items of a tuple after validating them, so that an invalid item would + also be reported as missing. The JSON Schema states the counts via json_schema_extra. + """ + + def items(count: int) -> str: + return f"{count} item" if count == 1 else f"{count} items" + + def check(value: Any) -> Any: + if isinstance(value, list | tuple): + if len(value) < min_length: + raise PydanticCustomError( + "too_short", f"Should have at least {items(min_length)}, not {len(value)}" + ) + if max_length is not None and len(value) > max_length: + raise PydanticCustomError( + "too_long", f"Should have at most {items(max_length)}, not {len(value)}" + ) + # a tuple, since strict validation after a before validator rejects lists + return tuple(value) + return value + + return BeforeValidator(check) + + +class ConventionModel(BaseModel): + """Base of the models of the metadata conventions + + The models are immutable, including the values of extra fields, so that pandas objects can + share them instead of copying them: pandas deep-copies the attrs of an object for (almost) + every object derived from it. A changed version of a model is a new model, which shares all + unchanged parts with the original. + """ + + # Validation is lax, so that Python input may contain lists for the tuples. Fields whose type + # lax validation would coerce (bool, int, float) use the strict types instead, so that the + # results equal strict validation of JSON input. + model_config = ConfigDict(extra="allow", frozen=True) + __pydantic_extra__: dict[str, _FrozenValue] + + def __deepcopy__(self, memo: dict[int, Any] | None = None) -> Self: + return self + + def model_copy(self, *, update: Mapping[str, Any] | None = None, deep: bool = False) -> Self: + """Copy with the given fields replaced, without validating them + + Always shallow, since the model is immutable. A deep copy would apply the update to the + original, since __deepcopy__ returns the original. + """ + return super().model_copy(update=update, deep=False) + + @classmethod + def __get_pydantic_json_schema__( + cls, core_schema: CoreSchema, handler: GetJsonSchemaHandler + ) -> JsonSchemaValue: + json_schema = handler(core_schema) + # Any JSON value is allowed in extra fields, as without the typed __pydantic_extra__ + handler.resolve_ref_schema(json_schema)["additionalProperties"] = True + return json_schema + + +class SharedValueDimensionMetadata(ConventionModel): + """Metadata of a value dimension shared by all metrics + + Every column of a MULTITSFRAME besides "timestamp" and "metric" and every column of a + SINGLETSFRAME besides "timestamp" defines a value dimension for each metric. A SERIES has + exactly one value dimension, which is "value". + """ + + column: str = Field( + ..., + description="The actual column name in the MULTITSFRAME or SINGLETSFRAME, connecting the" + ' value dimension to its column. "value" for a SERIES.', + ) + name: str | None = optional( + description='Name of the value dimension, e.g. "temperature". May equal column.' + ) + display_name: str | None = optional( + description="A shorter name for labels, captions or plot titles. Falls back to name.", + ) + short_display_name: str | None = optional( + description="A short name for legends in plots. Falls back to display_name." + ) + description: str | None = optional( + description="A description. Should be considered empty if missing." + ) + unit: str | None = optional( + description='Unit of the values, e.g. "°C" or "m/s", "" for dimensionless quantities', + ) + measurement: str | None = optional(description='The measured quantity, e.g. "temperature"') + value_data_type: ValueDataType | None = optional(description="Data type of the values") + deadband: StrictFloat | None = optional( + ge=0, + allow_inf_nan=False, + description="For a metric with recording_mode on_change: the change of the values, in" + " their unit, which has to be exceeded for a value to be recorded. 0 records every" + " change.", + ) + decimals: StrictInt | None = optional( + ge=0, + description="Number of decimal places numeric values should be displayed with, in their" + " unit, e.g. 2 for 12.35. Only affects how values are displayed, e.g. in tables, tooltips" + " or reports; the values themselves are not rounded. Up to the consumer if missing or" + " null.", + ) + + +class ValueDimensionMetadata(SharedValueDimensionMetadata): + """Metadata of a value dimension of a metric + + Every column of a MULTITSFRAME besides "timestamp" and "metric" and every column of a + SINGLETSFRAME besides "timestamp" defines a value dimension for each metric. A SERIES has + exactly one value dimension, which is "value". Each metric provides an entry for each of its + value dimensions. + """ + + unit: str | None = Field( + ..., + description='Unit of the values, e.g. "°C" or "m/s". "" for dimensionless quantities, null' + " if unknown.", + ) + measurement: str | None = Field( + ..., description='The measured quantity, e.g. "temperature". Null if unknown.' + ) + + +class InheritedMetricMetadata(ConventionModel): + """Properties a metric inherits from objects higher up in a hierarchy + + Arbitrary fields, e.g. the latitude of the plant a sensor belongs to. Typically the deepest + occurrence in the hierarchy wins. Should not contain the fields the conventions define for + metrics and value dimensions, like "name", "id" or "unit": Consumers only use inherited + properties as last resort and not for information describing the metric itself, like its + unit or names. + + The values should have the JSON type matching their meaning: numbers as numbers, booleans as + booleans and points in time as UTC isoformat strings, not e.g. a number as string. + + E.g. the hetida platform provides the effective property values of a signal here, keyed by + property label: its own values, otherwise the value of the nearest asset above it having one. + """ + + +class MetricCommentReply(ConventionModel): + """A reply to a comment attached to a metric""" + + id: str | None = optional(description="Id of the reply, e.g. a UUID") + created_at: UtcTimestamp | None = optional(description="When the reply was created") + edited_at: UtcTimestamp | None = optional( + description="When the reply was last edited. Null or missing if never edited." + ) + content: str | None = optional(description="Free text of the reply") + username: str | None = optional(description="Username of the author of the reply") + + +class MetricComment(ConventionModel): + """A comment attached to a metric, describing a time range of its timeseries data + + E.g. the hetida platform provides the comments on a signal whose time range overlaps the + queried interval and which existed at query time, newest first. + + Start and end of the time range must be provided together. They are equal for a comment on a + single point in time. + """ + + id: str | None = optional(description="Id of the comment, e.g. a UUID") + start_date: UtcTimestamp | None = optional( + description="Start of the time range the comment refers to" + ) + end_date: UtcTimestamp | None = optional( + description="End of the time range the comment refers to" + ) + created_at: UtcTimestamp | None = optional(description="When the comment was created") + edited_at: UtcTimestamp | None = optional( + description="When the comment was last edited. Null or missing if never edited." + ) + content: str | None = optional(description="Free text of the comment") + username: str | None = optional(description="Username of the author of the comment") + tags: tuple[str, ...] | None = optional(description="Free text tags classifying the comment") + replies: tuple[MetricCommentReply, ...] | None = optional( + description="Replies to the comment, oldest first" + ) + + +class MetricMetadata(ConventionModel): + """Metadata of a metric, i.e. of a single, possibly multidimensional timeseries + + A metric is identified by its field named by dataset_metadata.metric_key, whose values occur + in the metric column of a MULTITSFRAME. Every metric must have this field as a string, + unique among the metrics. + """ + + id: str = Field( + ..., + description="Id of the metric, also if another field identifies the metric in the data" + " via dataset_metadata.metric_key. E.g. the hetida platform provides the signal id here.", + ) + name: str = Field(..., description="Long name of the metric") + display_name: str | None = optional( + description="A shorter name for labels, captions or plot titles" + ) + short_display_name: str | None = optional(description="A short name for legends in plots") + description: str | None = optional(description="A one sentence description") + external_id: str | None = optional( + description="Unique id / key in external source systems. Often used to identify the" + " metric in the metric column, i.e. as metric_key.", + ) + signal_id: str | None = optional(description="Actual platform signal id") + channel_id: str | None = optional( + description="Actual platform channel id. Deprecated, since the platform calls channels" + " signals now.", + deprecated='Use "signal_id" instead.', + ) + value_dimensions: Annotated[tuple[ValueDimensionMetadata, ...], _item_count(1)] = Field( + ..., + json_schema_extra={"minItems": 1}, + description="Metadata of the value dimensions of this metric: an entry for each value" + ' column of the data, i.e. exactly the value dimension "value" for a SERIES. Each column' + " may occur at most once.", + ) + default_aggregation_method: AggregationMethod | None = optional( + description="How the timeseries should be aggregated by default, e.g. when components" + " plot or report it in coarser intervals. If missing or null: LAST if the value_data_type" + ' of the value dimension "value" is boolean or string, otherwise MEAN. AVG is an alias' + " of MEAN. E.g. the hetida platform provides the method it aggregates the signal with when" + " bucketing it, for a signal group member the one configured for the member.", + ) + default_timezone: str | None = optional( + description="Time zone the timeseries should be evaluated in by default, " + + _TIMEZONE_FORMAT + + "." + + _TIMEZONE_PURPOSE + + " Falls back to dataset_metadata.default_timezone.", + ) + path: tuple[str, ...] | None = optional( + description="Location of the metric in the hierarchy of its source system: the names of" + " the levels from the top down to the object the metric belongs to, without the metric" + ' itself, e.g. ["Plant A", "Pickling Unit", "Influx"]. Corresponds to the breadcrumb' + " path of the sources of generic REST adapters.", + ) + origin: str | None = optional( + description="Where the values come from, as free text of the source system, e.g." + ' "OPC_UA", "MQTT" or "FORM" for values entered manually.', + ) + recording_mode: RecordingMode | None = optional( + description="When the values are recorded. Unknown if missing or null. periodic requires" + " data_frequency (or dataset_metadata.ref_data_frequency).", + ) + data_frequency: str | None = optional( + description="Interval in which values of the metric are recorded " + + _FREQUENCY_FORMAT + + ": For recording_mode periodic the interval between subsequent values, for on_change" + " the interval after which the value is recorded again at the latest, also without a" + " change (heartbeat). Together with data_frequency_offset and data_frequency_timezone it" + " takes precedence over ref_data_frequency, ref_data_frequency_offset and" + " ref_data_frequency_timezone of the dataset metadata for this metric.", + ) + data_frequency_offset: str | None = optional( + description=_OFFSET_DESCRIPTION.format( + frequency="data_frequency", timezone="data_frequency_timezone" + ), + ) + data_frequency_timezone: str | None = optional( + description=_ALIGNMENT_DESCRIPTION.format(frequency="data_frequency") + ) + recording_aggregation_method: AggregationMethod | None = optional( + description="Only for recording_mode periodic: how each value aggregates the values of" + " the interval [timestamp, timestamp + data_frequency), e.g. MEAN for an average or MAX" + " for a maximum. Null or missing if the values are samples, e.g. the value holding at" + " the timestamp.", + ) + inherited: InheritedMetricMetadata | None = optional() + comments: tuple[MetricComment, ...] | None = optional( + description="Comments attached to the metric, describing situations in its timeseries data", + ) + + +class SignalGroupMetadata(ConventionModel): + """A group of signals a dataset was read from + + E.g. the hetida platform provides the signal group a MULTITSFRAME was read from. + """ + + id: str | None = optional(description="Id of the signal group") + name: str | None = optional(description="Name of the signal group") + description: str | None = optional(description="Description of the signal group") + + +REF_INTERVAL_FIELDS = ( + "ref_interval_start_timestamp", + "ref_interval_end_timestamp", + "ref_interval_type", +) + +# JSON Schema of the rule that the ref interval is required unless ref_dataset_discrete is true +_REF_INTERVAL_REQUIRED_SCHEMA: dict[str, Any] = { + "if": { + "properties": {"ref_dataset_discrete": {"const": True}}, + "required": ["ref_dataset_discrete"], + }, + "else": { + "required": list(REF_INTERVAL_FIELDS), + "properties": {field: {"type": "string"} for field in REF_INTERVAL_FIELDS}, + }, +} + + +class _DatasetMetadataBase(ConventionModel): + model_config = ConfigDict(json_schema_extra=_REF_INTERVAL_REQUIRED_SCHEMA) + + metric_key: str = Field( + ..., + description="Which field of the metric objects under metrics identifies a metric, i.e." + ' occurs in the metric column of a MULTITSFRAME, e.g. "id" or "external_id".', + ) + queried_metrics: tuple[str, ...] = Field( + ..., + description="The requested metrics. Since not all queried metrics may have data in the" + " queried interval, a metric may be missing in the data.", + ) + signal_group: SignalGroupMetadata | None = optional() + ref_dataset_discrete: StrictBool | None = optional( + False, + description="Whether the ref dataset should be considered as discrete, i.e. isolated" + " datapoints instead of an interval. If true, the ref interval is not required.", + ) + ref_interval_start_timestamp: UtcTimestamp | None = optional( + description="Start of the queried time interval (ref interval) in UTC isoformat." + " Required unless ref_dataset_discrete is true.", + ) + ref_interval_end_timestamp: UtcTimestamp | None = optional( + description="End of the queried time interval (ref interval) in UTC isoformat." + " Required unless ref_dataset_discrete is true.", + ) + ref_interval_type: IntervalType | None = optional( + description="Type of the ref interval. Required unless ref_dataset_discrete is true.", + ) + ref_data_frequency: str | None = optional( + description="Frequency of the timeseries " + + _FREQUENCY_FORMAT + + ": the data_frequency of all metrics without their own, with the same meaning as" + " there. Together with ref_data_frequency_offset and ref_data_frequency_timezone.", + ) + ref_data_frequency_offset: str | None = optional( + description=_OFFSET_DESCRIPTION.format( + frequency="ref_data_frequency", timezone="ref_data_frequency_timezone" + ) + + " Applies to all metrics without their own data_frequency.", + ) + ref_data_frequency_timezone: str | None = optional( + description=_ALIGNMENT_DESCRIPTION.format(frequency="ref_data_frequency") + + " Applies to all metrics without their own data_frequency.", + ) + query_timestamp: UtcTimestamp | None = optional( + description="The point in time whose state of the data was delivered in UTC isoformat:" + " Data stored, changed or invalidated later is not included. A source supporting it" + " delivers the same data again when queried for this point in time, e.g. for" + " reproducing an execution.", + ) + default_aggregation_interval: str | None = optional( + description="Interval to aggregate the timeseries to by default, e.g. when components" + " plot or report them, " + + _FREQUENCY_FORMAT + + ". The metrics are aggregated with their default_aggregation_method. E.g. a hetida" + " platform signal group is configured with such an interval.", + ) + default_timezone: str | None = optional( + description="Time zone the timeseries should be evaluated in by default, " + + _TIMEZONE_FORMAT + + "." + + _TIMEZONE_PURPOSE + + " Applies to all metrics without their own default_timezone.", + ) + invalidation_interval_start: UtcTimestamp | None = optional( + description="Start of an invalidation interval separate from the ref interval in UTC" + " isoformat", + ) + invalidation_interval_end: UtcTimestamp | None = optional( + description="End of an invalidation interval separate from the ref interval in UTC" + " isoformat", + ) + invalidation_interval_type: IntervalType | None = optional( + IntervalType.CLOSED, description="Type of the invalidation interval" + ) + invalidate_dataset: StrictBool | None = optional( + description="Whether existing data should be invalidated: in the invalidation interval if" + " specified, otherwise in the ref interval. If neither is specified, a closed ref interval" + " from the first to the last datapoint is inferred. Without datapoints no invalidation can" + " occur then. What null means is up to the adapter.", + ) + delete_invalidated: StrictBool | None = optional( + description="Whether data to be invalidated should be deleted instead. What null means is" + " up to the adapter. Adapters do not need to support deletion.", + ) + invalidation_timestamp: UtcTimestamp | None = optional( + description="Invalidation timestamp for all invalidated datapoints. If null, the" + " execution_timestamp query parameter sent to generic REST adapters or, if that is not" + " present either, the current time should be used.", + ) + only_invalidate: StrictBool | None = optional( + False, + description="If true, existing data should only be invalidated and the new data should" + " not be stored. Cannot be true if invalidate_dataset is false.", + ) + new_data_invalidation_date: UtcTimestamp | None = optional( + description="An invalidation date to be stored together with all new datapoints, i.e." + " invalidating them in advance. Ignored if null.", + ) + + +class SingleMetricDatasetMetadata(_DatasetMetadataBase): + """Metadata associated with a SERIES or SINGLETSFRAME as a whole + + E.g. the queried time interval or how existing data should be handled when the dataset is + sent to an adapter. The fields concerning invalidation and deletion are only relevant for + adapters actually supporting these operations, like the built-in SQL adapter. Other adapters + may ignore them. + + Start and end of an interval must be provided together. + """ + + single_metric: str = Field( + ..., description="The metric_key field value of the single metric under metrics." + ) + + +class MultiTSFrameDatasetMetadata(_DatasetMetadataBase): + """Metadata associated with a MULTITSFRAME as a whole + + E.g. the queried time interval or how existing data should be handled when the dataset is + sent to an adapter. The fields concerning invalidation and deletion are only relevant for + adapters actually supporting these operations, like the built-in SQL adapter. Other adapters + may ignore them. + + Start and end of an interval must be provided together. + """ + + single_metric: str | None = optional( + description="If the MULTITSFRAME holds a single metric: its metric_key field value. Must" + " not be set if more than one metric is present.", + ) + + +class _TimeseriesMetadataBase(MetadataReading, ConventionModel): + """Base of the root models, the metadata of a SERIES, SINGLETSFRAME or MULTITSFRAME + + The methods of MetadataReading look up information with the fallbacks of the conventions. + The with_* methods return changed versions, which share all unchanged parts with this one. + """ + + _data_type: ClassVar[TimeseriesDataType] + # the metrics by id, built on first use + _by_id: dict[str, MetricMetadata] | None = PrivateAttr(default=None) + + value_dimensions_shared: tuple[SharedValueDimensionMetadata, ...] | None = optional( + description="Metadata of value dimensions that have the same meaning for all metrics." + " Consumers use it as fallback, but it does not replace the value dimensions of the" + " metrics. Each column may occur at most once.", + ) + + @model_validator(mode="wrap") + @classmethod + def _conforms( + cls, data: Any, handler: ModelWrapValidatorHandler[Self], info: ValidationInfo + ) -> Self: + """Reject metadata violating the rules of the conventions which JSON Schema cannot express + + check_metadata_conventions reports these errors itself, at their exact locations. + """ + model = handler(data) + if (info.context or {}).get(_SKIP_RULES) is True: + return model + errors = _rule_errors(data, model, cls._data_type) + if len(errors) > 0: + raise MetadataConventionError(errors) + return model + + def model_copy(self, *, update: Mapping[str, Any] | None = None, deep: bool = False) -> Self: + copied = super().model_copy(update=update, deep=deep) + copied._by_id = None # the metrics may differ + return copied + + def with_metric(self, metric_id: str | None = None, /, **changes: Any) -> Self: + """A version with fields of a metric changed, the other metrics are shared + + The changed metric is validated, including the rules of the conventions which concern it. + """ + old = self.metric(metric_id) + new = type(old).model_validate(old.model_dump() | changes) + errors = _metric_rule_errors(self, new) + new_id = lookup(new, self.dataset_metadata.metric_key) + if new_id != lookup(old, self.dataset_metadata.metric_key) and new_id in self.metric_ids(): + errors.append(_error("metrics", f'Metric "{new_id}" occurs more than once.')) + if len(errors) > 0: + raise MetadataConventionError(errors) + return self.model_copy( + update={"metrics": tuple(new if metric is old else metric for metric in self.metrics)} + ) + + def with_value_dimension( + self, metric_id: str | None = None, column: str = "value", /, **changes: Any + ) -> Self: + """A version with fields of a value dimension of a metric changed, see with_metric""" + metric = self.metric(metric_id) + if column not in {value_dimension.column for value_dimension in metric.value_dimensions}: + raise KeyError(column) + return self.with_metric( + metric_id, + value_dimensions=tuple( + value_dimension.model_dump() | changes + if value_dimension.column == column + else value_dimension + for value_dimension in metric.value_dimensions + ), + ) + + def with_dataset_metadata(self, **changes: Any) -> Self: + """A version with fields of the dataset metadata changed, the metrics are shared + + Validated including all rules of the conventions, since the dataset metadata concerns + all metrics. + """ + old = self.dataset_metadata + changed = self.model_copy( + update={"dataset_metadata": type(old).model_validate(old.model_dump() | changes)} + ) + errors = _rule_errors(None, changed, self._data_type) + if len(errors) > 0: + raise MetadataConventionError(errors) + return changed + + +def _value_dimension(column: str, unit: str | None, measurement: str | None) -> dict[str, Any]: + return {"column": column, "unit": unit, "measurement": measurement} + + +SERIES_METADATA_EXAMPLE: dict[str, Any] = { + "dataset_metadata": { + "metric_key": "id", + "single_metric": "7ab8c419-8745-4714-b588-57c0c772cc95", + "queried_metrics": ["7ab8c419-8745-4714-b588-57c0c772cc95"], + "ref_interval_start_timestamp": "2026-01-01T00:00:00+00:00", + "ref_interval_end_timestamp": "2026-01-02T00:00:00+00:00", + "ref_interval_type": "right_open", + "query_timestamp": "2026-01-02T00:05:00+00:00", + }, + "metrics": [ + { + "id": "7ab8c419-8745-4714-b588-57c0c772cc95", + "name": "Water level north", + "value_dimensions": [_value_dimension("value", "m", "level") | {"decimals": 2}], + "default_aggregation_method": "MEAN", + "default_timezone": "Europe/Berlin", + "path": ["Waterworks North", "Gauge North"], + "origin": "OPC_UA", + "recording_mode": "periodic", + "data_frequency": "15min", + "recording_aggregation_method": "MEAN", + } + ], +} + +SINGLETSFRAME_METADATA_EXAMPLE: dict[str, Any] = { + "dataset_metadata": { + "metric_key": "external_id", + "single_metric": "abc.temp", + "queried_metrics": ["abc.temp"], + "ref_interval_start_timestamp": "2026-01-01T00:00:00+00:00", + "ref_interval_end_timestamp": "2026-01-02T00:00:00+00:00", + "ref_interval_type": "closed", + }, + "metrics": [ + { + "id": "3f2b1c4d-5e6f-4a7b-8c9d-0e1f2a3b4c5d", + "external_id": "abc.temp", + "name": "ABC temperature", + "value_dimensions": [ + _value_dimension("value", "°C", "temperature"), + _value_dimension("state", None, None) | {"name": "measurement state"}, + ], + } + ], +} + +MULTITSFRAME_METADATA_EXAMPLE: dict[str, Any] = { + "dataset_metadata": { + "metric_key": "external_id", + "queried_metrics": ["plant_a.pressure", "plant_a.temperature"], + "ref_interval_start_timestamp": "2026-01-01T00:00:00+00:00", + "ref_interval_end_timestamp": "2026-01-02T00:00:00+00:00", + "ref_interval_type": "closed", + "default_aggregation_interval": "1h", + "default_timezone": "Europe/Berlin", + }, + "metrics": [ + { + "id": "c5a8e2f1-0b3d-4e6a-9f7c-1d2e3f4a5b6c", + "external_id": "plant_a.pressure", + "name": "Plant A pressure", + "value_dimensions": [_value_dimension("value", "bar", "pressure")], + }, + { + "id": "8d7c6b5a-4f3e-4d2c-8b1a-0f9e8d7c6b5a", + "external_id": "plant_a.temperature", + "name": "Plant A temperature", + "value_dimensions": [ + _value_dimension("value", "°C", "temperature") | {"deadband": 0.5, "decimals": 1} + ], + "default_aggregation_method": "MAX", + "recording_mode": "on_change", + "data_frequency": "1h", + }, + ], +} + +_SINGLE_METRIC_DESCRIPTION = ( + "Metadata of the single metric: exactly one metric, which has the field named by" + " dataset_metadata.metric_key with the value of dataset_metadata.single_metric." +) + + +# the metric id field set when a metric gets another id, if the metric key field is taken +_METRIC_NAME_FIELD = "metric" + + +def _renamed[T: SharedValueDimensionMetadata]( + value_dimensions: tuple[T, ...] | None, column: str, new_column: str +) -> tuple[T, ...] | None: + """Only the value dimensions of column, renamed to new_column, None if there are none""" + renamed = tuple( + value_dimension.model_copy(update={"column": new_column}) + for value_dimension in value_dimensions or () + if value_dimension.column == column + ) + return renamed if len(renamed) > 0 else None + + +def _with_value_column(metric: MetricMetadata, column: str, new_column: str) -> MetricMetadata: + """The metric with only the value dimension of column, renamed to new_column""" + return metric.model_copy( + update={"value_dimensions": _renamed(metric.value_dimensions, column, new_column) or ()} + ) + + +class SeriesMetadata(_TimeseriesMetadataBase): + """hetida designer metadata conventions for a SERIES + + Structure of the metadata in the attrs attribute of a SERIES during execution, which is also + exchanged with adapters, e.g. via the Data-Attributes header of the generic REST adapter + timeseries endpoint (for each timeseries) or via the __metadata__ field of the wrapped format + of the direct provisioning adapter. + + The fields that are not required are optional. Additional fields are allowed at every level. + Timestamps are UTC isoformat strings with explicit offset (Z or +00:00). + """ + + model_config = ConfigDict(json_schema_extra={"examples": [SERIES_METADATA_EXAMPLE]}) + + _data_type: ClassVar[TimeseriesDataType] = "SERIES" + + dataset_metadata: SingleMetricDatasetMetadata + metrics: Annotated[tuple[MetricMetadata, ...], _item_count(1, 1)] = Field( + ..., + json_schema_extra={"minItems": 1, "maxItems": 1}, + description=_SINGLE_METRIC_DESCRIPTION, + ) + + def as_singletsframe(self, column: str = "value") -> "SingleTSFrameMetadata": + """The metadata of the SINGLETSFRAME with the values of the SERIES in column + + Renames the value dimension "value" of the metric and the shared one to column. + """ + return SingleTSFrameMetadata.model_validate( + { + "dataset_metadata": self.dataset_metadata, + "metrics": (_with_value_column(self.metric(), "value", column),), + "value_dimensions_shared": _renamed(self.value_dimensions_shared, "value", column), + } + ) + + +class SingleTSFrameMetadata(_TimeseriesMetadataBase): + """hetida designer metadata conventions for a SINGLETSFRAME + + Structure of the metadata in the attrs attribute of a SINGLETSFRAME during execution, which + is also exchanged with adapters, e.g. via the Data-Attributes header of the generic REST + adapter singletsframe endpoint or via the __metadata__ field of the wrapped format of the + direct provisioning adapter. + + The fields that are not required are optional. Additional fields are allowed at every level. + Timestamps are UTC isoformat strings with explicit offset (Z or +00:00). + """ + + model_config = ConfigDict(json_schema_extra={"examples": [SINGLETSFRAME_METADATA_EXAMPLE]}) + + _data_type: ClassVar[TimeseriesDataType] = "SINGLETSFRAME" + + dataset_metadata: SingleMetricDatasetMetadata + metrics: Annotated[tuple[MetricMetadata, ...], _item_count(1, 1)] = Field( + ..., + json_schema_extra={"minItems": 1, "maxItems": 1}, + description=_SINGLE_METRIC_DESCRIPTION, + ) + + def as_series(self, column: str = "value") -> SeriesMetadata: + """The metadata of the SERIES with the values of column + + Keeps only the value dimensions of column, renamed to "value". KeyError if the metric + has no value dimension of column. + """ + if column not in {vd.column for vd in self.metric().value_dimensions}: + raise KeyError(column) + return SeriesMetadata.model_validate( + { + "dataset_metadata": self.dataset_metadata, + "metrics": (_with_value_column(self.metric(), column, "value"),), + "value_dimensions_shared": _renamed(self.value_dimensions_shared, column, "value"), + } + ) + + def as_multitsframe(self, metric_id: str | None = None) -> "MultiTSFrameMetadata": + """The metadata of the MULTITSFRAME of the single metric, named metric_id if given + + If metric_id differs from the id of the metric, it becomes its id: in the field named by + the metric key if the metric does not have it, otherwise in the field "metric", which + then becomes the metric key. + """ + metric = self.metric() + dataset_metadata = self.dataset_metadata.model_dump() + if metric_id is not None and metric_id != self.metric_ids()[0]: + metric_key = dataset_metadata["metric_key"] + if lookup(metric, metric_key) is not None: + metric_key = dataset_metadata["metric_key"] = _METRIC_NAME_FIELD + metric = type(metric).model_validate(metric.model_dump() | {metric_key: metric_id}) + metric_id = lookup(metric, dataset_metadata["metric_key"]) + return MultiTSFrameMetadata.model_validate( + { + "dataset_metadata": dataset_metadata + | {"single_metric": metric_id, "queried_metrics": [metric_id]}, + "metrics": (metric,), + "value_dimensions_shared": self.value_dimensions_shared, + } + ) + + +class MultiTSFrameMetadata(_TimeseriesMetadataBase): + """hetida designer metadata conventions for a MULTITSFRAME + + Structure of the metadata in the attrs attribute of a MULTITSFRAME during execution, which + is also exchanged with adapters, e.g. via the Data-Attributes header of the generic REST + adapter multitsframe endpoint or via the __metadata__ field of the wrapped format of the + direct provisioning adapter. + + The fields that are not required are optional. Additional fields are allowed at every level. + Timestamps are UTC isoformat strings with explicit offset (Z or +00:00). + """ + + model_config = ConfigDict(json_schema_extra={"examples": [MULTITSFRAME_METADATA_EXAMPLE]}) + + _data_type: ClassVar[TimeseriesDataType] = "MULTITSFRAME" + + dataset_metadata: MultiTSFrameDatasetMetadata + metrics: tuple[MetricMetadata, ...] = Field( + ..., + description="Metadata of the metrics: one for each metric in the data. Each metric must" + " have the field named by dataset_metadata.metric_key as a string, unique among the" + " metrics.", + ) + + def narrowed_to(self, metric_id: str) -> SingleTSFrameMetadata: + """The metadata of the SINGLETSFRAME of one metric, KeyError if it has no metadata + + The metric is shared, not copied. It becomes the single and only queried metric. + """ + return SingleTSFrameMetadata.model_validate( + { + "dataset_metadata": self.dataset_metadata.model_dump() + | {"single_metric": metric_id, "queried_metrics": [metric_id]}, + "metrics": (self.metric(metric_id),), + "value_dimensions_shared": self.value_dimensions_shared, + } + ) + + +METADATA_MODELS: dict[str, type[_TimeseriesMetadataBase]] = { + "SERIES": SeriesMetadata, + "SINGLETSFRAME": SingleTSFrameMetadata, + "MULTITSFRAME": MultiTSFrameMetadata, +} + + +class MetadataIssueSeverity(StrEnum): + """Severity of an issue found when checking metadata against the metadata conventions + + error: Violates the conventions, so helpers and components may misinterpret the metadata. + warning: Likely a mistake, e.g. metadata not matching the data. + """ + + ERROR = "error" + WARNING = "warning" + + +class MetadataConventionIssue(BaseModel): + """Issue found when checking metadata against the metadata conventions""" + + severity: MetadataIssueSeverity + location: str = Field( + ..., + description="Where the issue occurs, e.g. metrics[0].value_dimensions[1].column in the" + " checked metadata. Empty for the checked object as a whole.", + ) + message: str + + +class MetadataConventionError(ValueError): + """Metadata violating the metadata conventions, with the issues found""" + + def __init__(self, issues: list[MetadataConventionIssue]) -> None: + self.issues = issues + super().__init__( + "Violates the metadata conventions: " + + _listing([f"{issue.location}: {issue.message}" for issue in issues], 5) + ) + + +def _error(location: str, message: str) -> MetadataConventionIssue: + return MetadataConventionIssue( + severity=MetadataIssueSeverity.ERROR, location=location, message=message + ) + + +def _warning(location: str, message: str) -> MetadataConventionIssue: + return MetadataConventionIssue( + severity=MetadataIssueSeverity.WARNING, location=location, message=message + ) + + +def _location(loc: tuple[int | str, ...]) -> str: + """Format a pydantic error location like metrics[0].value_dimensions[1].column""" + location = "" + for part in loc: + if isinstance(part, int): + location += f"[{part}]" + else: + location += f".{part}" if location != "" else part + return location + + +def _listing(items: list[str], max_items: int = 10) -> str: + listed = ", ".join(f'"{item}"' for item in items[:max_items]) + if len(items) > max_items: + listed += f" and {len(items) - max_items} more" + return listed + + +def _check_duplicate_columns( + value_dimensions: list[dict[str, Any]], location: str +) -> list[MetadataConventionIssue]: + counts = Counter(value_dimension["column"] for value_dimension in value_dimensions) + return [ + _error(location, f'Value dimension column "{column}" occurs more than once.') + for column, count in counts.items() + if count > 1 + ] + + +def _check_interval( + obj: dict[str, Any], location: str, start_key: str, end_key: str +) -> list[MetadataConventionIssue]: + """Check an interval given by start and end fields of the object at location""" + start = obj.get(start_key) + end = obj.get(end_key) + if (start is None) != (end is None): + missing_key = start_key if start is None else end_key + return [ + _error( + f"{location}.{missing_key}", + f"Start and end of an interval must be provided together, but {missing_key}" + " is missing.", + ) + ] + if start is not None and end is not None and pd.Timestamp(start) > pd.Timestamp(end): + return [_error(f"{location}.{start_key}", f"Start is after end {end}.")] + return [] + + +@lru_cache(maxsize=1024) +def _is_frequency(value: str) -> bool: + """Whether value is a Pandas date offset alias or a timedelta string + + A unit is required, since e.g. "1" would be a timedelta string of 1 nanosecond. + """ + if re.search("[A-Za-z]", value) is None: + return False + try: + pd.tseries.frequencies.to_offset(value) + except ValueError: + try: + # e.g. "nan" gives NaT + return bool(pd.notna(pd.to_timedelta(value))) + except ValueError: + return False + return True + + +@lru_cache(maxsize=1024) +def _duration(value: str) -> pd.Timedelta | None: + """The fixed duration of a frequency, None if calendar based like "MS" or no frequency""" + try: + offset = pd.tseries.frequencies.to_offset(value) + except ValueError: + try: + # e.g. "P1D" or "1 days" + duration = pd.to_timedelta(value) + except ValueError: + return None + return None if pd.isna(duration) else duration + if isinstance(offset, pd.offsets.Day): + # a calendar day since pandas 3, i.e. 24 hours in UTC + return pd.Timedelta(days=offset.n) + if isinstance(offset, pd.offsets.Tick): + return pd.Timedelta(offset) + # calendar based, e.g. months + return None + + +@lru_cache(maxsize=1024) +def _offset_count(value: str) -> int: + """The count n of a calendar based frequency, e.g. 3 for "3MS\"""" + return int(pd.tseries.frequencies.to_offset(value).n) + + +def _not_a_frequency(location: str, value: str) -> MetadataConventionIssue: + return _error( + location, + f'"{value}" is not a Pandas date offset alias or timedelta string, e.g. "15min".', + ) + + +def _check_frequency( + obj: dict[str, Any], field: str, location: str +) -> list[MetadataConventionIssue]: + """Check that a frequency field is a positive frequency""" + value = obj.get(field) + if value is None: + return [] + if not _is_frequency(value): + return [_not_a_frequency(f"{location}.{field}", value)] + duration = _duration(value) + positive = duration > pd.Timedelta(0) if duration is not None else _offset_count(value) > 0 + return [] if positive else [_error(f"{location}.{field}", f'"{value}" is not positive.')] + + +def _check_offset( + obj: dict[str, Any], frequency_field: str, offset_field: str, location: str +) -> list[MetadataConventionIssue]: + """Check the offset of the timestamps against the intervals of a valid frequency""" + offset = obj[offset_field] + offset_location = f"{location}.{offset_field}" + if not _is_frequency(offset): + return [_not_a_frequency(offset_location, offset)] + offset_duration = _duration(offset) + if offset_duration is None or offset_duration < pd.Timedelta(0): + return [ + _error(offset_location, f'"{offset}" is not a non-negative timedelta, e.g. "4min".') + ] + # None for calendar based frequencies, which offsets cannot be compared with + frequency_duration = _duration(obj[frequency_field]) + if frequency_duration is not None and offset_duration >= frequency_duration: + return [ + _error( + offset_location, + f'"{offset}" is not smaller than {frequency_field} "{obj[frequency_field]}".', + ) + ] + return [] + + +def _check_frequency_with_alignment( + obj: dict[str, Any], frequency_field: str, offset_field: str, timezone_field: str, location: str +) -> list[MetadataConventionIssue]: + """Check a frequency and the fields aligning its intervals, the offset and the time zone + + They are only meaningful together with the frequency on the same level, since the frequency + of a metric falls back to the one of the dataset metadata as a whole. + """ + issues = _check_frequency(obj, frequency_field, location) + valid_frequency = len(issues) == 0 and obj.get(frequency_field) is not None + for field in (offset_field, timezone_field): + if obj.get(field) is not None and obj.get(frequency_field) is None: + issues.append(_error(f"{location}.{field}", f"Requires {frequency_field}.")) + issues += _check_timezone(obj, location, timezone_field) + if obj.get(offset_field) is not None and valid_frequency: + issues += _check_offset(obj, frequency_field, offset_field, location) + return issues + + +_UTC_OFFSET_PATTERN = re.compile(r"^[+-](0[0-9]|1[0-4]):[0-5][0-9]$") + + +@cache +def _timezone_names() -> set[str]: + # Independent of the system, unlike ZoneInfo, which also accepts e.g. "posixrules" + return available_timezones() + + +def _is_timezone(value: str) -> bool: + """Whether value is an IANA time zone name or a fixed UTC offset, e.g. "+02:00" """ + return _UTC_OFFSET_PATTERN.match(value) is not None or value in _timezone_names() + + +def _check_timezone( + obj: dict[str, Any], location: str, field: str = "default_timezone" +) -> list[MetadataConventionIssue]: + timezone = obj.get(field) + if timezone is None or _is_timezone(timezone): + return [] + return [ + _error( + f"{location}.{field}", + f'"{timezone}" is neither an IANA time zone name like "Europe/Berlin" nor a UTC' + ' offset like "+02:00".', + ) + ] + + +def _check_recording( + metric: dict[str, Any], dataset_metadata: dict[str, Any], location: str +) -> list[MetadataConventionIssue]: + """Check that the recording information of a metric fits its recording mode""" + issues = [] + recording_mode = metric.get("recording_mode") + if ( + metric.get("recording_aggregation_method") is not None + and recording_mode != RecordingMode.PERIODIC + ): + issues.append( + _error( + f"{location}.recording_aggregation_method", + "Only meaningful for recording_mode periodic.", + ) + ) + if ( + recording_mode == RecordingMode.PERIODIC + and metric.get("data_frequency") is None + and dataset_metadata.get("ref_data_frequency") is None + ): + issues.append( + _error( + f"{location}.recording_mode", + "periodic requires data_frequency or dataset_metadata.ref_data_frequency, the" + " interval of the values.", + ) + ) + if recording_mode != RecordingMode.ON_CHANGE: + issues += [ + _warning( + f"{location}.value_dimensions[{vd_index}].deadband", + "Only meaningful if the recording_mode of the metric is on_change.", + ) + for vd_index, value_dimension in enumerate(metric["value_dimensions"]) + if value_dimension.get("deadband") is not None + ] + return issues + + +def _known_fields(model: type[BaseModel]) -> set[str]: + return set(model.model_fields) + + +# hints for fields placed where the conventions do not expect them +_MISPLACED_FIELD_HINTS: dict[tuple[type[BaseModel], str], str] = { + (MetricMetadata, "unit"): 'Provide the unit of the value dimension "value" under' + " value_dimensions instead.", + (MetricMetadata, "measurement"): 'Provide the measurement of the value dimension "value"' + " under value_dimensions instead.", +} + + +def _unknown_field_issues( + obj: dict[str, Any], model: type[BaseModel], location: str, ignored: set[str] | None = None +) -> list[MetadataConventionIssue]: + """Warnings for fields not defined by the conventions, which may be misspelled""" + known = _known_fields(model) + issues = [] + for field in obj: + if field in known or (ignored is not None and field in ignored): + continue + hint = _MISPLACED_FIELD_HINTS.get((model, field)) + if hint is None: + snake_case = re.sub(r"(? 0 + else "Additional fields are allowed, but make sure it is not misspelled." + ) + issues.append( + _warning( + f"{location}.{field}" if location != "" else field, + f'Unknown field "{field}". {hint}', + ) + ) + return issues + + +def _dicts(value: Any) -> list[tuple[int, dict[str, Any]]]: + """The dicts in value with their index, if value is a list""" + if not isinstance(value, list): + return [] + return [(index, item) for index, item in enumerate(value) if isinstance(item, dict)] + + +def _check_unknown_fields( + attrs: Any, model: type[_TimeseriesMetadataBase] +) -> list[MetadataConventionIssue]: + """Warnings for fields not defined by the conventions + + Also applied to structurally invalid metadata as far as possible, since a misspelled field + often explains a missing required one. + """ + if not isinstance(attrs, dict): + return [] + issues = _unknown_field_issues(attrs, model, "") + + dataset_metadata = attrs.get("dataset_metadata") + metric_key = None + if isinstance(dataset_metadata, dict): + metric_key = dataset_metadata.get("metric_key") + dataset_metadata_model = model.model_fields["dataset_metadata"].annotation + assert isinstance(dataset_metadata_model, type) # for mypy # noqa: S101 + issues += _unknown_field_issues( + dataset_metadata, dataset_metadata_model, "dataset_metadata" + ) + if isinstance(dataset_metadata.get("signal_group"), dict): + issues += _unknown_field_issues( + dataset_metadata["signal_group"], + SignalGroupMetadata, + "dataset_metadata.signal_group", + ) + + for index, value_dimension in _dicts(attrs.get("value_dimensions_shared")): + issues += _unknown_field_issues( + value_dimension, SharedValueDimensionMetadata, f"value_dimensions_shared[{index}]" + ) + for index, metric in _dicts(attrs.get("metrics")): + location = f"metrics[{index}]" + # the field identifying the metric may be any field + issues += _unknown_field_issues(metric, MetricMetadata, location, ignored={str(metric_key)}) + for vd_index, value_dimension in _dicts(metric.get("value_dimensions")): + issues += _unknown_field_issues( + value_dimension, ValueDimensionMetadata, f"{location}.value_dimensions[{vd_index}]" + ) + for comment_index, comment in _dicts(metric.get("comments")): + comment_location = f"{location}.comments[{comment_index}]" + issues += _unknown_field_issues(comment, MetricComment, comment_location) + for reply_index, reply in _dicts(comment.get("replies")): + issues += _unknown_field_issues( + reply, MetricCommentReply, f"{comment_location}.replies[{reply_index}]" + ) + return issues + + +_DEPRECATED_METRIC_FIELDS = { + name: field.deprecated + for name, field in MetricMetadata.model_fields.items() + if field.deprecated is not None +} + + +def _check_consistency(attrs: dict[str, Any]) -> list[MetadataConventionIssue]: + """Check the rules of the conventions which the JSON Schema cannot express + + Expects attrs to be valid against the JSON Schema of the data type. + """ + issues: list[MetadataConventionIssue] = [] + dataset_metadata = attrs["dataset_metadata"] + metrics = attrs["metrics"] + metric_key = dataset_metadata["metric_key"] + + metric_ids = [] + for index, metric in enumerate(metrics): + metric_id = metric.get(metric_key) + if not isinstance(metric_id, str): + issues.append( + _error( + f"metrics[{index}]", + f'Metric has no string field "{metric_key}", which identifies metrics' + " according to dataset_metadata.metric_key.", + ) + ) + else: + metric_ids.append(metric_id) + issues += [ + _warning(f"metrics[{index}].{name}", f"Deprecated. {deprecated}") + for name, deprecated in _DEPRECATED_METRIC_FIELDS.items() + if metric.get(name) is not None + ] + issues += _check_duplicate_columns( + metric["value_dimensions"], f"metrics[{index}].value_dimensions" + ) + issues += _check_frequency_with_alignment( + metric, + "data_frequency", + "data_frequency_offset", + "data_frequency_timezone", + f"metrics[{index}]", + ) + issues += _check_timezone(metric, f"metrics[{index}]") + issues += _check_recording(metric, dataset_metadata, f"metrics[{index}]") + for comment_index, comment in enumerate(metric.get("comments") or []): + issues += _check_interval( + comment, f"metrics[{index}].comments[{comment_index}]", "start_date", "end_date" + ) + issues += [ + _error("metrics", f'Metric "{metric_id}" occurs more than once.') + for metric_id, count in Counter(metric_ids).items() + if count > 1 + ] + issues += _check_duplicate_columns( + attrs.get("value_dimensions_shared") or [], "value_dimensions_shared" + ) + + single_metric = dataset_metadata.get("single_metric") + if single_metric is not None and len(metrics) > 1: + issues.append( + _error( + "dataset_metadata.single_metric", + f"Must not be set, since {len(metrics)} metrics are present under metrics.", + ) + ) + elif single_metric is not None and len(metric_ids) == 1 and single_metric not in metric_ids: + issues.append( + _error( + "dataset_metadata.single_metric", + f'"{single_metric}" is not the "{metric_key}" of the metric under metrics.', + ) + ) + elif single_metric is not None and single_metric not in dataset_metadata["queried_metrics"]: + issues.append(_error("dataset_metadata.single_metric", "Missing in queried_metrics.")) + + if dataset_metadata.get("only_invalidate") is True and ( + dataset_metadata.get("invalidate_dataset") is False + ): + issues.append( + _error( + "dataset_metadata.only_invalidate", + "Cannot be true if invalidate_dataset is false.", + ) + ) + + if dataset_metadata.get("ref_dataset_discrete") is not True: + issues += [ + _error( + f"dataset_metadata.{field}", + "Required unless dataset_metadata.ref_dataset_discrete is true.", + ) + for field in REF_INTERVAL_FIELDS + if dataset_metadata.get(field) is None + ] + issues += _check_interval( + dataset_metadata, + "dataset_metadata", + "ref_interval_start_timestamp", + "ref_interval_end_timestamp", + ) + issues += _check_interval( + dataset_metadata, + "dataset_metadata", + "invalidation_interval_start", + "invalidation_interval_end", + ) + issues += _check_frequency_with_alignment( + dataset_metadata, + "ref_data_frequency", + "ref_data_frequency_offset", + "ref_data_frequency_timezone", + "dataset_metadata", + ) + issues += _check_frequency(dataset_metadata, "default_aggregation_interval", "dataset_metadata") + issues += _check_timezone(dataset_metadata, "dataset_metadata") + return issues + + +_CLOSED_BY_INTERVAL_TYPE: dict[str, Literal["both", "neither", "left", "right"]] = { + "closed": "both", + "open": "neither", + "left_closed": "left", + "right_open": "left", + "right_closed": "right", + "left_open": "right", +} + + +def _check_data_in_ref_interval( + dataset_metadata: dict[str, Any], data: pd.Series | pd.DataFrame +) -> list[MetadataConventionIssue]: + """Warn if the data has timestamps outside of the ref interval""" + start = dataset_metadata.get("ref_interval_start_timestamp") + end = dataset_metadata.get("ref_interval_end_timestamp") + interval_type = dataset_metadata.get("ref_interval_type") + timestamps = data.index if isinstance(data, pd.Series) else data.get("timestamp") + if ( + dataset_metadata.get("ref_dataset_discrete") is True + or start is None + or end is None + or interval_type is None + or pd.Timestamp(start) > pd.Timestamp(end) + or timestamps is None + or len(timestamps) == 0 + or not pd.api.types.is_datetime64_any_dtype(timestamps) + ): + return [] + ref_interval = pd.Interval( + pd.Timestamp(start), pd.Timestamp(end), closed=_CLOSED_BY_INTERVAL_TYPE[interval_type] + ) + # naive timestamps are considered to be UTC + first, last = ( + timestamp if timestamp.tzinfo is not None else timestamp.tz_localize("UTC") + for timestamp in (timestamps.min(), timestamps.max()) + ) + if first in ref_interval and last in ref_interval: + return [] + return [ + _warning( + "dataset_metadata", + f"The data has timestamps outside of the {interval_type} ref interval: The first is" + f" {first.isoformat()}, the last {last.isoformat()}.", + ) + ] + + +def _check_metrics_of_data( + attrs: dict[str, Any], data_type: TimeseriesDataType, data: pd.Series | pd.DataFrame | None +) -> list[MetadataConventionIssue]: + """Check that the metrics of the data and the queried metrics are described""" + issues: list[MetadataConventionIssue] = [] + dataset_metadata = attrs["dataset_metadata"] + metric_ids = {metric.get(dataset_metadata["metric_key"]) for metric in attrs["metrics"]} + data_metrics: list[str] = [] + if data_type == "MULTITSFRAME" and isinstance(data, pd.DataFrame) and "metric" in data.columns: + data_metrics = [str(metric) for metric in data["metric"].unique()] + if len(missing := sorted(set(data_metrics) - metric_ids)) > 0: + issues.append( + _error( + "metrics", + f"Metrics of the metric column missing under metrics: {_listing(missing)}", + ) + ) + queried_metrics = dataset_metadata["queried_metrics"] + if len(unqueried := sorted(set(data_metrics) - set(queried_metrics))) > 0: + issues.append( + _error( + "dataset_metadata.queried_metrics", + f"Metrics of the metric column missing in queried_metrics:" + f" {_listing(unqueried)}", + ) + ) + # queried metrics occurring in the data are reported as missing under metrics above + undescribed_metrics = sorted( + set(dataset_metadata["queried_metrics"]) - metric_ids - set(data_metrics) + ) + if len(undescribed_metrics) > 0: + issues.append( + _warning( + "metrics", + f"Queried metrics without metadata under metrics: {_listing(undescribed_metrics)}", + ) + ) + return issues + + +def _check_consistency_with_data_type( + attrs: dict[str, Any], data_type: TimeseriesDataType, data: pd.Series | pd.DataFrame | None +) -> list[MetadataConventionIssue]: + """Check the rules specific to the data type and whether the metadata matches the data""" + metrics = attrs["metrics"] + issues = _check_metrics_of_data(attrs, data_type, data) + if data is not None: + issues += _check_data_in_ref_interval(attrs["dataset_metadata"], data) + + # the value columns of frames are only known from their data + value_columns: set[str] | None = None + if data_type == "SERIES": + value_columns = {"value"} + elif isinstance(data, pd.DataFrame): + value_columns = set(data.columns) - ( + {"timestamp", "metric"} if data_type == "MULTITSFRAME" else {"timestamp"} + ) + if value_columns is None: + return issues + + for index, metric in enumerate(metrics): + described_columns = { + value_dimension["column"] for value_dimension in metric["value_dimensions"] + } + if len(undescribed := sorted(value_columns - described_columns)) > 0: + issues.append( + _error( + f"metrics[{index}].value_dimensions", + f"Value columns of the {data_type} without value dimension:" + f" {_listing(undescribed)}", + ) + ) + + value_dimension_lists = [ + (f"metrics[{index}].value_dimensions", metric["value_dimensions"]) + for index, metric in enumerate(metrics) + ] + [("value_dimensions_shared", attrs.get("value_dimensions_shared") or [])] + for location, value_dimensions in value_dimension_lists: + for index, value_dimension in enumerate(value_dimensions): + if value_dimension["column"] not in value_columns: + issues.append( + _warning( + f"{location}[{index}].column", + f'Column "{value_dimension["column"]}" is not a value column of the' + f" {data_type}.", + ) + ) + return issues + + +_SKIP_RULES = "skip_rules_of_metadata_conventions" + + +def _rule_errors( + data: Any, model: _TimeseriesMetadataBase, data_type: TimeseriesDataType +) -> list[MetadataConventionIssue]: + """The errors of the rules which JSON Schema cannot express, for valid structure + + The rules are checked on the input if it is in JSON form, otherwise on the model's dump, e.g. + for input with nested models. + """ + + def errors(attrs: dict[str, Any]) -> list[MetadataConventionIssue]: + issues = _check_consistency(attrs) + _check_consistency_with_data_type( + attrs, data_type, None + ) + return [issue for issue in issues if issue.severity == MetadataIssueSeverity.ERROR] + + if isinstance(data, dict): + try: + return errors(data) + except (AttributeError, KeyError, TypeError): + pass + return errors(model.model_dump(mode="json")) + + +def _metric_rule_errors( + meta: _TimeseriesMetadataBase, metric: MetricMetadata +) -> list[MetadataConventionIssue]: + """The rule errors concerning a single metric, together with the dataset metadata""" + attrs = { + "dataset_metadata": meta.dataset_metadata.model_dump(mode="json"), + "metrics": [metric.model_dump(mode="json")], + } + if meta.value_dimensions_shared is not None: + attrs["value_dimensions_shared"] = [ + value_dimension.model_dump(mode="json") + for value_dimension in meta.value_dimensions_shared + ] + issues = _check_consistency(attrs) + _check_consistency_with_data_type( + attrs, meta._data_type, None + ) + return [issue for issue in issues if issue.severity == MetadataIssueSeverity.ERROR] + + +def check_metadata_conventions( + obj: pd.Series | pd.DataFrame | Mapping[str, Any], + data_type: TimeseriesDataType | None = None, +) -> list[MetadataConventionIssue]: + """Check metadata against the metadata conventions of its data type + + obj is either a SERIES, SINGLETSFRAME or MULTITSFRAME, whose attrs are then also checked for + consistency with its data, or the metadata itself. For metadata, data_type is required. For + data it is inferred if not provided: A DataFrame with a "metric" column is considered to be a + MULTITSFRAME, otherwise a SINGLETSFRAME. + + The metadata is checked in its JSON representation, as it is exchanged with adapters. + Consistency is only checked if the structure is valid. + + Returns the found issues, i.e. an empty list if the metadata adheres to the conventions. + """ + # imported here, since hdhelpers.metadata.attached imports this module + from hdhelpers.metadata.attached import attrs_to_json # noqa: PLC0415 + + data = obj if isinstance(obj, pd.Series | pd.DataFrame) else None + attrs = data.attrs if data is not None else obj + if isinstance(attrs, Mapping): + # attached metadata in its JSON form + attrs = attrs_to_json(attrs) + if data_type is None and isinstance(data, pd.Series): + data_type = "SERIES" + elif data_type is None and isinstance(data, pd.DataFrame): + data_type = "MULTITSFRAME" if "metric" in data.columns else "SINGLETSFRAME" + elif data_type is None: + raise ValueError("The data type is required for checking metadata without data.") + model = METADATA_MODELS[data_type] + + try: + attrs_json = json.dumps(attrs, allow_nan=False) + except (TypeError, ValueError) as e: + return [ + _error( + "", + f"Metadata is not JSON serializable, so it cannot be exchanged with adapters: {e}", + ) + ] + + try: + model.model_validate_json(attrs_json, context={_SKIP_RULES: True}) + except ValidationError as e: + issues = [] + for error in e.errors(): + message = error["msg"].removeprefix("Value error, ") + if (error.get("ctx") or {}).get("pattern") == UTC_TIMESTAMP_PATTERN: + message = ( + "Should be a UTC isoformat timestamp with explicit offset, i.e. ending with Z" + f" or +00:00, got {error['input']!r}." + ) + elif ( + error["loc"] == ("metrics",) + and isinstance(attrs, Mapping) + and isinstance(attrs.get("metrics"), dict) + ): + message += ( + ". Metrics as a mapping is an older metadata convention, the current one" + " expects a list of metric objects." + ) + issues.append(_error(_location(error["loc"]), message)) + return issues + _check_unknown_fields(json.loads(attrs_json), model) + + json_attrs = json.loads(attrs_json) + return ( + _check_consistency(json_attrs) + + _check_consistency_with_data_type(json_attrs, data_type, data) + + _check_unknown_fields(json_attrs, model) + ) diff --git a/src/hdhelpers/metadata/helpers.py b/src/hdhelpers/metadata/helpers.py index 0d68e19..9d62ad2 100644 --- a/src/hdhelpers/metadata/helpers.py +++ b/src/hdhelpers/metadata/helpers.py @@ -1,1404 +1,570 @@ -"""Helper functions for metadata extraction +"""Functions accessing the metadata of SERIES, SINGLETSFRAMEs and MULTITSFRAMEs -This module provides functions to help extracting information from metadata -provided as .attrs with pandas DataFrame / Series objects following the hetida -designer metadata conventions. +The functions are thin wrappers over the methods of the models of the metadata conventions, +see read_metadata. The methods resolve the fallbacks of the conventions, e.g. a missing display +name falls back to the name, see hdhelpers.metadata._reading. -They properly cascade defaults / fallbacks -and try to provide backwards compatible access to metadata for different versions -of the metadata conventions or simpler metadata structures. +The metadata is the model attached to the object (see attach_metadata), whatever data type it +belongs to, e.g. the metadata of a SINGLETSFRAME on a Series taken from it. Otherwise the +metadata in attrs is parsed as the data type of the function. Metadata which does not follow +the conventions gives no information: None or empty defaultdicts. For a SERIES or SINGLETSFRAME, +the single metric is the one named by dataset_metadata.single_metric, otherwise the only one. + +Fields of the functions taking one can be dotted paths, e.g. "inherited.site". """ import datetime from collections import defaultdict -from typing import Any, cast +from collections.abc import Callable +from typing import Any import pandas as pd -from glom import Coalesce, Spec, glom -import hdhelpers.metadata._specs as _specs -from hdhelpers.metadata._recording import ( +from hdhelpers.metadata._reading import ( + DISPLAY_NAME_FIELDS, + SHORT_DISPLAY_NAME_FIELDS, Recording, - default_aggregation_method_by_metric, - recording_by_metric, - single_metric_default_aggregation_method, - single_metric_recording, -) -from hdhelpers.metadata._helpers import ( - check_dataframe, - check_series, - check_series_or_dataframe, - default_timezone_by_metric, - extract_from_metadata, - get_value_dimension_info, - metric_info_by_metric, - series_value_info, - singlets_metric_info, - singlets_value_dimension_info, - single_metric_default_timezone, - singlets_value_dimension_info_with_fallbacks, - spec_not_none, - value_dimension_info_with_fallbacks, + dataset_default_timezone, + dataset_recording, ) +from hdhelpers.metadata.attached import META_KEY, AnyMetadata, infer_data_type, read_metadata +from hdhelpers.metadata.conventions import TimeseriesDataType, _TimeseriesMetadataBase +_DEFAULT_AGGREGATION_METHOD = "MEAN" -def get_units( - multitsframe: pd.DataFrame, *, include_inherited: bool = False -) -> defaultdict[str, defaultdict[str, str | None]]: - """Gets unit of value dimensions in MTSF metrics from Metadata - Args: - multitsframe (pd.DataFrame): MTSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. +def check_dataframe(data: pd.DataFrame) -> None: + if not isinstance(data, pd.DataFrame): + raise TypeError("Input is not a pd.DataFrame") - Returns: - dict[str, dict[str | None] | None]: Dictionary of metrics containing the names of the value dimensions. - If the short display name of the value_dimension is not present it returns the result of get_value_dimension_info(). - Raises: - TypeError: If `multitsframe` is not a DataFrame. +def check_series(series: pd.Series) -> None: + if not isinstance(series, pd.Series): + raise TypeError("Input is not a pd.Series.") - Code example: - .. doctest:: metadata.get_units +def check_series_or_dataframe(data: pd.Series | pd.DataFrame) -> None: + if not isinstance(data, pd.Series) and not isinstance(data, pd.DataFrame): + raise TypeError("Input is not a pd.Series or pd.DataFrame.") - >>> attr = { - ... "by_metric": { - ... "metric1": { - ... "value_dimensions": { - ... "value_dim_1": { - ... "unit": "m" - ... } - ... } - ... }, - ... "metric3": { - ... "value_dimensions": { - ... "value_dim_1": { - ... "unit": None, - ... } - ... } - ... } - ... } - ... } - >>> dataframe.attrs = attr - >>> result = get_units(dataframe) - >>> result["metric1"]['value_dim_1'] - 'm' - >>> result["metric3"]['value_dim_1'] is None - True - >>> result["metric2"]['value_dim_1'] is None - True - """ - check_dataframe(multitsframe) - return get_value_dimension_info(multitsframe, _specs.UNIT, include_inherited) +def _metadata(data: pd.Series | pd.DataFrame, data_type: TimeseriesDataType) -> AnyMetadata | None: + """The attached metadata of any data type, otherwise the attrs parsed as data_type""" + attached = data.attrs.get(META_KEY) + if isinstance(attached, _TimeseriesMetadataBase): + return attached # type: ignore[return-value] + return read_metadata(data, data_type) -def get_names( - multitsframe: pd.DataFrame, *, include_inherited: bool = False -) -> defaultdict[str, defaultdict[str, str | None]]: - """Gets names of the MTSF metrics from the metadata +def _single_metric(meta: AnyMetadata | None) -> str | None: + """The single metric: dataset_metadata.single_metric if described, otherwise the only one""" + if meta is None: + return None + metric_ids = meta.metric_ids() + single_metric = meta.dataset_metadata.single_metric + if single_metric is not None: + return single_metric if single_metric in metric_ids else None + return metric_ids[0] if len(metric_ids) == 1 else None - Args: - multitsframe (pd.DataFrame): MTSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. - Returns: - dict[str, str | None]: Dictionary of metrics containing the names. If the name is not present for a metric the corresponding value is None. +def _by_metric_by_column( + data: pd.DataFrame, fields: str | tuple[str, ...], include_inherited: bool +) -> defaultdict[str, defaultdict[str, Any]]: + check_dataframe(data) + meta = _metadata(data, "MULTITSFRAME") + infos = ( + {} + if meta is None + else meta.value_dimension_infos(fields, include_inherited=include_inherited) + ) + return defaultdict( + lambda: defaultdict(lambda: None), + {metric: defaultdict(lambda: None, info) for metric, info in infos.items()}, + ) - Raises: - TypeError: If `multitsframe` is not a DataFrame. - Code example: +def _single_metric_by_column( + data: pd.DataFrame, fields: str | tuple[str, ...], include_inherited: bool +) -> defaultdict[str, Any]: + check_dataframe(data) + meta = _metadata(data, "SINGLETSFRAME") + metric = _single_metric(meta) + if meta is None or metric is None: + return defaultdict(lambda: None) + return defaultdict( + lambda: None, + meta.value_dimension_infos(fields, include_inherited=include_inherited)[metric], + ) - .. doctest:: metadata.get_names - >>> attr = { "by_metric": { "metric1": {"metric": {"name": "name_of_metric1"}}, - ... "metric2": {"metric": {"name": None }} }} - >>> dataframe.attrs = attr - >>> result = get_names(dataframe) - >>> result["metric1"]["value"] - 'name_of_metric1' - >>> result["metric2"]["value"] is None - True +def _series_value(series: pd.Series, fields: str | tuple[str, ...], include_inherited: bool) -> Any: + check_series(series) + meta = _metadata(series, "SERIES") + metric = _single_metric(meta) + if meta is None or metric is None: + return None + return meta.value_dimension_info(fields, metric, "value", include_inherited=include_inherited) - Lets try another MTSF format: - .. doctest:: metadata.get_names +def _single_metric_or_dataset( + data: pd.Series | pd.DataFrame, + data_type: TimeseriesDataType, + of_metric: Callable[[AnyMetadata, str], Any], + of_dataset: Callable[[AnyMetadata], Any], + empty: Any, +) -> Any: + """Information of the single metric, of the dataset metadata if the metric is not found""" + meta = _metadata(data, data_type) + if meta is None: + return empty + metric = _single_metric(meta) + return of_dataset(meta) if metric is None else of_metric(meta, metric) - >>> attr = { "dataset_metadata": {"metric_key": "external_id"}, - ... "metrics": [{"external_id": "ruhr-temperature", - ... "name": "Ruhr temperature [°C]", - ... "display_name": "temperature [°C]", - ... "short_display_name": "[°C]", - ... "value_dimensions": [{"column": "temp", "measurement": "temperature", "unit": "°C"}]}]} - >>> dataframe.attrs = attr - >>> result = get_names(dataframe) - >>> result["ruhr-temperature"]["value"] - 'Ruhr temperature [°C]' - """ - check_dataframe(multitsframe) - return get_value_dimension_info(multitsframe, _specs.NAME, include_inherited) +# value dimensions of MULTITSFRAMEs -def get_display_names( +def get_units( multitsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, defaultdict[str, str | None]]: - """Gets display names of the MTSF metrics from the metadata + """Units of the value dimensions of the metrics of a MULTITSFRAME + + The value dimensions of a metric are its own ones, "value" and the shared ones + (value_dimensions_shared). Args: - multitsframe (pd.DataFrame): MTSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. + multitsframe (pd.DataFrame): MULTITSFRAME with metadata following the conventions. + include_inherited (bool): Whether the properties a metric inherits from objects higher up in an asset hierarchy ("inherited") are the last resort for the value dimension "value". Defaults to False, since this information should be provided for the metric itself. Returns: - defaultdict[str, defaultdict[str, str | None]]: Dictionary of metrics containing the display names. - If the display name of the metrics is not present it returns the result of :func:`hdhelpers.metadata.get_names`. - For metadata following the current convention, the display name is looked up on the value dimension, on - the metric (for the value dimension "value") and in "value_dimensions_shared" before falling back to the - name. Fields provided as null count as missing. - - Raises: - TypeError: If `multitsframe` is not a DataFrame. + defaultdict[str, defaultdict[str, str | None]]: The unit by column by metric, None if unknown. Code example: - .. doctest:: metadata.get_display_names - - >>> attr = { "by_metric": { "metric1": {"metric": {"display_name": "display_name_of_metric1"}}, - ... "metric2": {"metric": {"name": "name_of_metric2"}}}} - >>> dataframe.attrs = attr - >>> result = get_display_names(dataframe) - >>> result["metric1"]["value"] - 'display_name_of_metric1' - >>> result["metric2"]["value"] - 'name_of_metric2' - - Lets try another MTSF format: - - .. doctest:: metadata.get_display_names - - >>> attr = { "dataset_metadata": {"metric_key": "external_id"}, - ... "metrics": [{"external_id": "ruhr-temperature", - ... "name": "Ruhr temperature [°C]", - ... "display_name": "temperature [°C]", - ... "short_display_name": "[°C]", - ... "value_dimensions": [{"column": "temp", "measurement": "temperature", "unit": "°C"}]}]} - >>> dataframe.attrs = attr - >>> result = get_display_names(dataframe) - >>> result["ruhr-temperature"]["value"] - 'temperature [°C]' - """ + .. doctest:: metadata.get_units - check_dataframe(multitsframe) - return value_dimension_info_with_fallbacks( - multitsframe, _specs.DISPLAY_NAME_FIELDS, _specs.DISPLAY_NAME, include_inherited - ) + >>> from hdhelpers.metadata.conventions import MULTITSFRAME_METADATA_EXAMPLE + >>> multitsframe.attrs = MULTITSFRAME_METADATA_EXAMPLE + >>> units = get_units(multitsframe) + >>> units["plant_a.temperature"]["value"] + '°C' + >>> units["unknown metric"]["value"] is None + True + """ # noqa: E501 + return _by_metric_by_column(multitsframe, "unit", include_inherited) -def get_short_display_names( +def get_names( multitsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, defaultdict[str, str | None]]: - """Gets short display names of the MTSF metrics from the metadata + """Names of the value dimensions of the metrics of a MULTITSFRAME, see get_units - Args: - multitsframe (pd.DataFrame): MTSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. - - Returns: - defaultdict[str, defaultdict[str, str | None]]: Dictionary of metrics containing the short display names. - If the short display name of the metrics is not present it returns the result of :func:`hdhelpers.metadata.get_display_names`. - For metadata following the current convention, each of short display name, display name and name is - looked up on the value dimension, on the metric (for the value dimension "value") and in - "value_dimensions_shared" before falling back to the next one. Fields provided as null count as missing. - - Raises: - TypeError: If `multitsframe` is not a DataFrame. + The name of the value dimension "value" falls back to the name of the metric. Code example: - .. doctest:: metadata.get_short_display_names - - >>> attr = { "by_metric": { "metric1": {"metric": {"short_display_name": "short_display_name_of_metric1"}}, - ... "metric2": {"metric": {"name": "name_of_metric2"}}, - ... "metric3": {"metric": {"name": None}} }} - >>> dataframe.attrs = attr - >>> result = get_short_display_names(dataframe) - >>> result["metric1"]["value"] - 'short_display_name_of_metric1' - >>> result["metric2"]["value"] - 'name_of_metric2' - >>> result["metric3"]["value"] is None - True - - Lets try another MTSF format: - - .. doctest:: metadata.get_short_display_names + .. doctest:: metadata.get_names - >>> attr = { "dataset_metadata": {"metric_key": "external_id"}, - ... "metrics": [{"external_id": "ruhr-temperature", - ... "name": "Ruhr temperature [°C]", - ... "display_name": "temperature [°C]", - ... "short_display_name": "temp. [°C]", - ... "value_dimensions": [{"column": "temp", "measurement": "temperature", "unit": "°C"}]}]} - >>> dataframe.attrs = attr - >>> result = get_short_display_names(dataframe) - >>> result["ruhr-temperature"]["value"] - 'temp. [°C]' + >>> from hdhelpers.metadata.conventions import MULTITSFRAME_METADATA_EXAMPLE + >>> multitsframe.attrs = MULTITSFRAME_METADATA_EXAMPLE + >>> get_names(multitsframe)["plant_a.pressure"]["value"] + 'Plant A pressure' """ - - check_dataframe(multitsframe) - return value_dimension_info_with_fallbacks( - multitsframe, _specs.SHORT_DISPLAY_NAME_FIELDS, _specs.SHORT_DISPLAY_NAME, include_inherited - ) + return _by_metric_by_column(multitsframe, "name", include_inherited) -def get_measurements( +def get_display_names( multitsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, defaultdict[str, str | None]]: - """Gets measurement (type) of the MTSF metrics from the metadata + """Display names of the value dimensions of the metrics of a MULTITSFRAME, see get_units - Args: - multitsframe (pd.DataFrame): MTSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. - - Returns: - defaultdict[str, defaultdict[str, str | None]]: Dictionary of metrics containing the measurement (type) of the MTSF metrics. - If the short measurement (type) of the MTSF metrics is not present it returns None. + Fall back to the names. + """ + return _by_metric_by_column(multitsframe, DISPLAY_NAME_FIELDS, include_inherited) - Raises: - TypeError: If `multitsframe` is not a DataFrame. - Code example: +def get_short_display_names( + multitsframe: pd.DataFrame, *, include_inherited: bool = False +) -> defaultdict[str, defaultdict[str, str | None]]: + """Short display names of the value dimensions of the metrics of a MULTITSFRAME - .. doctest:: metadata.get_measurements + Fall back to the display names and the names, see get_units. + """ + return _by_metric_by_column(multitsframe, SHORT_DISPLAY_NAME_FIELDS, include_inherited) - >>> attr = { "dataset_metadata": {"metric_key": "external_id"}, - ... "metrics": [{"external_id": "column_name", "value_dimensions": [{"column": "temp", "measurement": "temperature"}]}]} - >>> dataframe.attrs = attr - >>> result = get_measurements(dataframe) - >>> result["column_name"]["temp"] - 'temperature' +def get_measurements( + multitsframe: pd.DataFrame, *, include_inherited: bool = False +) -> defaultdict[str, defaultdict[str, str | None]]: + """Measured quantities of the value dimensions of the metrics of a MULTITSFRAME + See get_units. """ - check_dataframe(multitsframe) - return get_value_dimension_info(multitsframe, _specs.MEASUREMENT, include_inherited) + return _by_metric_by_column(multitsframe, "measurement", include_inherited) def get_metric_info( - multitsframe: pd.DataFrame, metric_info: str | Spec, *, include_inherited: bool = True + multitsframe: pd.DataFrame, metric_info: str, *, include_inherited: bool = True ) -> defaultdict[str, Any]: - """Gets a dictionary of metadata associated to metrics. - - In contrast to metadata associated to concrete value dimensions, this - function abstracts access to metadata associated to the underlying metric. + """A field of the metrics of a MULTITSFRAME by metric - The information is looked up in the metric itself, then in its value dimension "value", - then in "value_dimensions_shared" and finally in the properties the metric inherits from - objects higher up in an asset hierarchy (given under "inherited" in the metric metadata), - e.g. the latitude of the plant a sensor belongs to. The latter can be switched off via - `include_inherited`. Information provided as null counts as missing. Note that in contrast - to this function, the functions for dedicated information like :func:`get_units` or - :func:`get_display_names` do not fall back to inherited properties by default. + The field of the metric itself, otherwise of its value dimension "value", of the shared value + dimension "value" and, if include_inherited, of the properties the metric inherits. Args: - multitsframe (pd.DataFrame): MTSF with metadata following the convention. - metric_info (str | Spec): Information to retrieve, interpreted as a glom Spec: e.g. a field name, a dotted path like "sensor.manufacturer" into sub dictionaries or any other glom Spec. See "How to query nested information?" in the first steps. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the metric, its value dimension "value" nor "value_dimensions_shared" provide the information. Defaults to True. + multitsframe (pd.DataFrame): MULTITSFRAME with metadata following the conventions. + metric_info (str): Field or dotted path of fields, e.g. "inherited.site". + include_inherited (bool): Whether to fall back to the inherited properties. Returns: - defaultdict[str, Any]: Dictionary, where keys are the entries of the metrics metadata specified via "metric_key" in "dataset_metadata" and values are the entries specified via "metric_info" in the metrics metadata. Defaults to None, also if there is no metric metadata at all. + defaultdict[str, Any]: The information by metric, None if unknown. Code example: .. doctest:: metadata.get_metric_info - >>> multitsframe.attrs = { - ... "dataset_metadata": { - ... "metric_key": "id" - ... }, - ... "metrics": [ - ... { - ... "id": "first", - ... "external_id": "external_first", - ... "unit": "m", - ... "display_name": "first display name", - ... "sensor": {"manufacturer": "Acme"}, - ... "value_dimensions": [ - ... { - ... "column": "temp", - ... "unit": "C", - ... "measurement": "temperature" - ... } - ... ] - ... }, - ... { - ... "id": "second", - ... "name": "second name", - ... "external_id": "external_second", - ... "value_dimensions": [ - ... { - ... "column": "temp", - ... "unit": "C" - ... } - ... ] - ... } - ... ] - ... } - >>> result = get_metric_info(multitsframe, "external_id") - >>> result["first"] - 'external_first' - >>> result["second"] - 'external_second' - >>> result["not-given"] is None - True - - A dotted path reaches into sub dictionaries: - - .. doctest:: metadata.get_metric_info - - >>> result = get_metric_info(multitsframe, "sensor.manufacturer") - >>> result["first"] - 'Acme' - >>> result["second"] is None - True + >>> from hdhelpers.metadata.conventions import MULTITSFRAME_METADATA_EXAMPLE + >>> multitsframe.attrs = MULTITSFRAME_METADATA_EXAMPLE + >>> get_metric_info(multitsframe, "recording_mode")["plant_a.temperature"] + 'on_change' """ check_dataframe(multitsframe) - return defaultdict( - lambda: None, metric_info_by_metric(multitsframe, metric_info, include_inherited) + meta = _metadata(multitsframe, "MULTITSFRAME") + infos = ( + {} if meta is None else meta.metric_infos(metric_info, include_inherited=include_inherited) ) + return defaultdict(lambda: None, infos) -def get_series_info( - series: pd.Series, value_dim_info: str | Spec, *, include_inherited: bool = True -) -> Any: - """Gets an arbitrary series info +# the value dimension "value" of SERIES - Since a series has only one value dimension named "value", this information is - equivalent to information on the metric. - Args: - series (pd.Series): Series with metadata following the convention. - value_dim_info (str | Spec): Information to retrieve, interpreted as a glom Spec: e.g. a field name, a dotted path like "sensor.manufacturer" into sub dictionaries or any other glom Spec. See "How to query nested information?" in the first steps. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to True. +def get_series_info( + series: pd.Series, value_dim_info: str, *, include_inherited: bool = True +) -> Any: + """A field of the value dimension "value" of the single metric of a SERIES - Returns: - Any: Retrieved information defined by `value_dim_info` + Looked up on the value dimension, the metric, the shared value dimension and, if + include_inherited, the properties the metric inherits. Code example: .. doctest:: metadata.get_series_info - >>> attr = { "by_metric": { "series": {"value_dimensions": {"value": {"unit": "m"}}}}} - >>> series.attrs = attr - >>> get_series_info(series, "unit") - 'm' - >>> get_series_info(series, "not-given") is None - True + >>> from hdhelpers.metadata.conventions import SERIES_METADATA_EXAMPLE + >>> series.attrs = SERIES_METADATA_EXAMPLE + >>> get_series_info(series, "decimals") + 2 """ - - check_series(series) - return _series_info(series, (value_dim_info,), value_dim_info, include_inherited) - - -def _series_info( - series: pd.Series, fields: tuple[str | Spec, ...], legacy_spec: Spec, include_inherited: bool -) -> Any: - """Info of a Series, falling back to further fields - - Since a Series has only one value dimension named "value", it falls back to the metric - metadata. For metadata following the current metadata convention, each field is looked up - on the value dimension, the metric and "value_dimensions_shared" before the next field is - tried, see value_dimension_info_with_fallbacks. Older metadata structures are handled via - legacy_spec, keeping their established behaviour. - """ - from_metrics = series_value_info(series, fields, legacy_spec, include_inherited) - if from_metrics is not None: - return from_metrics - - # compatibility with some older format - return glom( - series.attrs, - Coalesce( - spec_not_none( - ( - "single_metric_metadata.structured_metadata.value_dimensions.value", - legacy_spec, - ) - ), - spec_not_none( - ( - "single_metric_metadata.structured_metadata.metric", - legacy_spec, - ) - ), - default=None, - ), - ) - - -def _series_info_with_fallbacks( - series: pd.Series, fields: tuple[str | Spec, ...], include_inherited: bool -) -> Any: - """Info of a Series, falling back to further fields, which may be provided as null - - Older metadata structures keep their established behaviour of using the first of the - fields present on the first level providing any. - """ - legacy_spec = Coalesce(*(spec_not_none(field) for field in fields)) - return _series_info(series, fields, legacy_spec, include_inherited) + return _series_value(series, value_dim_info, include_inherited) def get_series_unit(series: pd.Series, *, include_inherited: bool = False) -> str | None: - """Gets name of the series from metadata - - Args: - series (pd.Series): Series with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. - - Returns: - str | None: - Returns the unit of series. - If the unit of the series is not present it returns None. - - Raises: - TypeError: If `series` is not a Series. - - Let's test what happens if series has no attr: - - .. doctest:: metadata.get_series_unit - - >>> series.attrs = {} - >>> get_series_unit(series) is None - True - - Let's test what happens if series has attr but no entry for unit: - - .. doctest:: metadata.get_series_unit - - >>> attr = { "by_metric": { "series": {"value_dimensions": {"value": {"unit":None}}}}} - >>> series.attrs = attr - >>> get_series_unit(series) is None - True + """Unit of the values of a SERIES - Let's test what happens if series has unit in attr: + Code example: .. doctest:: metadata.get_series_unit - >>> attr = { "by_metric": { "series": {"value_dimensions": {"value": {"unit":"m/s"}}}}} - >>> series.attrs = attr + >>> from hdhelpers.metadata.conventions import SERIES_METADATA_EXAMPLE + >>> series.attrs = SERIES_METADATA_EXAMPLE >>> get_series_unit(series) - 'm/s' + 'm' """ - - check_series(series) - return cast( - str | None, - _series_info_with_fallbacks(series, (_specs.UNIT,), include_inherited), - ) + return _series_value(series, "unit", include_inherited) # type: ignore[no-any-return] def get_series_name(series: pd.Series, *, include_inherited: bool = False) -> str | None: - """Gets name of the series from metadata - - Args: - series (pd.Series): Series with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. - - Returns: - str | None: - Returns the name of the value. - If the name of the metric is not present it returns the name of the value. - If the value name is not present it returns None. + """Name of the values of a SERIES, falling back to the name of its metric - Raises: - TypeError: If `series` is not a Series. - - Let's test what happens if series has name in value_dimensions: - - .. doctest:: metadata.get_series_name - - >>> attr = { "by_metric": { "series": {"value_dimensions": {"value": {"name": "value_name_of_series"}}}}} - >>> series.attrs = attr - >>> get_series_name(series) - 'value_name_of_series' - - Let's test what happens if series has name in metric: - - .. doctest:: metadata.get_series_name - - >>> attr = { "by_metric": { "series": {"metric": {"name": "name_of_series_1"}}}} - >>> series.attrs = attr - >>> get_series_name(series) - 'name_of_series_1' - - Let's test what happens if series has name in metric and value_dimensions: + Code example: .. doctest:: metadata.get_series_name - >>> attr = { "single_metric_metadata": { "structured_metadata": {"metric": {"name": "name_of_series_2"}}, - ... "value_dimensions": {"value": {"name": "value_name_of_series"}}}} - >>> series.attrs = attr + >>> from hdhelpers.metadata.conventions import SERIES_METADATA_EXAMPLE + >>> series.attrs = SERIES_METADATA_EXAMPLE >>> get_series_name(series) - 'name_of_series_2' + 'Water level north' """ - check_series(series) - return cast( - str | None, - _series_info_with_fallbacks(series, ("name",), include_inherited), - ) + return _series_value(series, "name", include_inherited) # type: ignore[no-any-return] def get_series_display_name(series: pd.Series, *, include_inherited: bool = False) -> str | None: - """Gets display name of the series from metadata - - Args: - series (pd.Series): Series with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. - - Returns: - str | None: - Returns the display name of the value. - If the display name of the metric is not present it returns the display name of the value. - If the metric display name is not present it returns the result of :func:`hdhelpers.metadata.get_series_name`. - - Raises: - TypeError: If `series` is not a Series. - - Code example: - - .. doctest:: metadata.get_series_display_name - - >>> attr = { "by_metric": { "series": {"metric": {"display_name": "display_name_of_series"}}}} - >>> series.attrs = attr - >>> get_series_display_name(series) - 'display_name_of_series' - """ - check_series(series) - return cast( - str | None, - _series_info_with_fallbacks(series, _specs.DISPLAY_NAME_FIELDS, include_inherited), + """Display name of the values of a SERIES, falling back to the name""" + return _series_value( # type: ignore[no-any-return] + series, DISPLAY_NAME_FIELDS, include_inherited ) def get_series_short_display_name( series: pd.Series, *, include_inherited: bool = False ) -> str | None: - """Gets short display name of the Series from metadata - - Args: - series (pd.Series): Series with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. - - Returns: - str | None: - Returns the short display name of the value. - If the short display name of the metric is not present it returns the short display name of the value. - If the metric short display name is not present it returns the result of :func:`hdhelpers.metadata.get_series_display_name`. - - Raises: - TypeError: If `series` is not a Series. - - Code example: - - .. doctest:: metadata.get_series_short_display_name - - >>> attr = { "by_metric": { "series": { "metric": {"short_display_name": "short_display_name_of_series"}}}} - >>> series.attrs = attr - >>> get_series_short_display_name(series) - 'short_display_name_of_series' - """ - check_series(series) - return cast( - str | None, - _series_info_with_fallbacks(series, _specs.SHORT_DISPLAY_NAME_FIELDS, include_inherited), + """Short display name of the values of a SERIES, falling back to display name and name""" + return _series_value( # type: ignore[no-any-return] + series, SHORT_DISPLAY_NAME_FIELDS, include_inherited ) def get_series_measurement(series: pd.Series, *, include_inherited: bool = False) -> str | None: - """Gets measurement (type) of the Series from metadata + """Measured quantity of the values of a SERIES""" + return _series_value(series, "measurement", include_inherited) # type: ignore[no-any-return] - Args: - series (pd.Series): Series with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. - Returns: - str | None: - Returns the measurement (type) of the value. - If "measurement" is not given in the metadata, None is returned. - - Raises: - TypeError: If `series` is not a Series. - - Code example: - - .. doctest:: metadata.get_series_measurement - - >>> attr = { "by_metric": { "series": { "metric": {"measurement": "temperature"}}}} - >>> series.attrs = attr - >>> get_series_measurement(series) - 'temperature' - """ - check_series(series) - return cast( - str | None, get_series_info(series, "measurement", include_inherited=include_inherited) - ) +# value dimensions of SINGLETSFRAMEs def get_singlets_info( - singletsframe: pd.DataFrame, value_dim_info: str | Spec, *, include_inherited: bool = True -) -> Any: - """Gets an arbitrary info of the value dimensions of a SingleTSFrame - - A SingleTSFrame (STSF) holds exactly one metric - like a Series - but arbitrarily many - value dimensions - like a MultiTSFrame. Therefore, in contrast to the MTSF functions, - the result is keyed by value dimension only and not by metric first. - - The single metric is identified via "single_metric" in "dataset_metadata". If that is - missing but the metadata contains exactly one metric, that metric is used, since a STSF - cannot be ambiguous in this respect. - - Args: - singletsframe (pd.DataFrame): STSF with metadata following the convention. - value_dim_info (str | Spec): Information to retrieve, interpreted as a glom Spec: e.g. a field name, a dotted path like "sensor.manufacturer" into sub dictionaries or any other glom Spec. See "How to query nested information?" in the first steps. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to True. - - Returns: - defaultdict[str, Any]: Dictionary of value dimensions containing the retrieved information. If the information is not present for a value dimension the corresponding value is None. - - Raises: - TypeError: If `singletsframe` is not a DataFrame. + singletsframe: pd.DataFrame, value_dim_info: str, *, include_inherited: bool = True +) -> defaultdict[str, Any]: + """A field of the value dimensions of the single metric of a SINGLETSFRAME by column Code example: .. doctest:: metadata.get_singlets_info - >>> dataframe.attrs = { - ... "dataset_metadata": {"single_metric": "abc.temp", "metric_key": "external_id"}, - ... "metrics": [ - ... { - ... "external_id": "abc.temp", - ... "value_dimensions": [ - ... {"column": "value", "unit": "°C"}, - ... {"column": "state"}, - ... ], - ... } - ... ], - ... } - >>> result = get_singlets_info(dataframe, "unit") - >>> result["value"] - '°C' - >>> result["state"] is None - True + >>> from hdhelpers.metadata.conventions import SINGLETSFRAME_METADATA_EXAMPLE + >>> dataframe.attrs = SINGLETSFRAME_METADATA_EXAMPLE + >>> get_singlets_info(dataframe, "name")["state"] + 'measurement state' """ - - check_dataframe(singletsframe) - return singlets_value_dimension_info(singletsframe, value_dim_info, include_inherited) + return _single_metric_by_column(singletsframe, value_dim_info, include_inherited) def get_singlets_units( singletsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, str | None]: - """Gets units of the value dimensions of a SingleTSFrame from metadata + """Units of the value dimensions of a SINGLETSFRAME by column - Args: - singletsframe (pd.DataFrame): STSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. - - Returns: - defaultdict[str, str | None]: Dictionary of value dimensions containing the units. If the unit is not present for a value dimension the corresponding value is None. - - Raises: - TypeError: If `singletsframe` is not a DataFrame. - - Note that a value dimension falls back to "value_dimensions_shared" if it carries no unit - of its own: - - .. doctest:: metadata.get_singlets_units - - >>> dataframe.attrs = { - ... "dataset_metadata": {"single_metric": "abc.temp", "metric_key": "external_id"}, - ... "metrics": [ - ... { - ... "external_id": "abc.temp", - ... "value_dimensions": [ - ... {"column": "value", "unit": "°C"}, - ... {"column": "state"}, - ... ], - ... } - ... ], - ... "value_dimensions_shared": [{"column": "state", "unit": "UNKNOWN"}], - ... } - >>> result = get_singlets_units(dataframe) - >>> result["value"] - '°C' - >>> result["state"] - 'UNKNOWN' - >>> result["not-given"] is None - True - - The "metric_key" is optional and defaults to "id": + Code example: .. doctest:: metadata.get_singlets_units - >>> dataframe.attrs = { - ... "dataset_metadata": {"single_metric": "abc.temp"}, - ... "metrics": [ - ... {"id": "abc.temp", "value_dimensions": [{"column": "value", "unit": "m"}]} - ... ], - ... } + >>> from hdhelpers.metadata.conventions import SINGLETSFRAME_METADATA_EXAMPLE + >>> dataframe.attrs = SINGLETSFRAME_METADATA_EXAMPLE >>> get_singlets_units(dataframe)["value"] - 'm' + '°C' """ - - check_dataframe(singletsframe) - return singlets_value_dimension_info(singletsframe, _specs.UNIT, include_inherited) + return _single_metric_by_column(singletsframe, "unit", include_inherited) def get_singlets_names( singletsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, str | None]: - """Gets names of the value dimensions of a SingleTSFrame from metadata - - Args: - singletsframe (pd.DataFrame): STSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. - - Returns: - defaultdict[str, str | None]: Dictionary of value dimensions containing the names. If the name is not present for a value dimension the corresponding value is None. - - Raises: - TypeError: If `singletsframe` is not a DataFrame. - - Code example: - - .. doctest:: metadata.get_singlets_names - - >>> dataframe.attrs = { - ... "dataset_metadata": {"single_metric": "abc.temp", "metric_key": "external_id"}, - ... "metrics": [ - ... { - ... "external_id": "abc.temp", - ... "value_dimensions": [ - ... {"column": "value", "name": "temperature"}, - ... {"column": "state"}, - ... ], - ... } - ... ], - ... } - >>> result = get_singlets_names(dataframe) - >>> result["value"] - 'temperature' - >>> result["state"] is None - True - """ - - check_dataframe(singletsframe) - return singlets_value_dimension_info(singletsframe, _specs.NAME, include_inherited) + """Names of the value dimensions of a SINGLETSFRAME by column, see get_singlets_units""" + return _single_metric_by_column(singletsframe, "name", include_inherited) def get_singlets_display_names( singletsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, str | None]: - """Gets display names of the value dimensions of a SingleTSFrame from metadata - - Falls back to the name of the value dimension if no display name is given. The display name - is looked up on the value dimension, on the metric (for the value dimension "value") and in - "value_dimensions_shared" before falling back to the name. Fields provided as null count as - missing. - - Args: - singletsframe (pd.DataFrame): STSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. - - Returns: - defaultdict[str, str | None]: Dictionary of value dimensions containing the display names. If neither display name nor name is present for a value dimension the corresponding value is None. - - Raises: - TypeError: If `singletsframe` is not a DataFrame. - - Code example: - - .. doctest:: metadata.get_singlets_display_names - - >>> dataframe.attrs = { - ... "dataset_metadata": {"single_metric": "abc.temp", "metric_key": "external_id"}, - ... "metrics": [ - ... { - ... "external_id": "abc.temp", - ... "value_dimensions": [ - ... {"column": "value", "name": "temperature", "display_name": "temp"}, - ... {"column": "state", "name": "measurement state"}, - ... ], - ... } - ... ], - ... } - >>> result = get_singlets_display_names(dataframe) - >>> result["value"] - 'temp' - >>> result["state"] - 'measurement state' - """ - - check_dataframe(singletsframe) - return singlets_value_dimension_info_with_fallbacks( - singletsframe, _specs.DISPLAY_NAME_FIELDS, _specs.DISPLAY_NAME, include_inherited - ) + """Display names of the value dimensions of a SINGLETSFRAME, falling back to the names""" + return _single_metric_by_column(singletsframe, DISPLAY_NAME_FIELDS, include_inherited) def get_singlets_short_display_names( singletsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, str | None]: - """Gets short display names of the value dimensions of a SingleTSFrame from metadata - - Falls back to the display name and then to the name of the value dimension if no short - display name is given. Each of them is looked up on the value dimension, on the metric (for - the value dimension "value") and in "value_dimensions_shared" before falling back to the next - one. Fields provided as null count as missing. - - Args: - singletsframe (pd.DataFrame): STSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. - - Returns: - defaultdict[str, str | None]: Dictionary of value dimensions containing the short display names. If none of short display name, display name and name is present for a value dimension the corresponding value is None. - - Raises: - TypeError: If `singletsframe` is not a DataFrame. + """Short display names of the value dimensions of a SINGLETSFRAME, see get_singlets_units - Code example: - - .. doctest:: metadata.get_singlets_short_display_names - - >>> dataframe.attrs = { - ... "dataset_metadata": {"single_metric": "abc.temp", "metric_key": "external_id"}, - ... "metrics": [ - ... { - ... "external_id": "abc.temp", - ... "value_dimensions": [ - ... {"column": "value", "name": "temperature", "short_display_name": "T"}, - ... {"column": "state", "display_name": "state"}, - ... ], - ... } - ... ], - ... } - >>> result = get_singlets_short_display_names(dataframe) - >>> result["value"] - 'T' - >>> result["state"] - 'state' + Fall back to the display names and the names. """ - - check_dataframe(singletsframe) - return singlets_value_dimension_info_with_fallbacks( - singletsframe, - _specs.SHORT_DISPLAY_NAME_FIELDS, - _specs.SHORT_DISPLAY_NAME, - include_inherited, - ) + return _single_metric_by_column(singletsframe, SHORT_DISPLAY_NAME_FIELDS, include_inherited) def get_singlets_measurements( singletsframe: pd.DataFrame, *, include_inherited: bool = False ) -> defaultdict[str, str | None]: - """Gets measurements (types) of the value dimensions of a SingleTSFrame from metadata - - Args: - singletsframe (pd.DataFrame): STSF with metadata following the convention. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the value dimension, the metric nor "value_dimensions_shared" provide the information. Defaults to False, since this information should be provided for the metric itself. - - Returns: - defaultdict[str, str | None]: Dictionary of value dimensions containing the measurements. If "measurement" is not given for a value dimension the corresponding value is None. - - Raises: - TypeError: If `singletsframe` is not a DataFrame. - - Code example: - - .. doctest:: metadata.get_singlets_measurements - - >>> dataframe.attrs = { - ... "dataset_metadata": {"single_metric": "abc.temp", "metric_key": "external_id"}, - ... "metrics": [ - ... { - ... "external_id": "abc.temp", - ... "value_dimensions": [ - ... {"column": "value", "measurement": "temperature"}, - ... ], - ... } - ... ], - ... } - >>> get_singlets_measurements(dataframe)["value"] - 'temperature' - """ - - check_dataframe(singletsframe) - return singlets_value_dimension_info(singletsframe, _specs.MEASUREMENT, include_inherited) + """Measured quantities of the value dimensions of a SINGLETSFRAME by column""" + return _single_metric_by_column(singletsframe, "measurement", include_inherited) def get_singlets_metric_info( - singletsframe: pd.DataFrame, metric_info: str | Spec, *, include_inherited: bool = True + singletsframe: pd.DataFrame, metric_info: str, *, include_inherited: bool = True ) -> Any: - """Gets metadata associated to the single metric of a SingleTSFrame - - Counterpart of :func:`get_metric_info` for SingleTSFrames: instead of a dictionary keyed - by metric this directly returns the requested information for the one metric. Like - :func:`get_metric_info`, it falls back to the value dimension "value", to - "value_dimensions_shared" and finally to the properties the metric inherits. - - Args: - singletsframe (pd.DataFrame): STSF with metadata following the convention. - metric_info (str | Spec): Information to retrieve, interpreted as a glom Spec: e.g. a field name, a dotted path like "sensor.manufacturer" into sub dictionaries or any other glom Spec. See "How to query nested information?" in the first steps. - include_inherited (bool): Whether to fall back to the properties a metric inherits from objects higher up in an asset hierarchy (given under "inherited" in the metric metadata) as last resort, i.e. if neither the metric, its value dimension "value" nor "value_dimensions_shared" provide the information. Defaults to True. - - Returns: - Any: Retrieved information defined by `metric_info`, or None if it cannot be determined - which includes the case of absent metric metadata. - - Raises: - TypeError: If `singletsframe` is not a DataFrame. - - Code example: - - .. doctest:: metadata.get_singlets_metric_info - - >>> dataframe.attrs = { - ... "dataset_metadata": {"single_metric": "abc.temp", "metric_key": "external_id"}, - ... "metrics": [{"external_id": "abc.temp", "name": "ABC temperature"}], - ... } - >>> get_singlets_metric_info(dataframe, "name") - 'ABC temperature' - >>> get_singlets_metric_info(dataframe, "not-given") is None - True - - Without any metadata None is returned: + """A field of the single metric of a SINGLETSFRAME, see get_metric_info for the fallbacks""" + check_dataframe(singletsframe) + return _single_metric_or_dataset( + singletsframe, + "SINGLETSFRAME", + lambda meta, metric: meta.metric_info( + metric_info, metric, include_inherited=include_inherited + ), + lambda meta: None, + empty=None, + ) - .. doctest:: metadata.get_singlets_metric_info - >>> dataframe.attrs = {} - >>> get_singlets_metric_info(dataframe, "name") is None - True - """ - - check_dataframe(singletsframe) - return singlets_metric_info(singletsframe, metric_info, include_inherited) +# queried interval, time zones, recording and aggregation def get_queried_interval( data: pd.Series | pd.DataFrame, ) -> tuple[datetime.datetime | None, datetime.datetime | None]: - """Get queried interval from metadata - - Args: - data (pd.Series | pd.DataFrame): Series or Dataframe with metadata following the convention - - Returns: - tuple[datetime.datetime | None, datetime.datetime | None]: Tuple of available start and end date of requested interval. - - Raises: - ValueError: If metadata of `data` is not None and not convertible to a datetime-object (ISO-format is expected). - TypeError: If `data` is not a Series or Dataframe. + """Start and end of the queried interval (the ref interval), None if unknown Code example: .. doctest:: metadata.get_queried_interval - >>> attr = { - ... "dataset_metadata": { - ... "ref_interval_start_timestamp": "2025-11-05T13:28:00Z", - ... "ref_interval_end_timestamp": "2025-11-06T13:28:00Z" - ... } - ... } - >>> series.attrs = attr + >>> from hdhelpers.metadata.conventions import SERIES_METADATA_EXAMPLE + >>> series.attrs = SERIES_METADATA_EXAMPLE >>> start, end = get_queried_interval(series) >>> start.isoformat() - '2025-11-05T13:28:00+00:00' - >>> end.isoformat() - '2025-11-06T13:28:00+00:00' + '2026-01-01T00:00:00+00:00' """ check_series_or_dataframe(data) - start = extract_from_metadata(data.attrs, key="ref_interval_start_timestamp", default=None) - end = extract_from_metadata(data.attrs, key="ref_interval_end_timestamp", default=None) - - formatted_start = datetime.datetime.fromisoformat(start) if start else None - formatted_end = datetime.datetime.fromisoformat(end) if end else None - - return formatted_start, formatted_end + meta = _metadata(data, infer_data_type(data)) + return (None, None) if meta is None else meta.queried_interval() def get_default_timezones(multitsframe: pd.DataFrame) -> defaultdict[str, str | None]: - """Gets the default time zone of each metric of a MultiTSFrame - - The default time zone is meant for computations: It is the time zone the timeseries should - be evaluated in by default, e.g. for bucketing days when aggregating, so that each day - starts at midnight of this time zone. It belongs to the data, e.g. it is the time zone of - the plant a sensor belongs to. It is not meant for displaying timestamps - that is what the - time zone of the plot target settings is for, which e.g. :func:`hdhelpers.helpers.modify_timezone` - uses by default. See "Which time zone to compute in?" in the first steps. - - The time zone is an IANA time zone name like "Europe/Berlin" or a fixed UTC offset like - "+02:00", both accepted by pandas, e.g. by ``tz_convert``. - - The default time zone of a metric falls back to the one of the dataset metadata, which - applies to all metrics without their own. It does not fall back to value dimensions or - inherited properties. + """The default time zone of each metric of a MULTITSFRAME - Args: - multitsframe (pd.DataFrame): MTSF with metadata following the convention. - - Returns: - defaultdict[str, str | None]: Dictionary, where keys are the entries of the metrics metadata specified via "metric_key" in "dataset_metadata" and values are their default time zones. Defaults to the default time zone of the dataset metadata, also for metrics without metadata, and to None if that is missing as well. - - Raises: - TypeError: If `multitsframe` is not a DataFrame. + The one of the metric, otherwise the one of the dataset metadata, which is also the default + of the returned defaultdict. Code example: .. doctest:: metadata.get_default_timezones - >>> multitsframe.attrs = { - ... "dataset_metadata": {"metric_key": "id", "default_timezone": "Europe/Berlin"}, - ... "metrics": [ - ... {"id": "new_york_sensor", "default_timezone": "America/New_York"}, - ... {"id": "berlin_sensor"}, - ... ], - ... } - >>> timezones = get_default_timezones(multitsframe) - >>> timezones["new_york_sensor"] - 'America/New_York' - >>> timezones["berlin_sensor"] - 'Europe/Berlin' - >>> timezones["metric_without_metadata"] + >>> from hdhelpers.metadata.conventions import MULTITSFRAME_METADATA_EXAMPLE + >>> multitsframe.attrs = MULTITSFRAME_METADATA_EXAMPLE + >>> get_default_timezones(multitsframe)["plant_a.pressure"] 'Europe/Berlin' """ check_dataframe(multitsframe) - return default_timezone_by_metric(multitsframe) + meta = _metadata(multitsframe, "MULTITSFRAME") + if meta is None: + return defaultdict(lambda: None) + dataset_timezone = dataset_default_timezone(meta) + return defaultdict(lambda: dataset_timezone, meta.default_timezones()) def get_series_default_timezone(series: pd.Series) -> str | None: - """Gets the default time zone of a Series - - The default time zone is meant for computations: It is the time zone the timeseries should - be evaluated in by default, e.g. for bucketing days when aggregating, so that each day - starts at midnight of this time zone. It belongs to the data, e.g. it is the time zone of - the plant a sensor belongs to. It is not meant for displaying timestamps - that is what the - time zone of the plot target settings is for, which e.g. :func:`hdhelpers.helpers.modify_timezone` - uses by default. See "Which time zone to compute in?" in the first steps. - - The time zone is an IANA time zone name like "Europe/Berlin" or a fixed UTC offset like - "+02:00", both accepted by pandas, e.g. by ``tz_convert``. - - The default time zone of the metric falls back to the one of the dataset metadata. - - Args: - series (pd.Series): Series with metadata following the convention. - - Returns: - str | None: The default time zone, or None if neither the metric nor the dataset metadata provide one. - - Raises: - TypeError: If `series` is not a Series. - - Code example: - - .. doctest:: metadata.get_series_default_timezone - - >>> series.attrs = { - ... "dataset_metadata": {"single_metric": "water.north.level", "metric_key": "external_id"}, - ... "metrics": [{"external_id": "water.north.level", "default_timezone": "Europe/Berlin"}], - ... } - >>> get_series_default_timezone(series) - 'Europe/Berlin' - >>> series.attrs = {} - >>> get_series_default_timezone(series) is None - True - """ + """The default time zone of the single metric of a SERIES, see get_default_timezones""" check_series(series) - return single_metric_default_timezone(series) + return _single_metric_or_dataset( # type: ignore[no-any-return] + series, + "SERIES", + lambda meta, metric: meta.default_timezone(metric), + dataset_default_timezone, + empty=None, + ) def get_singlets_default_timezone(singletsframe: pd.DataFrame) -> str | None: - """Gets the default time zone of the single metric of a SingleTSFrame - - The default time zone is meant for computations: It is the time zone the timeseries should - be evaluated in by default, e.g. for bucketing days when aggregating, so that each day - starts at midnight of this time zone. It belongs to the data, e.g. it is the time zone of - the plant a sensor belongs to. It is not meant for displaying timestamps - that is what the - time zone of the plot target settings is for, which e.g. :func:`hdhelpers.helpers.modify_timezone` - uses by default. See "Which time zone to compute in?" in the first steps. - - The time zone is an IANA time zone name like "Europe/Berlin" or a fixed UTC offset like - "+02:00", both accepted by pandas, e.g. by ``tz_convert``. - - The default time zone of the metric falls back to the one of the dataset metadata. - - Args: - singletsframe (pd.DataFrame): STSF with metadata following the convention. - - Returns: - str | None: The default time zone, or None if neither the metric nor the dataset metadata provide one. - - Raises: - TypeError: If `singletsframe` is not a DataFrame. - - Code example: - - .. doctest:: metadata.get_singlets_default_timezone - - >>> dataframe.attrs = { - ... "dataset_metadata": {"single_metric": "abc.temp", "default_timezone": "+02:00"}, - ... "metrics": [{"id": "abc.temp", "default_timezone": None}], - ... } - >>> get_singlets_default_timezone(dataframe) - '+02:00' - """ + """The default time zone of the single metric of a SINGLETSFRAME, see get_default_timezones""" check_dataframe(singletsframe) - return single_metric_default_timezone(singletsframe) + return _single_metric_or_dataset( # type: ignore[no-any-return] + singletsframe, + "SINGLETSFRAME", + lambda meta, metric: meta.default_timezone(metric), + dataset_default_timezone, + empty=None, + ) def get_recordings(multitsframe: pd.DataFrame) -> defaultdict[str, Recording]: - """Gets how the values of each metric of a MultiTSFrame are recorded + """How the values of each metric of a MULTITSFRAME are recorded - The recording consists of the "recording_mode" of the metric (e.g. "periodic" or - "on_change"), the frequency of its values with offset and time zone and its - "recording_aggregation_method", see :class:`Recording` for what they mean. - - The frequency falls back as a whole: If the metric has a "data_frequency", it applies - together with the metric's "data_frequency_offset" and "data_frequency_timezone". Otherwise - "ref_data_frequency" of the dataset metadata applies together with - "ref_data_frequency_offset" and "ref_data_frequency_timezone". The values are provided as - given in the metadata. - - Args: - multitsframe (pd.DataFrame): MTSF with metadata following the convention. - - Returns: - defaultdict[str, Recording]: Dictionary, where keys are the entries of the metrics metadata specified via "metric_key" in "dataset_metadata" and values are their recordings. Defaults to a recording with the frequency of the dataset metadata, also for metrics without metadata. - - Raises: - TypeError: If `multitsframe` is not a DataFrame. + The frequency, its offset and time zone fall back to the ones of the dataset metadata as a + whole. The default of the returned defaultdict is the recording of a metric without + metadata, i.e. with the frequency of the dataset metadata. Code example: .. doctest:: metadata.get_recordings - >>> multitsframe.attrs = { - ... "dataset_metadata": {"metric_key": "id", "ref_data_frequency": "1h"}, - ... "metrics": [ - ... { - ... "id": "average_level", - ... "recording_mode": "periodic", - ... "data_frequency": "15min", - ... "recording_aggregation_method": "MEAN", - ... }, - ... {"id": "pump_state", "recording_mode": "on_change"}, - ... ], - ... } - >>> recordings = get_recordings(multitsframe) - >>> recordings["average_level"].data_frequency - '15min' - >>> recordings["pump_state"].recording_mode + >>> from hdhelpers.metadata.conventions import MULTITSFRAME_METADATA_EXAMPLE + >>> multitsframe.attrs = MULTITSFRAME_METADATA_EXAMPLE + >>> get_recordings(multitsframe)["plant_a.temperature"].recording_mode 'on_change' - >>> recordings["pump_state"].data_frequency - '1h' """ check_dataframe(multitsframe) - return recording_by_metric(multitsframe) + meta = _metadata(multitsframe, "MULTITSFRAME") + if meta is None: + return defaultdict(Recording) + no_metadata = dataset_recording(meta) + return defaultdict(lambda: no_metadata, meta.recordings()) def get_series_recording(series: pd.Series) -> Recording: - """Gets how the values of a Series are recorded - - See :func:`get_recordings` for the recording and how its frequency falls back to the one of - the dataset metadata. - - Args: - series (pd.Series): Series with metadata following the convention. - - Returns: - Recording: The recording of the metric, with the frequency of the dataset metadata if the metric does not provide one. - - Raises: - TypeError: If `series` is not a Series. - - Code example: - - .. doctest:: metadata.get_series_recording - - >>> series.attrs = { - ... "dataset_metadata": {"single_metric": "water.north.level", "metric_key": "external_id"}, - ... "metrics": [ - ... { - ... "external_id": "water.north.level", - ... "recording_mode": "periodic", - ... "data_frequency": "2h", - ... "data_frequency_timezone": "Europe/Berlin", - ... } - ... ], - ... } - >>> recording = get_series_recording(series) - >>> recording.data_frequency, recording.data_frequency_offset, recording.data_frequency_timezone - ('2h', None, 'Europe/Berlin') - """ + """How the values of a SERIES are recorded, see get_recordings""" check_series(series) - return single_metric_recording(series) + return _single_metric_or_dataset( # type: ignore[no-any-return] + series, + "SERIES", + lambda meta, metric: meta.recording(metric), + dataset_recording, + empty=Recording(), + ) def get_singlets_recording(singletsframe: pd.DataFrame) -> Recording: - """Gets how the values of the single metric of a SingleTSFrame are recorded - - See :func:`get_recordings` for the recording and how its frequency falls back to the one of - the dataset metadata. - - Args: - singletsframe (pd.DataFrame): STSF with metadata following the convention. - - Returns: - Recording: The recording of the metric, with the frequency of the dataset metadata if the metric does not provide one. - - Raises: - TypeError: If `singletsframe` is not a DataFrame. - - Code example: - - .. doctest:: metadata.get_singlets_recording - - >>> dataframe.attrs = { - ... "dataset_metadata": { - ... "single_metric": "abc.temp", - ... "ref_data_frequency": "5min", - ... "ref_data_frequency_offset": "1min", - ... }, - ... "metrics": [{"id": "abc.temp", "recording_mode": "periodic"}], - ... } - >>> recording = get_singlets_recording(dataframe) - >>> recording.recording_mode, recording.data_frequency, recording.data_frequency_offset - ('periodic', '5min', '1min') - """ + """How the values of a SINGLETSFRAME are recorded, see get_recordings""" check_dataframe(singletsframe) - return single_metric_recording(singletsframe) + return _single_metric_or_dataset( # type: ignore[no-any-return] + singletsframe, + "SINGLETSFRAME", + lambda meta, metric: meta.recording(metric), + dataset_recording, + empty=Recording(), + ) def get_default_aggregation_methods(multitsframe: pd.DataFrame) -> defaultdict[str, str]: - """Gets the method each metric of a MultiTSFrame should be aggregated with by default + """The default aggregation method of each metric of a MULTITSFRAME - E.g. for plotting or reporting it in coarser intervals. It is the "default_aggregation_method" - of the metric, with "AVG" normalized to its alias "MEAN". If the metric does not provide one, - it is "LAST" for values of the data type "boolean" or "string" (the "value_data_type" of the - value dimension "value"), i.e. the state holding at the end of the interval, and "MEAN" - otherwise. It is not taken from inherited properties. - - Args: - multitsframe (pd.DataFrame): MTSF with metadata following the convention. - - Returns: - defaultdict[str, str]: Dictionary, where keys are the entries of the metrics metadata specified via "metric_key" in "dataset_metadata" and values are their default aggregation methods, e.g. "MEAN", "SUM" or "LAST". Defaults to "MEAN" for metrics without metadata. - - Raises: - TypeError: If `multitsframe` is not a DataFrame. + The method of the metric, with "AVG" as "MEAN". Otherwise "LAST" for boolean and string + values and "MEAN" for all others, which is also the default of the returned defaultdict. Code example: .. doctest:: metadata.get_default_aggregation_methods - >>> multitsframe.attrs = { - ... "dataset_metadata": {"metric_key": "id"}, - ... "metrics": [ - ... {"id": "level", "default_aggregation_method": "AVG"}, - ... {"id": "rainfall", "default_aggregation_method": "SUM"}, - ... { - ... "id": "pump_running", - ... "value_dimensions": [{"column": "value", "value_data_type": "boolean"}], - ... }, - ... ], - ... } + >>> from hdhelpers.metadata.conventions import MULTITSFRAME_METADATA_EXAMPLE + >>> multitsframe.attrs = MULTITSFRAME_METADATA_EXAMPLE >>> methods = get_default_aggregation_methods(multitsframe) - >>> methods["level"], methods["rainfall"], methods["pump_running"] - ('MEAN', 'SUM', 'LAST') - >>> methods["metric_without_metadata"] - 'MEAN' + >>> methods["plant_a.temperature"], methods["plant_a.pressure"] + ('MAX', 'MEAN') """ check_dataframe(multitsframe) - return default_aggregation_method_by_metric(multitsframe) + meta = _metadata(multitsframe, "MULTITSFRAME") + methods = {} if meta is None else meta.default_aggregation_methods() + return defaultdict(lambda: _DEFAULT_AGGREGATION_METHOD, methods) def get_series_default_aggregation_method(series: pd.Series) -> str: - """Gets the method a Series should be aggregated with by default - - See :func:`get_default_aggregation_methods` for the method and its default. - - Args: - series (pd.Series): Series with metadata following the convention. - - Returns: - str: The default aggregation method, e.g. "MEAN", "SUM" or "LAST". - - Raises: - TypeError: If `series` is not a Series. - - Code example: - - .. doctest:: metadata.get_series_default_aggregation_method - - >>> series.attrs = { - ... "dataset_metadata": {"single_metric": "rainfall"}, - ... "metrics": [{"id": "rainfall", "default_aggregation_method": "SUM"}], - ... } - >>> get_series_default_aggregation_method(series) - 'SUM' - >>> series.attrs = {} - >>> get_series_default_aggregation_method(series) - 'MEAN' - """ + """The default aggregation method of a SERIES, see get_default_aggregation_methods""" check_series(series) - return single_metric_default_aggregation_method(series) + return _single_metric_or_dataset( # type: ignore[no-any-return] + series, + "SERIES", + lambda meta, metric: meta.default_aggregation_method(metric), + lambda meta: _DEFAULT_AGGREGATION_METHOD, + empty=_DEFAULT_AGGREGATION_METHOD, + ) def get_singlets_default_aggregation_method(singletsframe: pd.DataFrame) -> str: - """Gets the method the single metric of a SingleTSFrame should be aggregated with by default - - See :func:`get_default_aggregation_methods` for the method and its default. - - Args: - singletsframe (pd.DataFrame): STSF with metadata following the convention. - - Returns: - str: The default aggregation method, e.g. "MEAN", "SUM" or "LAST". - - Raises: - TypeError: If `singletsframe` is not a DataFrame. - - Code example: - - .. doctest:: metadata.get_singlets_default_aggregation_method - - >>> dataframe.attrs = { - ... "dataset_metadata": {"single_metric": "valve"}, - ... "metrics": [ - ... { - ... "id": "valve", - ... "value_dimensions": [{"column": "value", "value_data_type": "string"}], - ... } - ... ], - ... } - >>> get_singlets_default_aggregation_method(dataframe) - 'LAST' - """ + """The default aggregation method of a SINGLETSFRAME, see get_default_aggregation_methods""" check_dataframe(singletsframe) - return single_metric_default_aggregation_method(singletsframe) + return _single_metric_or_dataset( # type: ignore[no-any-return] + singletsframe, + "SINGLETSFRAME", + lambda meta, metric: meta.default_aggregation_method(metric), + lambda meta: _DEFAULT_AGGREGATION_METHOD, + empty=_DEFAULT_AGGREGATION_METHOD, + ) def without_metadata[T: (pd.Series, pd.DataFrame)](data: T) -> T: - """Gets a copy of a Series or DataFrame without its metadata, to compute on - - pandas deep-copies the metadata (``attrs``) of a Series or DataFrame to every object an - operation creates from it, e.g. when filtering rows, selecting a column or grouping. For - metadata of many metrics, e.g. with comments, loops over the metrics then take much longer - than the computation itself. Compute on the copy without metadata instead and attach the - metadata of the result once at the end. + """Gets a copy of a Series or DataFrame without its metadata The copy is cheap: It shares the data with ``data`` until one of them is changed (copy on - write), so changing the copy does not change ``data``. + write), so changing the copy does not change ``data``. pandas operations do not copy + metadata attached as model (see attach_metadata), but they copy plain metadata in attrs + deeply for every object they create. Args: data (pd.Series | pd.DataFrame): Series or DataFrame, e.g. a MTSF with metadata. diff --git a/tests/benchmarks/benchmark_metadata.py b/tests/benchmarks/benchmark_metadata.py new file mode 100644 index 0000000..fac6f38 --- /dev/null +++ b/tests/benchmarks/benchmark_metadata.py @@ -0,0 +1,104 @@ +"""Timings of the metadata models for MULTITSFRAMEs with many metrics, against the budget + +Not part of the test run. Run with: uv run python tests/benchmarks/benchmark_metadata.py [metrics] +""" + +import copy +import sys +import timeit +from collections.abc import Callable +from typing import Any + +import numpy as np +import pandas as pd + +from hdhelpers.metadata import META_KEY, attach_metadata, attrs_to_json, get_units +from hdhelpers.metadata.conventions import MultiTSFrameMetadata + +# milliseconds per 1000 metrics, for metadata with nested extra fields, recording information and +# comments as below, unless independent of the number of metrics (FIXED) +BUDGET = { + "parse (model_validate incl. rules)": 12.0, + "deepcopy(attrs) with attached model": 0.01, + "filter -> sort -> reset_index": 1.0, + "with_metric": 0.5, + "attrs_to_json": 6.0, + "get_units": 5.0, +} +# pandas shares the attached model instead of copying it +FIXED = {"deepcopy(attrs) with attached model", "filter -> sort -> reset_index"} + + +def attrs_of(metric_count: int) -> dict[str, Any]: + metrics = [] + for index in range(metric_count): + metric: dict[str, Any] = { + "id": f"00000000-0000-4000-8000-{index:012d}", + "external_id": f"plant.m{index}", + "name": f"Metric {index}", + "path": ["site", "area", f"unit{index % 10}"], + "value_dimensions": [{"column": "value", "unit": "bar", "measurement": "pressure"}], + "inherited": {"site": "Site A", "asset": {"type": "pump", "tags": ["a", "b"]}}, + "recording_mode": "periodic", + "data_frequency": "1h", + "default_timezone": "Europe/Berlin", + } + if index % 10 == 0: + metric["comments"] = [ + {"content": "checked", "username": "x", "created_at": "2026-01-01T00:00:00Z"} + ] + metrics.append(metric) + return { + "dataset_metadata": { + "metric_key": "external_id", + "queried_metrics": [f"plant.m{index}" for index in range(metric_count)], + "ref_interval_start_timestamp": "2026-01-01T00:00:00+00:00", + "ref_interval_end_timestamp": "2026-01-02T00:00:00+00:00", + "ref_interval_type": "closed", + }, + "metrics": metrics, + } + + +def milliseconds(function: Callable[[], Any], number: int = 20) -> float: + return min(timeit.repeat(function, number=number, repeat=5)) / number * 1e3 + + +def main(metric_count: int) -> None: + attrs = attrs_of(metric_count) + frame = pd.DataFrame( + { + "timestamp": pd.date_range("2026-01-01", periods=100, freq="h", tz="UTC"), + "metric": "plant.m1", + "value": np.arange(100.0), + } + ) + frame.attrs = copy.deepcopy(attrs) + meta = attach_metadata(frame) + assert isinstance(frame.attrs[META_KEY], MultiTSFrameMetadata) + assert meta is not None + + timings = { + "parse (model_validate incl. rules)": milliseconds( + lambda: MultiTSFrameMetadata.model_validate(attrs), 5 + ), + "deepcopy(attrs) with attached model": milliseconds( + lambda: copy.deepcopy(frame.attrs), 1000 + ), + "filter -> sort -> reset_index": milliseconds( + lambda: frame[frame["value"] > 10].sort_values("value").reset_index(drop=True) + ), + "with_metric": milliseconds(lambda: meta.with_metric("plant.m1", display_name="M1")), + "attrs_to_json": milliseconds(lambda: attrs_to_json(frame.attrs), 5), + "get_units": milliseconds(lambda: get_units(frame), 5), + } + scale = metric_count / 1000 + print(f"{metric_count} metrics, pandas {pd.__version__}") + for name, value in timings.items(): + budget = BUDGET[name] * (1 if name in FIXED else scale) + status = "ok" if value <= budget else "OVER BUDGET" + print(f" {name:40s} {value:9.3f} ms (budget {budget:7.3f} ms) {status}") + + +if __name__ == "__main__": + main(int(sys.argv[1]) if len(sys.argv) > 1 else 1000) diff --git a/tests/conftest.py b/tests/conftest.py index 3d2aac5..4aa3efc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,12 +4,3 @@ import pytest from .data.fixtures.helpers import dataframe, multicolumn_frame, series_summer, series_winter -from .data.fixtures.metadata import ( - empty_mts_with_attr, - empty_mts_with_old_attr, - empty_mts_with_old_attr_real, - empty_series_with_attr, - empty_series_with_old_attr, - empty_series_with_old_attr_real, - empty_singletsframe_with_attr, -) diff --git a/tests/data/convention_schemas.json b/tests/data/convention_schemas.json new file mode 100644 index 0000000..50b9400 --- /dev/null +++ b/tests/data/convention_schemas.json @@ -0,0 +1,1727 @@ +{ + "$defs": { + "AggregationMethod": { + "description": "How a timeseries should be aggregated by default, e.g. in coarser intervals\n\nAVG is an alias of MEAN.", + "enum": [ + "MEAN", + "AVG", + "MEDIAN", + "SUM", + "MIN", + "MAX", + "FIRST", + "LAST", + "COUNT" + ], + "title": "AggregationMethod", + "type": "string" + }, + "InheritedMetricMetadata": { + "additionalProperties": true, + "description": "Properties a metric inherits from objects higher up in a hierarchy\n\nArbitrary fields, e.g. the latitude of the plant a sensor belongs to. Typically the deepest\noccurrence in the hierarchy wins. Should not contain the fields the conventions define for\nmetrics and value dimensions, like \"name\", \"id\" or \"unit\": Consumers only use inherited\nproperties as last resort and not for information describing the metric itself, like its\nunit or names.\n\nThe values should have the JSON type matching their meaning: numbers as numbers, booleans as\nbooleans and points in time as UTC isoformat strings, not e.g. a number as string.\n\nE.g. the hetida platform provides the effective property values of a signal here, keyed by\nproperty label: its own values, otherwise the value of the nearest asset above it having one.", + "properties": {}, + "title": "InheritedMetricMetadata", + "type": "object" + }, + "IntervalType": { + "description": "Which boundaries belong to a time interval\n\nclosed: both, open: none, left_closed / right_open: only the start,\nright_closed / left_open: only the end.", + "enum": [ + "closed", + "left_closed", + "right_closed", + "open", + "left_open", + "right_open" + ], + "title": "IntervalType", + "type": "string" + }, + "MetricComment": { + "additionalProperties": true, + "description": "A comment attached to a metric, describing a time range of its timeseries data\n\nE.g. the hetida platform provides the comments on a signal whose time range overlaps the\nqueried interval and which existed at query time, newest first.\n\nStart and end of the time range must be provided together. They are equal for a comment on a\nsingle point in time.", + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Free text of the comment", + "title": "Content" + }, + "created_at": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "When the comment was created", + "title": "Created At" + }, + "edited_at": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "When the comment was last edited. Null or missing if never edited.", + "title": "Edited At" + }, + "end_date": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End of the time range the comment refers to", + "title": "End Date" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Id of the comment, e.g. a UUID", + "title": "Id" + }, + "replies": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/MetricCommentReply" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Replies to the comment, oldest first", + "title": "Replies" + }, + "start_date": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start of the time range the comment refers to", + "title": "Start Date" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Free text tags classifying the comment", + "title": "Tags" + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Username of the author of the comment", + "title": "Username" + } + }, + "title": "MetricComment", + "type": "object" + }, + "MetricCommentReply": { + "additionalProperties": true, + "description": "A reply to a comment attached to a metric", + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Free text of the reply", + "title": "Content" + }, + "created_at": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "When the reply was created", + "title": "Created At" + }, + "edited_at": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "When the reply was last edited. Null or missing if never edited.", + "title": "Edited At" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Id of the reply, e.g. a UUID", + "title": "Id" + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Username of the author of the reply", + "title": "Username" + } + }, + "title": "MetricCommentReply", + "type": "object" + }, + "MetricMetadata": { + "additionalProperties": true, + "description": "Metadata of a metric, i.e. of a single, possibly multidimensional timeseries\n\nA metric is identified by its field named by dataset_metadata.metric_key, whose values occur\nin the metric column of a MULTITSFRAME. Every metric must have this field as a string,\nunique among the metrics.", + "properties": { + "channel_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "deprecated": true, + "description": "Actual platform channel id. Deprecated, since the platform calls channels signals now.", + "title": "Channel Id" + }, + "comments": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/MetricComment" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Comments attached to the metric, describing situations in its timeseries data", + "title": "Comments" + }, + "data_frequency": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Interval in which values of the metric are recorded as Pandas date offset alias or timedelta string, e.g. \"15min\" or \"1D\": For recording_mode periodic the interval between subsequent values, for on_change the interval after which the value is recorded again at the latest, also without a change (heartbeat). Together with data_frequency_offset and data_frequency_timezone it takes precedence over ref_data_frequency, ref_data_frequency_offset and ref_data_frequency_timezone of the dataset metadata for this metric.", + "title": "Data Frequency" + }, + "data_frequency_offset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset of the timestamps against the starts of the intervals of data_frequency (see data_frequency_timezone) as timedelta string, e.g. \"4min\": not negative and smaller than data_frequency. 0 if missing or null. Requires data_frequency.", + "title": "Data Frequency Offset" + }, + "data_frequency_timezone": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Time zone the intervals of data_frequency are aligned to, as IANA time zone name like \"Europe/Berlin\" or \"UTC\" (preferred, since it respects daylight saving time) or as fixed UTC offset like \"+02:00\". Without it, the intervals start at the multiples of data_frequency since 1970-01-01T00:00:00Z. With it, for a data_frequency shorter than a day, the intervals restart at each local midnight: They start at local midnight plus multiples of data_frequency, and the last interval of a day ends at the next local midnight, so it may be shorter, e.g. on the day of a daylight saving time change. For a data_frequency of a day or longer, the intervals follow the calendar of the time zone, e.g. days from local midnight to local midnight. The offset shifts the intervals in both cases. Requires data_frequency.", + "title": "Data Frequency Timezone" + }, + "default_aggregation_method": { + "anyOf": [ + { + "$ref": "#/$defs/AggregationMethod" + }, + { + "type": "null" + } + ], + "default": null, + "description": "How the timeseries should be aggregated by default, e.g. when components plot or report it in coarser intervals. If missing or null: LAST if the value_data_type of the value dimension \"value\" is boolean or string, otherwise MEAN. AVG is an alias of MEAN. E.g. the hetida platform provides the method it aggregates the signal with when bucketing it, for a signal group member the one configured for the member." + }, + "default_timezone": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Time zone the timeseries should be evaluated in by default, as IANA time zone name like \"Europe/Berlin\" or \"UTC\" (preferred, since it respects daylight saving time) or as fixed UTC offset like \"+02:00\". Components evaluating the timeseries in calendar intervals use it, e.g. a report aggregating days, which start at midnight of this time zone, so the results differ between time zones. This is not the time zone timestamps are displayed in: That is the plot target timezone of the execution context, which depends on the viewer, while this time zone belongs to the data, e.g. the time zone of the plant. Falls back to dataset_metadata.default_timezone.", + "title": "Default Timezone" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A one sentence description", + "title": "Description" + }, + "display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A shorter name for labels, captions or plot titles", + "title": "Display Name" + }, + "external_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unique id / key in external source systems. Often used to identify the metric in the metric column, i.e. as metric_key.", + "title": "External Id" + }, + "id": { + "description": "Id of the metric, also if another field identifies the metric in the data via dataset_metadata.metric_key. E.g. the hetida platform provides the signal id here.", + "title": "Id", + "type": "string" + }, + "inherited": { + "anyOf": [ + { + "$ref": "#/$defs/InheritedMetricMetadata" + }, + { + "type": "null" + } + ], + "default": null + }, + "name": { + "description": "Long name of the metric", + "title": "Name", + "type": "string" + }, + "origin": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Where the values come from, as free text of the source system, e.g. \"OPC_UA\", \"MQTT\" or \"FORM\" for values entered manually.", + "title": "Origin" + }, + "path": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Location of the metric in the hierarchy of its source system: the names of the levels from the top down to the object the metric belongs to, without the metric itself, e.g. [\"Plant A\", \"Pickling Unit\", \"Influx\"]. Corresponds to the breadcrumb path of the sources of generic REST adapters.", + "title": "Path" + }, + "recording_aggregation_method": { + "anyOf": [ + { + "$ref": "#/$defs/AggregationMethod" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Only for recording_mode periodic: how each value aggregates the values of the interval [timestamp, timestamp + data_frequency), e.g. MEAN for an average or MAX for a maximum. Null or missing if the values are samples, e.g. the value holding at the timestamp." + }, + "recording_mode": { + "anyOf": [ + { + "$ref": "#/$defs/RecordingMode" + }, + { + "type": "null" + } + ], + "default": null, + "description": "When the values are recorded. Unknown if missing or null. periodic requires data_frequency (or dataset_metadata.ref_data_frequency)." + }, + "short_display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short name for legends in plots", + "title": "Short Display Name" + }, + "signal_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Actual platform signal id", + "title": "Signal Id" + }, + "value_dimensions": { + "description": "Metadata of the value dimensions of this metric: an entry for each value column of the data, i.e. exactly the value dimension \"value\" for a SERIES. Each column may occur at most once.", + "items": { + "$ref": "#/$defs/ValueDimensionMetadata" + }, + "minItems": 1, + "title": "Value Dimensions", + "type": "array" + } + }, + "required": [ + "id", + "name", + "value_dimensions" + ], + "title": "MetricMetadata", + "type": "object" + }, + "MultiTSFrameDatasetMetadata": { + "additionalProperties": true, + "description": "Metadata associated with a MULTITSFRAME as a whole\n\nE.g. the queried time interval or how existing data should be handled when the dataset is\nsent to an adapter. The fields concerning invalidation and deletion are only relevant for\nadapters actually supporting these operations, like the built-in SQL adapter. Other adapters\nmay ignore them.\n\nStart and end of an interval must be provided together.", + "else": { + "properties": { + "ref_interval_end_timestamp": { + "type": "string" + }, + "ref_interval_start_timestamp": { + "type": "string" + }, + "ref_interval_type": { + "type": "string" + } + }, + "required": [ + "ref_interval_start_timestamp", + "ref_interval_end_timestamp", + "ref_interval_type" + ] + }, + "if": { + "properties": { + "ref_dataset_discrete": { + "const": true + } + }, + "required": [ + "ref_dataset_discrete" + ] + }, + "properties": { + "default_aggregation_interval": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Interval to aggregate the timeseries to by default, e.g. when components plot or report them, as Pandas date offset alias or timedelta string, e.g. \"15min\" or \"1D\". The metrics are aggregated with their default_aggregation_method. E.g. a hetida platform signal group is configured with such an interval.", + "title": "Default Aggregation Interval" + }, + "default_timezone": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Time zone the timeseries should be evaluated in by default, as IANA time zone name like \"Europe/Berlin\" or \"UTC\" (preferred, since it respects daylight saving time) or as fixed UTC offset like \"+02:00\". Components evaluating the timeseries in calendar intervals use it, e.g. a report aggregating days, which start at midnight of this time zone, so the results differ between time zones. This is not the time zone timestamps are displayed in: That is the plot target timezone of the execution context, which depends on the viewer, while this time zone belongs to the data, e.g. the time zone of the plant. Applies to all metrics without their own default_timezone.", + "title": "Default Timezone" + }, + "delete_invalidated": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Whether data to be invalidated should be deleted instead. What null means is up to the adapter. Adapters do not need to support deletion.", + "title": "Delete Invalidated" + }, + "invalidate_dataset": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Whether existing data should be invalidated: in the invalidation interval if specified, otherwise in the ref interval. If neither is specified, a closed ref interval from the first to the last datapoint is inferred. Without datapoints no invalidation can occur then. What null means is up to the adapter.", + "title": "Invalidate Dataset" + }, + "invalidation_interval_end": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End of an invalidation interval separate from the ref interval in UTC isoformat", + "title": "Invalidation Interval End" + }, + "invalidation_interval_start": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start of an invalidation interval separate from the ref interval in UTC isoformat", + "title": "Invalidation Interval Start" + }, + "invalidation_interval_type": { + "anyOf": [ + { + "$ref": "#/$defs/IntervalType" + }, + { + "type": "null" + } + ], + "default": "closed", + "description": "Type of the invalidation interval" + }, + "invalidation_timestamp": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Invalidation timestamp for all invalidated datapoints. If null, the execution_timestamp query parameter sent to generic REST adapters or, if that is not present either, the current time should be used.", + "title": "Invalidation Timestamp" + }, + "metric_key": { + "description": "Which field of the metric objects under metrics identifies a metric, i.e. occurs in the metric column of a MULTITSFRAME, e.g. \"id\" or \"external_id\".", + "title": "Metric Key", + "type": "string" + }, + "new_data_invalidation_date": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An invalidation date to be stored together with all new datapoints, i.e. invalidating them in advance. Ignored if null.", + "title": "New Data Invalidation Date" + }, + "only_invalidate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "If true, existing data should only be invalidated and the new data should not be stored. Cannot be true if invalidate_dataset is false.", + "title": "Only Invalidate" + }, + "queried_metrics": { + "description": "The requested metrics. Since not all queried metrics may have data in the queried interval, a metric may be missing in the data.", + "items": { + "type": "string" + }, + "title": "Queried Metrics", + "type": "array" + }, + "query_timestamp": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The point in time whose state of the data was delivered in UTC isoformat: Data stored, changed or invalidated later is not included. A source supporting it delivers the same data again when queried for this point in time, e.g. for reproducing an execution.", + "title": "Query Timestamp" + }, + "ref_data_frequency": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Frequency of the timeseries as Pandas date offset alias or timedelta string, e.g. \"15min\" or \"1D\": the data_frequency of all metrics without their own, with the same meaning as there. Together with ref_data_frequency_offset and ref_data_frequency_timezone.", + "title": "Ref Data Frequency" + }, + "ref_data_frequency_offset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset of the timestamps against the starts of the intervals of ref_data_frequency (see ref_data_frequency_timezone) as timedelta string, e.g. \"4min\": not negative and smaller than ref_data_frequency. 0 if missing or null. Requires ref_data_frequency. Applies to all metrics without their own data_frequency.", + "title": "Ref Data Frequency Offset" + }, + "ref_data_frequency_timezone": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Time zone the intervals of ref_data_frequency are aligned to, as IANA time zone name like \"Europe/Berlin\" or \"UTC\" (preferred, since it respects daylight saving time) or as fixed UTC offset like \"+02:00\". Without it, the intervals start at the multiples of ref_data_frequency since 1970-01-01T00:00:00Z. With it, for a ref_data_frequency shorter than a day, the intervals restart at each local midnight: They start at local midnight plus multiples of ref_data_frequency, and the last interval of a day ends at the next local midnight, so it may be shorter, e.g. on the day of a daylight saving time change. For a ref_data_frequency of a day or longer, the intervals follow the calendar of the time zone, e.g. days from local midnight to local midnight. The offset shifts the intervals in both cases. Requires ref_data_frequency. Applies to all metrics without their own data_frequency.", + "title": "Ref Data Frequency Timezone" + }, + "ref_dataset_discrete": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Whether the ref dataset should be considered as discrete, i.e. isolated datapoints instead of an interval. If true, the ref interval is not required.", + "title": "Ref Dataset Discrete" + }, + "ref_interval_end_timestamp": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End of the queried time interval (ref interval) in UTC isoformat. Required unless ref_dataset_discrete is true.", + "title": "Ref Interval End Timestamp" + }, + "ref_interval_start_timestamp": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start of the queried time interval (ref interval) in UTC isoformat. Required unless ref_dataset_discrete is true.", + "title": "Ref Interval Start Timestamp" + }, + "ref_interval_type": { + "anyOf": [ + { + "$ref": "#/$defs/IntervalType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Type of the ref interval. Required unless ref_dataset_discrete is true." + }, + "signal_group": { + "anyOf": [ + { + "$ref": "#/$defs/SignalGroupMetadata" + }, + { + "type": "null" + } + ], + "default": null + }, + "single_metric": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If the MULTITSFRAME holds a single metric: its metric_key field value. Must not be set if more than one metric is present.", + "title": "Single Metric" + } + }, + "required": [ + "metric_key", + "queried_metrics" + ], + "title": "MultiTSFrameDatasetMetadata", + "type": "object" + }, + "MultiTSFrameMetadata": { + "additionalProperties": true, + "description": "hetida designer metadata conventions for a MULTITSFRAME\n\nStructure of the metadata in the attrs attribute of a MULTITSFRAME during execution, which\nis also exchanged with adapters, e.g. via the Data-Attributes header of the generic REST\nadapter multitsframe endpoint or via the __metadata__ field of the wrapped format of the\ndirect provisioning adapter.\n\nThe fields that are not required are optional. Additional fields are allowed at every level.\nTimestamps are UTC isoformat strings with explicit offset (Z or +00:00).", + "examples": [ + { + "dataset_metadata": { + "default_aggregation_interval": "1h", + "default_timezone": "Europe/Berlin", + "metric_key": "external_id", + "queried_metrics": [ + "plant_a.pressure", + "plant_a.temperature" + ], + "ref_interval_end_timestamp": "2026-01-02T00:00:00+00:00", + "ref_interval_start_timestamp": "2026-01-01T00:00:00+00:00", + "ref_interval_type": "closed" + }, + "metrics": [ + { + "external_id": "plant_a.pressure", + "id": "c5a8e2f1-0b3d-4e6a-9f7c-1d2e3f4a5b6c", + "name": "Plant A pressure", + "value_dimensions": [ + { + "column": "value", + "measurement": "pressure", + "unit": "bar" + } + ] + }, + { + "data_frequency": "1h", + "default_aggregation_method": "MAX", + "external_id": "plant_a.temperature", + "id": "8d7c6b5a-4f3e-4d2c-8b1a-0f9e8d7c6b5a", + "name": "Plant A temperature", + "recording_mode": "on_change", + "value_dimensions": [ + { + "column": "value", + "deadband": 0.5, + "decimals": 1, + "measurement": "temperature", + "unit": "°C" + } + ] + } + ] + } + ], + "properties": { + "dataset_metadata": { + "$ref": "#/$defs/MultiTSFrameDatasetMetadata" + }, + "metrics": { + "description": "Metadata of the metrics: one for each metric in the data. Each metric must have the field named by dataset_metadata.metric_key as a string, unique among the metrics.", + "items": { + "$ref": "#/$defs/MetricMetadata" + }, + "title": "Metrics", + "type": "array" + }, + "value_dimensions_shared": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/SharedValueDimensionMetadata" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Metadata of value dimensions that have the same meaning for all metrics. Consumers use it as fallback, but it does not replace the value dimensions of the metrics. Each column may occur at most once.", + "title": "Value Dimensions Shared" + } + }, + "required": [ + "dataset_metadata", + "metrics" + ], + "title": "MultiTSFrameMetadata", + "type": "object" + }, + "RecordingMode": { + "description": "When the values of a timeseries are recorded\n\nperiodic: one value per interval of data_frequency, timestamped with the start of the\ninterval. A missing value is a gap.\non_change: a value is recorded when it changes (by more than the deadband of its value\ndimension) and holds until the next one. Missing timestamps are no gaps, as long as\ndata_frequency (the heartbeat) is not exceeded.\nirregular: values at arbitrary points in time.", + "enum": [ + "periodic", + "on_change", + "irregular" + ], + "title": "RecordingMode", + "type": "string" + }, + "SeriesMetadata": { + "additionalProperties": true, + "description": "hetida designer metadata conventions for a SERIES\n\nStructure of the metadata in the attrs attribute of a SERIES during execution, which is also\nexchanged with adapters, e.g. via the Data-Attributes header of the generic REST adapter\ntimeseries endpoint (for each timeseries) or via the __metadata__ field of the wrapped format\nof the direct provisioning adapter.\n\nThe fields that are not required are optional. Additional fields are allowed at every level.\nTimestamps are UTC isoformat strings with explicit offset (Z or +00:00).", + "examples": [ + { + "dataset_metadata": { + "metric_key": "id", + "queried_metrics": [ + "7ab8c419-8745-4714-b588-57c0c772cc95" + ], + "query_timestamp": "2026-01-02T00:05:00+00:00", + "ref_interval_end_timestamp": "2026-01-02T00:00:00+00:00", + "ref_interval_start_timestamp": "2026-01-01T00:00:00+00:00", + "ref_interval_type": "right_open", + "single_metric": "7ab8c419-8745-4714-b588-57c0c772cc95" + }, + "metrics": [ + { + "data_frequency": "15min", + "default_aggregation_method": "MEAN", + "default_timezone": "Europe/Berlin", + "id": "7ab8c419-8745-4714-b588-57c0c772cc95", + "name": "Water level north", + "origin": "OPC_UA", + "path": [ + "Waterworks North", + "Gauge North" + ], + "recording_aggregation_method": "MEAN", + "recording_mode": "periodic", + "value_dimensions": [ + { + "column": "value", + "decimals": 2, + "measurement": "level", + "unit": "m" + } + ] + } + ] + } + ], + "properties": { + "dataset_metadata": { + "$ref": "#/$defs/SingleMetricDatasetMetadata" + }, + "metrics": { + "description": "Metadata of the single metric: exactly one metric, which has the field named by dataset_metadata.metric_key with the value of dataset_metadata.single_metric.", + "items": { + "$ref": "#/$defs/MetricMetadata" + }, + "maxItems": 1, + "minItems": 1, + "title": "Metrics", + "type": "array" + }, + "value_dimensions_shared": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/SharedValueDimensionMetadata" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Metadata of value dimensions that have the same meaning for all metrics. Consumers use it as fallback, but it does not replace the value dimensions of the metrics. Each column may occur at most once.", + "title": "Value Dimensions Shared" + } + }, + "required": [ + "dataset_metadata", + "metrics" + ], + "title": "SeriesMetadata", + "type": "object" + }, + "SharedValueDimensionMetadata": { + "additionalProperties": true, + "description": "Metadata of a value dimension shared by all metrics\n\nEvery column of a MULTITSFRAME besides \"timestamp\" and \"metric\" and every column of a\nSINGLETSFRAME besides \"timestamp\" defines a value dimension for each metric. A SERIES has\nexactly one value dimension, which is \"value\".", + "properties": { + "column": { + "description": "The actual column name in the MULTITSFRAME or SINGLETSFRAME, connecting the value dimension to its column. \"value\" for a SERIES.", + "title": "Column", + "type": "string" + }, + "deadband": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "For a metric with recording_mode on_change: the change of the values, in their unit, which has to be exceeded for a value to be recorded. 0 records every change.", + "title": "Deadband" + }, + "decimals": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number of decimal places numeric values should be displayed with, in their unit, e.g. 2 for 12.35. Only affects how values are displayed, e.g. in tables, tooltips or reports; the values themselves are not rounded. Up to the consumer if missing or null.", + "title": "Decimals" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description. Should be considered empty if missing.", + "title": "Description" + }, + "display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A shorter name for labels, captions or plot titles. Falls back to name.", + "title": "Display Name" + }, + "measurement": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The measured quantity, e.g. \"temperature\"", + "title": "Measurement" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Name of the value dimension, e.g. \"temperature\". May equal column.", + "title": "Name" + }, + "short_display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short name for legends in plots. Falls back to display_name.", + "title": "Short Display Name" + }, + "unit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unit of the values, e.g. \"°C\" or \"m/s\", \"\" for dimensionless quantities", + "title": "Unit" + }, + "value_data_type": { + "anyOf": [ + { + "$ref": "#/$defs/ValueDataType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Data type of the values" + } + }, + "required": [ + "column" + ], + "title": "SharedValueDimensionMetadata", + "type": "object" + }, + "SignalGroupMetadata": { + "additionalProperties": true, + "description": "A group of signals a dataset was read from\n\nE.g. the hetida platform provides the signal group a MULTITSFRAME was read from.", + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Description of the signal group", + "title": "Description" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Id of the signal group", + "title": "Id" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Name of the signal group", + "title": "Name" + } + }, + "title": "SignalGroupMetadata", + "type": "object" + }, + "SingleMetricDatasetMetadata": { + "additionalProperties": true, + "description": "Metadata associated with a SERIES or SINGLETSFRAME as a whole\n\nE.g. the queried time interval or how existing data should be handled when the dataset is\nsent to an adapter. The fields concerning invalidation and deletion are only relevant for\nadapters actually supporting these operations, like the built-in SQL adapter. Other adapters\nmay ignore them.\n\nStart and end of an interval must be provided together.", + "else": { + "properties": { + "ref_interval_end_timestamp": { + "type": "string" + }, + "ref_interval_start_timestamp": { + "type": "string" + }, + "ref_interval_type": { + "type": "string" + } + }, + "required": [ + "ref_interval_start_timestamp", + "ref_interval_end_timestamp", + "ref_interval_type" + ] + }, + "if": { + "properties": { + "ref_dataset_discrete": { + "const": true + } + }, + "required": [ + "ref_dataset_discrete" + ] + }, + "properties": { + "default_aggregation_interval": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Interval to aggregate the timeseries to by default, e.g. when components plot or report them, as Pandas date offset alias or timedelta string, e.g. \"15min\" or \"1D\". The metrics are aggregated with their default_aggregation_method. E.g. a hetida platform signal group is configured with such an interval.", + "title": "Default Aggregation Interval" + }, + "default_timezone": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Time zone the timeseries should be evaluated in by default, as IANA time zone name like \"Europe/Berlin\" or \"UTC\" (preferred, since it respects daylight saving time) or as fixed UTC offset like \"+02:00\". Components evaluating the timeseries in calendar intervals use it, e.g. a report aggregating days, which start at midnight of this time zone, so the results differ between time zones. This is not the time zone timestamps are displayed in: That is the plot target timezone of the execution context, which depends on the viewer, while this time zone belongs to the data, e.g. the time zone of the plant. Applies to all metrics without their own default_timezone.", + "title": "Default Timezone" + }, + "delete_invalidated": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Whether data to be invalidated should be deleted instead. What null means is up to the adapter. Adapters do not need to support deletion.", + "title": "Delete Invalidated" + }, + "invalidate_dataset": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Whether existing data should be invalidated: in the invalidation interval if specified, otherwise in the ref interval. If neither is specified, a closed ref interval from the first to the last datapoint is inferred. Without datapoints no invalidation can occur then. What null means is up to the adapter.", + "title": "Invalidate Dataset" + }, + "invalidation_interval_end": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End of an invalidation interval separate from the ref interval in UTC isoformat", + "title": "Invalidation Interval End" + }, + "invalidation_interval_start": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start of an invalidation interval separate from the ref interval in UTC isoformat", + "title": "Invalidation Interval Start" + }, + "invalidation_interval_type": { + "anyOf": [ + { + "$ref": "#/$defs/IntervalType" + }, + { + "type": "null" + } + ], + "default": "closed", + "description": "Type of the invalidation interval" + }, + "invalidation_timestamp": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Invalidation timestamp for all invalidated datapoints. If null, the execution_timestamp query parameter sent to generic REST adapters or, if that is not present either, the current time should be used.", + "title": "Invalidation Timestamp" + }, + "metric_key": { + "description": "Which field of the metric objects under metrics identifies a metric, i.e. occurs in the metric column of a MULTITSFRAME, e.g. \"id\" or \"external_id\".", + "title": "Metric Key", + "type": "string" + }, + "new_data_invalidation_date": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An invalidation date to be stored together with all new datapoints, i.e. invalidating them in advance. Ignored if null.", + "title": "New Data Invalidation Date" + }, + "only_invalidate": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "If true, existing data should only be invalidated and the new data should not be stored. Cannot be true if invalidate_dataset is false.", + "title": "Only Invalidate" + }, + "queried_metrics": { + "description": "The requested metrics. Since not all queried metrics may have data in the queried interval, a metric may be missing in the data.", + "items": { + "type": "string" + }, + "title": "Queried Metrics", + "type": "array" + }, + "query_timestamp": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The point in time whose state of the data was delivered in UTC isoformat: Data stored, changed or invalidated later is not included. A source supporting it delivers the same data again when queried for this point in time, e.g. for reproducing an execution.", + "title": "Query Timestamp" + }, + "ref_data_frequency": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Frequency of the timeseries as Pandas date offset alias or timedelta string, e.g. \"15min\" or \"1D\": the data_frequency of all metrics without their own, with the same meaning as there. Together with ref_data_frequency_offset and ref_data_frequency_timezone.", + "title": "Ref Data Frequency" + }, + "ref_data_frequency_offset": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset of the timestamps against the starts of the intervals of ref_data_frequency (see ref_data_frequency_timezone) as timedelta string, e.g. \"4min\": not negative and smaller than ref_data_frequency. 0 if missing or null. Requires ref_data_frequency. Applies to all metrics without their own data_frequency.", + "title": "Ref Data Frequency Offset" + }, + "ref_data_frequency_timezone": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Time zone the intervals of ref_data_frequency are aligned to, as IANA time zone name like \"Europe/Berlin\" or \"UTC\" (preferred, since it respects daylight saving time) or as fixed UTC offset like \"+02:00\". Without it, the intervals start at the multiples of ref_data_frequency since 1970-01-01T00:00:00Z. With it, for a ref_data_frequency shorter than a day, the intervals restart at each local midnight: They start at local midnight plus multiples of ref_data_frequency, and the last interval of a day ends at the next local midnight, so it may be shorter, e.g. on the day of a daylight saving time change. For a ref_data_frequency of a day or longer, the intervals follow the calendar of the time zone, e.g. days from local midnight to local midnight. The offset shifts the intervals in both cases. Requires ref_data_frequency. Applies to all metrics without their own data_frequency.", + "title": "Ref Data Frequency Timezone" + }, + "ref_dataset_discrete": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Whether the ref dataset should be considered as discrete, i.e. isolated datapoints instead of an interval. If true, the ref interval is not required.", + "title": "Ref Dataset Discrete" + }, + "ref_interval_end_timestamp": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End of the queried time interval (ref interval) in UTC isoformat. Required unless ref_dataset_discrete is true.", + "title": "Ref Interval End Timestamp" + }, + "ref_interval_start_timestamp": { + "anyOf": [ + { + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]+)?(Z|\\+00:00)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start of the queried time interval (ref interval) in UTC isoformat. Required unless ref_dataset_discrete is true.", + "title": "Ref Interval Start Timestamp" + }, + "ref_interval_type": { + "anyOf": [ + { + "$ref": "#/$defs/IntervalType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Type of the ref interval. Required unless ref_dataset_discrete is true." + }, + "signal_group": { + "anyOf": [ + { + "$ref": "#/$defs/SignalGroupMetadata" + }, + { + "type": "null" + } + ], + "default": null + }, + "single_metric": { + "description": "The metric_key field value of the single metric under metrics.", + "title": "Single Metric", + "type": "string" + } + }, + "required": [ + "metric_key", + "queried_metrics", + "single_metric" + ], + "title": "SingleMetricDatasetMetadata", + "type": "object" + }, + "SingleTSFrameMetadata": { + "additionalProperties": true, + "description": "hetida designer metadata conventions for a SINGLETSFRAME\n\nStructure of the metadata in the attrs attribute of a SINGLETSFRAME during execution, which\nis also exchanged with adapters, e.g. via the Data-Attributes header of the generic REST\nadapter singletsframe endpoint or via the __metadata__ field of the wrapped format of the\ndirect provisioning adapter.\n\nThe fields that are not required are optional. Additional fields are allowed at every level.\nTimestamps are UTC isoformat strings with explicit offset (Z or +00:00).", + "examples": [ + { + "dataset_metadata": { + "metric_key": "external_id", + "queried_metrics": [ + "abc.temp" + ], + "ref_interval_end_timestamp": "2026-01-02T00:00:00+00:00", + "ref_interval_start_timestamp": "2026-01-01T00:00:00+00:00", + "ref_interval_type": "closed", + "single_metric": "abc.temp" + }, + "metrics": [ + { + "external_id": "abc.temp", + "id": "3f2b1c4d-5e6f-4a7b-8c9d-0e1f2a3b4c5d", + "name": "ABC temperature", + "value_dimensions": [ + { + "column": "value", + "measurement": "temperature", + "unit": "°C" + }, + { + "column": "state", + "measurement": null, + "name": "measurement state", + "unit": null + } + ] + } + ] + } + ], + "properties": { + "dataset_metadata": { + "$ref": "#/$defs/SingleMetricDatasetMetadata" + }, + "metrics": { + "description": "Metadata of the single metric: exactly one metric, which has the field named by dataset_metadata.metric_key with the value of dataset_metadata.single_metric.", + "items": { + "$ref": "#/$defs/MetricMetadata" + }, + "maxItems": 1, + "minItems": 1, + "title": "Metrics", + "type": "array" + }, + "value_dimensions_shared": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/SharedValueDimensionMetadata" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Metadata of value dimensions that have the same meaning for all metrics. Consumers use it as fallback, but it does not replace the value dimensions of the metrics. Each column may occur at most once.", + "title": "Value Dimensions Shared" + } + }, + "required": [ + "dataset_metadata", + "metrics" + ], + "title": "SingleTSFrameMetadata", + "type": "object" + }, + "ValueDataType": { + "description": "Data type of the values of a value dimension", + "enum": [ + "float", + "integer", + "boolean", + "string" + ], + "title": "ValueDataType", + "type": "string" + }, + "ValueDimensionMetadata": { + "additionalProperties": true, + "description": "Metadata of a value dimension of a metric\n\nEvery column of a MULTITSFRAME besides \"timestamp\" and \"metric\" and every column of a\nSINGLETSFRAME besides \"timestamp\" defines a value dimension for each metric. A SERIES has\nexactly one value dimension, which is \"value\". Each metric provides an entry for each of its\nvalue dimensions.", + "properties": { + "column": { + "description": "The actual column name in the MULTITSFRAME or SINGLETSFRAME, connecting the value dimension to its column. \"value\" for a SERIES.", + "title": "Column", + "type": "string" + }, + "deadband": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "For a metric with recording_mode on_change: the change of the values, in their unit, which has to be exceeded for a value to be recorded. 0 records every change.", + "title": "Deadband" + }, + "decimals": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number of decimal places numeric values should be displayed with, in their unit, e.g. 2 for 12.35. Only affects how values are displayed, e.g. in tables, tooltips or reports; the values themselves are not rounded. Up to the consumer if missing or null.", + "title": "Decimals" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description. Should be considered empty if missing.", + "title": "Description" + }, + "display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A shorter name for labels, captions or plot titles. Falls back to name.", + "title": "Display Name" + }, + "measurement": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The measured quantity, e.g. \"temperature\". Null if unknown.", + "title": "Measurement" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Name of the value dimension, e.g. \"temperature\". May equal column.", + "title": "Name" + }, + "short_display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short name for legends in plots. Falls back to display_name.", + "title": "Short Display Name" + }, + "unit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Unit of the values, e.g. \"°C\" or \"m/s\". \"\" for dimensionless quantities, null if unknown.", + "title": "Unit" + }, + "value_data_type": { + "anyOf": [ + { + "$ref": "#/$defs/ValueDataType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Data type of the values" + } + }, + "required": [ + "column", + "unit", + "measurement" + ], + "title": "ValueDimensionMetadata", + "type": "object" + } + } +} diff --git a/tests/data/fixtures/metadata.py b/tests/data/fixtures/metadata.py deleted file mode 100644 index 88b9a90..0000000 --- a/tests/data/fixtures/metadata.py +++ /dev/null @@ -1,74 +0,0 @@ -import json - -import pandas as pd -import pytest - - -@pytest.fixture(scope="function") -def empty_mts_with_old_attr() -> pd.DataFrame: - with open("tests/data/json_templates/old_mts_attrs.json", "r") as file: - metadata_for_mts = json.load(file) - - empty_mts = pd.DataFrame() - empty_mts.attrs = metadata_for_mts - return empty_mts - - -@pytest.fixture(scope="function") -def empty_mts_with_attr() -> pd.DataFrame: - with open("tests/data/json_templates/mts_attrs.json", "r") as file: - metadata_for_mts = json.load(file) - - empty_mts = pd.DataFrame() - empty_mts.attrs = metadata_for_mts - return empty_mts - - -@pytest.fixture(scope="function") -def empty_mts_with_old_attr_real() -> pd.DataFrame: - with open("tests/data/json_templates/mts_attrs_old_real.json", "r") as file: - metadata_for_mts = json.load(file) - - empty_mts = pd.DataFrame() - empty_mts.attrs = metadata_for_mts - return empty_mts - - -@pytest.fixture(scope="function") -def empty_singletsframe_with_attr() -> pd.DataFrame: - with open("tests/data/json_templates/singletsframe_attrs.json", "r") as file: - metadata_for_sts = json.load(file) - - empty_sts = pd.DataFrame() - empty_sts.attrs = metadata_for_sts - return empty_sts - - -@pytest.fixture(scope="function") -def empty_series_with_old_attr_real() -> pd.Series: - with open("tests/data/json_templates/series_attrs_old_real.json", "r") as file: - metadata_for_series = json.load(file) - - empty_series = pd.Series() - empty_series.attrs = metadata_for_series - return empty_series - - -@pytest.fixture(scope="function") -def empty_series_with_old_attr() -> pd.Series: - with open("tests/data/json_templates/old_series_attrs.json", "r") as file: - metadata_for_series = json.load(file) - - empty_series = pd.Series() - empty_series.attrs = metadata_for_series - return empty_series - - -@pytest.fixture(scope="function") -def empty_series_with_attr() -> pd.Series: - with open("tests/data/json_templates/series_attrs.json", "r") as file: - metadata_for_series = json.load(file) - - empty_series = pd.Series() - empty_series.attrs = metadata_for_series - return empty_series diff --git a/tests/data/json_templates/mts_attrs.json b/tests/data/json_templates/mts_attrs.json deleted file mode 100644 index 055166b..0000000 --- a/tests/data/json_templates/mts_attrs.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "dataset_metadata": { - "metric_key": "id", - "ref_interval_start_timestamp": "2023-01-01T00:00:00+00:00", - "ref_interval_end_timestamp": "2023-01-02T00:00:00Z" - }, - "value_dimensions_shared": [ - { - "column": "pressure", - "unit": "bar", - "name": "shared value_dimension pressure name", - "measurement": "pressure" - }, - { - "column": "temp", - "measurement": "temperature" - } - ], - "metrics": [ - { - "id": "first", - "external_id": "external_first", - "unit": "m", - "display_name": "first display name", - "value_dimensions": [ - { - "column": "temp", - "unit": "C", - "measurement": "temperature" - } - ] - }, - { - "id": "second", - "name": "second name", - "external_id": "external_second", - "value_dimensions": [ - { - "column": "temp", - "unit": "C" - } - ] - }, - { - "id": "third", - "external_id": "external_third", - "measurement": "height", - "value_dimensions": [ - { - "column": "pressure", - "display_name": "thirds's pressure" - } - ] - }, - { - "id": "fourth", - "external_id": "external_fourth", - "unit": "l", - "value_dimensions": [ - { - "column": "pressure", - "unit" : "Pa" - } - ] - }, - { - "id": "fifth", - "external_id": "external_fifth", - "unit": "l", - "value_dimensions": [ - { - "column": "value", - "unit" : "m^3" - } - ] - } - ] -} diff --git a/tests/data/json_templates/mts_attrs_old_real.json b/tests/data/json_templates/mts_attrs_old_real.json deleted file mode 100644 index f2e93a0..0000000 --- a/tests/data/json_templates/mts_attrs_old_real.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "dataset_metadata": { - "ref_interval_start_timestamp": "2025-11-05T13:28:00Z", - "ref_interval_end_timestamp": "2025-11-06T13:28:00Z", - "ref_interval_type": "closed", - "ref_metric": "Wasserstand Ruhr Meschede", - "ref_data_frequency": null, - "ref_data_frequency_offset": null, - "invalidation_interval_start": null, - "invalidation_interval_end": null, - "invalidation_interval_type": null, - "invalidate_dataset": null, - "delete_invalidated": null, - "only_invalidate": null, - "ref_dataset_discrete": null, - "invalidation_timestamp": null, - "new_data_invalidation_date": null - }, - "by_metric": { - "metric1": { - "metric": { - "name": "Wasserstand Ruhr Meschede", - "display_name": null, - "short_display_name": null, - "description": null, - "value_data_type": null, - "external_id": "wasserstand.ruhr.meschede2", - "channel_id": "303fc49a-b515-4fbc-b4fd-70594f053f58" - }, - "inherited": { - "Stationsname": "Meschede (Ruhr)", - "Latitude": 51.347759, - "Longitude": 8.280575 - }, - "value_dimensions": { - "value_dim1": { - "name": "Wasserstand Ruhr Meschede v1", - "display_name": null, - "short_display_name": null, - "description": null, - "unit": "m³/s", - "value_data_type": "float" - }, - "value_dim2": { - "name": "Wasserstand Ruhr Meschede v2", - "display_name": null, - "short_display_name": null, - "description": null, - "unit": "m³/s", - "value_data_type": "float" - } - } - }, - "metric2": { - "metric": { - "name": "Wasserstand Ruhr Meschede", - "display_name": null, - "short_display_name": null, - "description": null, - "value_data_type": null, - "external_id": "wasserstand.ruhr.meschede2", - "channel_id": "303fc49a-b515-4fbc-b4fd-70594f053f59" - }, - "inherited": { - "Stationsname": "Meschede (Ruhr)", - "Latitude": 51.347759, - "Longitude": 8.280575 - }, - "value_dimensions": { - "value_dim1": { - "name": "Wasserstand Ruhr Meschede (2)", - "display_name": null, - "short_display_name": null, - "description": null, - "unit": "m³/h", - "value_data_type": "float" - } - } - } - } -} diff --git a/tests/data/json_templates/old_mts_attrs.json b/tests/data/json_templates/old_mts_attrs.json deleted file mode 100644 index 847708f..0000000 --- a/tests/data/json_templates/old_mts_attrs.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "metrics": { - "test_channel": { - "tenantId": "tenant_a", - "id": "0eeccba1-b38d-48ac-bad7-630a743e241b", - "name": "Some name", - "icon": null, - "type": "FLOAT", - "unit": "l/min", - "externalTimeSeriesId": "some_external_id", - "description": null, - "measurement": null, - "visualization": null, - "virtualChannel": true, - "aggregationType": "ORIGINAL", - "aggregationInterval": null, - "schedulerJob": { - "id": "some_id" - }, - "singleTimeseries": true - }, - "some_other_metric": { - "unit": "l", - "measurement": "volume" - } - } -} diff --git a/tests/data/json_templates/old_series_attrs.json b/tests/data/json_templates/old_series_attrs.json deleted file mode 100644 index 4a9d188..0000000 --- a/tests/data/json_templates/old_series_attrs.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "dataset_metadata": { - "ref_interval_start_timestamp": "2026-02-06T12:45:00Z", - "ref_interval_end_timestamp": "2026-02-06T12:45:00Z", - "ref_interval_type": "closed", - "ref_metric": "Muster Channel", - "ref_data_frequency": null, - "ref_data_frequency_offset": null, - "invalidation_interval_start": null, - "invalidation_interval_end": null, - "invalidation_interval_type": null, - "invalidate_dataset": null, - "delete_invalidated": null, - "only_invalidate": null, - "ref_dataset_discrete": null, - "invalidation_timestamp": null, - "new_data_invalidation_date": null - }, - "single_metric_metadata": { - "structured_metadata": { - "metric": { - "name": "Muster Channel", - "display_name": null, - "short_display_name": "muster", - "description": "*neue Beschreibung", - "unit": "C°", - "value_data_type": null, - "external_id": "raspi_demo.Heizung4.Temperature", - "channel_id": "67d5fc88-af8f-4901-92d1-cc61090a2023" - }, - "inherited": {}, - "value_dimensions": { - "value": { - "display_name": null, - "short_display_name": null, - "description": null, - "unit": "C°", - "value_data_type": "float" - } - } - } - } - } diff --git a/tests/data/json_templates/series_attrs.json b/tests/data/json_templates/series_attrs.json deleted file mode 100644 index 52590ed..0000000 --- a/tests/data/json_templates/series_attrs.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "dataset_metadata": { - "metric_key": "id", - "single_metric": "series", - "ref_interval_start_timestamp": "2023-01-01T00:00:00+00:00", - "ref_interval_end_timestamp": "2023-01-02T00:00:00Z" - }, - "metrics": [ - { - "id": "series", - "external_id": "external_first", - "unit": "m", - "name": "first name" - } - ] -} diff --git a/tests/data/json_templates/series_attrs_old_real.json b/tests/data/json_templates/series_attrs_old_real.json deleted file mode 100644 index 2246548..0000000 --- a/tests/data/json_templates/series_attrs_old_real.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "dataset_metadata": { - "ref_interval_start_timestamp": "2025-11-05T13:28:00Z", - "ref_interval_end_timestamp": "2025-11-06T13:28:00Z", - "ref_interval_type": "closed", - "ref_metric": "Wasserstand", - "ref_data_frequency": null, - "ref_data_frequency_offset": null, - "invalidation_interval_start": null, - "invalidation_interval_end": null, - "invalidation_interval_type": null, - "invalidate_dataset": null, - "delete_invalidated": null, - "only_invalidate": null, - "ref_dataset_discrete": null, - "invalidation_timestamp": null, - "new_data_invalidation_date": null - }, - "single_metric_metadata": { - "structured_metadata": { - "metric": { - "name": "Wasserstand", - "display_name": null, - "short_display_name": "W", - "description": "neu berechneter Wasserstand", - "unit": "m³/s", - "value_data_type": null, - "external_id": "virtual-channel.b4a6c64d-5e72-44a4-a551-e3f51f8e19b0", - "channel_id": "09922b06-2c62-4e18-9c1e-857949674ebb" - }, - "inherited": { - "Stationsname": "Mühlenstrang - Brücke Ruhr", - "Latitude": 51.436477, - "Longitude": 7.583455, - "Referenzniveau": 3000 - }, - "value_dimensions": { - "value": { - "name": "Wasserstand value", - "display_name": null, - "short_display_name": "W value", - "description": null, - "unit": "m³/s value", - "value_data_type": "float" - } - } - } - } -} diff --git a/tests/data/json_templates/singletsframe_attrs.json b/tests/data/json_templates/singletsframe_attrs.json deleted file mode 100644 index 1192ff2..0000000 --- a/tests/data/json_templates/singletsframe_attrs.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "dataset_metadata": { - "single_metric": "abc.temp", - "metric_key": "external_id" - }, - "metrics": [ - { - "external_id": "abc.temp", - "name": "ABC temperature", - "value_dimensions": [ - { - "column": "value", - "name": "temperature", - "unit": "°C" - }, - { - "column": "state", - "name": "measurement state" - } - ] - } - ], - "value_dimensions_shared": [ - { - "column": "state", - "unit": "UNKNOWN" - } - ] -} \ No newline at end of file diff --git a/tests/metadata/test_attached.py b/tests/metadata/test_attached.py new file mode 100644 index 0000000..807af33 --- /dev/null +++ b/tests/metadata/test_attached.py @@ -0,0 +1,159 @@ +import copy +import json +from typing import Any + +import numpy as np +import pandas as pd +import pytest + +from hdhelpers.metadata import ( + META_KEY, + MetadataConventionError, + attach_metadata, + attrs_to_json, + check_metadata_conventions, + read_metadata, + set_metadata, +) +from hdhelpers.metadata.attached import infer_data_type +from hdhelpers.metadata.conventions import ( + MULTITSFRAME_METADATA_EXAMPLE, + SINGLETSFRAME_METADATA_EXAMPLE, + MultiTSFrameMetadata, + SingleTSFrameMetadata, +) + +OTHER_ATTRS = {"hetida-designer_filters": {"from": "2026-01-01"}} + + +def multitsframe(attrs: dict[str, Any] | None = None) -> pd.DataFrame: + df = pd.DataFrame( + { + "timestamp": pd.date_range("2026-01-01T06:00", periods=4, freq="h", tz="UTC"), + "metric": ["plant_a.pressure", "plant_a.temperature"] * 2, + "value": np.arange(4.0), + } + ) + df.attrs = copy.deepcopy( + MULTITSFRAME_METADATA_EXAMPLE | OTHER_ATTRS if attrs is None else attrs + ) + return df + + +def singletsframe() -> pd.DataFrame: + df = pd.DataFrame( + { + "timestamp": pd.date_range("2026-01-01T06:00", periods=2, freq="h", tz="UTC"), + "value": [1.0, 2.0], + } + ) + df.attrs = copy.deepcopy(SINGLETSFRAME_METADATA_EXAMPLE) + return df + + +def test_infer_data_type() -> None: + assert infer_data_type(pd.Series([1.0])) == "SERIES" + assert infer_data_type(multitsframe()) == "MULTITSFRAME" + assert infer_data_type(singletsframe()) == "SINGLETSFRAME" + + +class TestAttach: + def test_model_replaces_the_convention_keys(self) -> None: + df = multitsframe() + + meta = attach_metadata(df) + + assert isinstance(meta, MultiTSFrameMetadata) + assert df.attrs == OTHER_ATTRS | {META_KEY: meta} + + def test_attached_model_stays(self) -> None: + df = multitsframe() + meta = attach_metadata(df) + + assert attach_metadata(df) is meta + + @pytest.mark.parametrize( + "attrs", + [ + {}, + OTHER_ATTRS, + {"metrics": []}, + MULTITSFRAME_METADATA_EXAMPLE | {"dataset_metadata": {"metric_key": "id"}}, + ], + ) + def test_attrs_without_valid_metadata_stay(self, attrs: dict[str, Any]) -> None: + df = multitsframe(attrs) + + assert attach_metadata(df) is None + assert df.attrs == attrs + + +class TestRead: + def test_attached_model(self) -> None: + df = multitsframe() + meta = attach_metadata(df) + + assert read_metadata(df) is meta + assert read_metadata(df, "MULTITSFRAME") is meta + assert read_metadata(df[df["value"] > 1]) is meta + + def test_metadata_of_another_data_type(self) -> None: + df = singletsframe() + meta = attach_metadata(df) + series = df["value"] + + assert read_metadata(series) is meta + assert read_metadata(series, "SERIES") is None + with pytest.raises(MetadataConventionError, match="SINGLETSFRAME metadata"): + read_metadata(series, "SERIES", required=True) + + def test_plain_attrs_are_parsed_without_attaching(self) -> None: + df = singletsframe() + + meta = read_metadata(df) + + assert isinstance(meta, SingleTSFrameMetadata) + assert df.attrs == SINGLETSFRAME_METADATA_EXAMPLE + + def test_no_metadata(self) -> None: + df = multitsframe(OTHER_ATTRS) + + assert read_metadata(df) is None + with pytest.raises(MetadataConventionError, match="No metadata"): + read_metadata(df, required=True) + + def test_invalid_metadata(self) -> None: + attrs = copy.deepcopy(MULTITSFRAME_METADATA_EXAMPLE) + attrs["metrics"][0]["default_timezone"] = "Mars/Base" + df = multitsframe(attrs) + + assert read_metadata(df) is None + with pytest.raises(MetadataConventionError) as raised: + read_metadata(df, required=True) + assert [issue.location for issue in raised.value.issues] == ["metrics[0].default_timezone"] + + +def test_set_metadata_replaces_attached_metadata_and_convention_keys() -> None: + df = multitsframe() + meta = MultiTSFrameMetadata.model_validate(MULTITSFRAME_METADATA_EXAMPLE) + + set_metadata(df, meta) + + assert df.attrs == OTHER_ATTRS | {META_KEY: meta} + + +def test_attrs_to_json_reproduces_the_attrs() -> None: + df = multitsframe() + attach_metadata(df) + + assert attrs_to_json(df.attrs) == MULTITSFRAME_METADATA_EXAMPLE | OTHER_ATTRS + assert json.loads(json.dumps(attrs_to_json(df.attrs))) == attrs_to_json(df.attrs) + assert attrs_to_json(OTHER_ATTRS) == OTHER_ATTRS + + +def test_check_metadata_conventions_with_attached_metadata() -> None: + df = multitsframe() + issues = check_metadata_conventions(df) + attach_metadata(df) + + assert check_metadata_conventions(df) == issues diff --git a/tests/metadata/test_conventions.py b/tests/metadata/test_conventions.py new file mode 100644 index 0000000..ee4c00f --- /dev/null +++ b/tests/metadata/test_conventions.py @@ -0,0 +1,1039 @@ +from copy import deepcopy +from typing import Any + +import numpy as np +import pandas as pd +import pytest + +from hdhelpers.metadata.conventions import ( + MULTITSFRAME_METADATA_EXAMPLE, + SERIES_METADATA_EXAMPLE, + SINGLETSFRAME_METADATA_EXAMPLE, + MetadataConventionIssue, + MetadataIssueSeverity, + check_metadata_conventions, +) + +SIGNAL_ID = "7ab8c419-8745-4714-b588-57c0c772cc95" + + +def value_dimension(column: str = "value", **fields: Any) -> dict[str, Any]: + return {"column": column, "unit": None, "measurement": None} | fields + + +def metric(metric_id: str, **fields: Any) -> dict[str, Any]: + return { + "id": metric_id, + "name": f"name of {metric_id}", + "value_dimensions": [value_dimension()], + } | fields + + +def dataset_metadata(**fields: Any) -> dict[str, Any]: + return { + "metric_key": "id", + "queried_metrics": [], + "ref_interval_start_timestamp": "2026-10-18T00:00:00Z", + "ref_interval_end_timestamp": "2026-10-19T00:00:00Z", + "ref_interval_type": "closed", + } | fields + + +def series_metadata(**metric_fields: Any) -> dict[str, Any]: + return { + "dataset_metadata": dataset_metadata(single_metric="a", queried_metrics=["a"]), + "metrics": [metric("a", **metric_fields)], + } + + +def multitsframe_metadata(*metric_ids: str) -> dict[str, Any]: + return { + "dataset_metadata": dataset_metadata(queried_metrics=list(metric_ids)), + "metrics": [metric(metric_id) for metric_id in metric_ids], + } + + +def with_dataset_metadata(metadata: dict[str, Any], **fields: Any) -> dict[str, Any]: + changed = deepcopy(metadata) + changed["dataset_metadata"] |= fields + return changed + + +def remove(metadata: dict[str, Any], *path: str | int) -> dict[str, Any]: + removed = deepcopy(metadata) + obj: Any = removed + for part in path[:-1]: + obj = obj[part] + del obj[path[-1]] + return removed + + +# a comment as the hetida platform sends it: Jackson serializes Instants as UTC isoformat strings +# with Z and includes null fields +platform_comment: dict[str, Any] = { + "id": "0b5f0f0e-6d1c-4f7e-9a51-1f3c2a9e8d11", + "start_date": "2026-10-18T06:00:00Z", + "end_date": "2026-10-18T07:30:00Z", + "created_at": "2026-10-18T08:12:45.123Z", + "edited_at": None, + "content": "Pump maintenance", + "username": "jdoe", + "tags": ["maintenance"], + "replies": [ + { + "id": "5e1c7d2a-8b3f-4c6e-9d0a-2f4b6c8e0a13", + "created_at": "2026-10-18T09:00:00Z", + "edited_at": "2026-10-18T09:05:00.500Z", + "content": "Done, pump runs again", + "username": "asmith", + } + ], +} + +# the metadata the hetida platform sends for a timeseries of a SERIES read +platform_series_metadata: dict[str, Any] = { + "dataset_metadata": { + "metric_key": "id", + "single_metric": SIGNAL_ID, + "queried_metrics": [SIGNAL_ID], + "signal_group": None, + "ref_interval_start_timestamp": "2026-10-18T00:00:00Z", + "ref_interval_end_timestamp": "2026-10-19T00:00:00Z", + "ref_interval_type": "closed", + "ref_data_frequency": None, + "ref_data_frequency_offset": None, + "invalidation_interval_start": None, + "invalidation_interval_end": None, + "invalidation_interval_type": None, + "invalidate_dataset": None, + "delete_invalidated": None, + "only_invalidate": None, + "ref_dataset_discrete": None, + "invalidation_timestamp": None, + "new_data_invalidation_date": None, + }, + "metrics": [ + { + "id": SIGNAL_ID, + "external_id": "water.north.level", + "name": "Water-lvl-north", + "display_name": None, + "description": None, + # property values of the platform, possibly as strings + "inherited": {"signal_role": "level", "latitude": "51.43"}, + "comments": [ + platform_comment, + # a comment on a single point in time without tags and replies + platform_comment + | { + "start_date": "2026-10-18T12:00:00Z", + "end_date": "2026-10-18T12:00:00Z", + "tags": [], + "replies": [], + }, + ], + "value_dimensions": [ + { + "column": "value", + "name": "Water-lvl-north", + "display_name": None, + "short_display_name": None, + "description": None, + "unit": "m", + "measurement": "level", + "value_data_type": "float", + } + ], + "default_aggregation_method": "AVG", + } + ], +} + +# the metadata the hetida platform sends for a MULTITSFRAME read of a signal group +platform_multitsframe_metadata: dict[str, Any] = { + "dataset_metadata": platform_series_metadata["dataset_metadata"] + | { + "single_metric": None, + "queried_metrics": [SIGNAL_ID, "pump"], + "signal_group": {"id": "3c2e1d4f", "name": "Pumps north", "description": None}, + }, + "metrics": [ + platform_series_metadata["metrics"][0] | {"display_name": "Water level"}, + platform_series_metadata["metrics"][0] + | { + "id": "pump", + "external_id": None, + "comments": [], + "value_dimensions": [value_dimension()], + "default_aggregation_method": None, + }, + ], +} + + +def multitsframe(attrs: dict[str, Any], metrics: list[str], **columns: list) -> pd.DataFrame: + frame = pd.DataFrame( + { + "timestamp": pd.date_range("2026-10-18T06:00:00Z", periods=len(metrics), freq="1h"), + "metric": metrics, + "value": np.arange(len(metrics), dtype=float), + } + | columns + ) + frame.attrs = deepcopy(attrs) + return frame + + +def series(attrs: dict[str, Any]) -> pd.Series: + ser = pd.Series([1.0, 2.0], index=pd.date_range("2026-10-18T06:00:00Z", periods=2, freq="1h")) + ser.attrs = deepcopy(attrs) + return ser + + +def singletsframe(attrs: dict[str, Any]) -> pd.DataFrame: + frame = pd.DataFrame( + { + # in the ref interval of SINGLETSFRAME_METADATA_EXAMPLE + "timestamp": pd.date_range("2026-01-01T06:00:00Z", periods=2, freq="1h"), + "value": [21.3, 21.5], + "state": [0, 1], + } + ) + frame.attrs = deepcopy(attrs) + return frame + + +def issues_as_tuples( + issues: list[MetadataConventionIssue], +) -> list[tuple[MetadataIssueSeverity, str]]: + return [(issue.severity, issue.location) for issue in issues] + + +def errors_of(issues: list[MetadataConventionIssue]) -> list[str]: + return [issue.location for issue in issues if issue.severity == MetadataIssueSeverity.ERROR] + + +@pytest.mark.parametrize( + ("obj", "data_type"), + [ + (SERIES_METADATA_EXAMPLE, "SERIES"), + (SINGLETSFRAME_METADATA_EXAMPLE, "SINGLETSFRAME"), + (MULTITSFRAME_METADATA_EXAMPLE, "MULTITSFRAME"), + (platform_series_metadata, "SERIES"), + (platform_multitsframe_metadata, "MULTITSFRAME"), + (multitsframe_metadata(), "MULTITSFRAME"), + (series(platform_series_metadata), None), + (series(series_metadata()), None), + (multitsframe(platform_multitsframe_metadata, [SIGNAL_ID, "pump"]), None), + (multitsframe(multitsframe_metadata("a", "b"), ["a", "b", "a"]), None), + (singletsframe(SINGLETSFRAME_METADATA_EXAMPLE), None), + ], +) +def test_conforming_metadata(obj: Any, data_type: Any) -> None: + assert check_metadata_conventions(obj, data_type) == [] + + +@pytest.mark.parametrize( + ("data_type", "metadata", "location"), + [ + ("SERIES", {}, "dataset_metadata"), + ("SERIES", remove(series_metadata(), "metrics"), "metrics"), + ( + "SERIES", + remove(series_metadata(), "dataset_metadata", "metric_key"), + "dataset_metadata.metric_key", + ), + ( + "SERIES", + remove(series_metadata(), "dataset_metadata", "queried_metrics"), + "dataset_metadata.queried_metrics", + ), + ( + "SERIES", + remove(series_metadata(), "dataset_metadata", "single_metric"), + "dataset_metadata.single_metric", + ), + ("SERIES", remove(series_metadata(), "metrics", 0, "id"), "metrics[0].id"), + ("SERIES", remove(series_metadata(), "metrics", 0, "name"), "metrics[0].name"), + ( + "SERIES", + remove(series_metadata(), "metrics", 0, "value_dimensions"), + "metrics[0].value_dimensions", + ), + ( + "SERIES", + remove(series_metadata(), "metrics", 0, "value_dimensions", 0, "column"), + "metrics[0].value_dimensions[0].column", + ), + ( + "SERIES", + remove(series_metadata(), "metrics", 0, "value_dimensions", 0, "unit"), + "metrics[0].value_dimensions[0].unit", + ), + ( + "SERIES", + remove(series_metadata(), "metrics", 0, "value_dimensions", 0, "measurement"), + "metrics[0].value_dimensions[0].measurement", + ), + ("SINGLETSFRAME", remove(series_metadata(), "metrics", 0, "name"), "metrics[0].name"), + ( + "MULTITSFRAME", + remove(multitsframe_metadata("a"), "dataset_metadata", "queried_metrics"), + "dataset_metadata.queried_metrics", + ), + ], +) +def test_required_fields(data_type: Any, metadata: dict[str, Any], location: str) -> None: + issues = check_metadata_conventions(metadata, data_type) + + assert location in errors_of(issues) + assert next(issue for issue in issues if issue.location == location).message == ( + "Field required" + ) + + +@pytest.mark.parametrize("data_type", ["SERIES", "SINGLETSFRAME"]) +@pytest.mark.parametrize("metric_count", [0, 2]) +def test_single_metric_types_have_exactly_one_metric(data_type: Any, metric_count: int) -> None: + metadata = series_metadata() + metadata["metrics"] = [metric(f"m{index}") for index in range(metric_count)] + + assert "metrics" in errors_of(check_metadata_conventions(metadata, data_type)) + + +def test_value_dimensions_must_not_be_empty() -> None: + assert "metrics[0].value_dimensions" in errors_of( + check_metadata_conventions(series_metadata(value_dimensions=[]), "SERIES") + ) + + +def test_ref_interval_is_required_unless_discrete() -> None: + metadata = with_dataset_metadata( + series_metadata(), + ref_interval_start_timestamp=None, + ref_interval_end_timestamp=None, + ref_interval_type=None, + ) + + assert errors_of(check_metadata_conventions(metadata, "SERIES")) == [ + "dataset_metadata.ref_interval_start_timestamp", + "dataset_metadata.ref_interval_end_timestamp", + "dataset_metadata.ref_interval_type", + ] + + metadata["dataset_metadata"]["ref_dataset_discrete"] = True + assert check_metadata_conventions(metadata, "SERIES") == [] + + +@pytest.mark.parametrize( + ("metadata", "location", "message_part"), + [ + ([1, 2], "", "object"), + ( + with_dataset_metadata(series_metadata(), ref_interval_type="half_open"), + "dataset_metadata.ref_interval_type", + "closed", + ), + ( + with_dataset_metadata( + series_metadata(), ref_interval_start_timestamp="2026-10-18T00:00:00+02:00" + ), + "dataset_metadata.ref_interval_start_timestamp", + "ending with Z or +00:00, got '2026-10-18T00:00:00+02:00'.", + ), + ( + with_dataset_metadata(series_metadata(), only_invalidate="true"), + "dataset_metadata.only_invalidate", + "boolean", + ), + (series_metadata(id=42), "metrics[0].id", "string"), + (series_metadata(name=None), "metrics[0].name", "string"), + ( + series_metadata(value_dimensions=[value_dimension(unit=3)]), + "metrics[0].value_dimensions[0].unit", + "string", + ), + ( + series_metadata(default_aggregation_method="mean"), + "metrics[0].default_aggregation_method", + "MEAN", + ), + ( + series_metadata(comments=[{"created_at": "2026-10-18T08:12:45"}]), + "metrics[0].comments[0].created_at", + "UTC isoformat timestamp", + ), + ( + series_metadata(comments=[platform_comment | {"tags": "maintenance"}]), + "metrics[0].comments[0].tags", + "array", + ), + ( + series_metadata() | {"metrics": {"a": {"unit": "m"}}}, + "metrics", + "older metadata convention", + ), + ], +) +def test_structure_errors(metadata: Any, location: str, message_part: str) -> None: + issues = check_metadata_conventions(metadata, "SERIES") + + assert issues_as_tuples(issues) == [(MetadataIssueSeverity.ERROR, location)] + assert message_part in issues[0].message + + +@pytest.mark.parametrize( + "method", ["MEAN", "AVG", "MEDIAN", "SUM", "MIN", "MAX", "FIRST", "LAST", "COUNT", None] +) +def test_aggregation_methods(method: Any) -> None: + metadata = series_metadata(default_aggregation_method=method) + + assert check_metadata_conventions(metadata, "SERIES") == [] + + +def test_inherited_properties_are_arbitrary() -> None: + metadata = series_metadata(inherited={"latitude": "51.43", "longitude": 7.58, "unit": None}) + + assert check_metadata_conventions(metadata, "SERIES") == [] + + +def test_recording_and_location_information() -> None: + metadata = with_dataset_metadata( + series_metadata( + path=["Waterworks North", "Gauge North"], + origin="OPC_UA", + recording_mode="periodic", + data_frequency="15min", + data_frequency_offset="0s", + recording_aggregation_method="AVG", + ), + query_timestamp="2026-10-19T00:05:00Z", + default_aggregation_interval="1h", + ) + + assert check_metadata_conventions(metadata, "SERIES") == [] + + +@pytest.mark.parametrize( + "metadata", + [ + # the frequency of the dataset defines the aggregated intervals + with_dataset_metadata( + series_metadata(recording_mode="periodic", recording_aggregation_method="MAX"), + ref_data_frequency="1h", + ), + series_metadata( + recording_mode="on_change", + data_frequency="1D", + value_dimensions=[value_dimension(deadband=0)], + ), + series_metadata(recording_mode="irregular"), + series_metadata(data_frequency="P1D"), + # calendar based frequencies and offsets of fixed duration + series_metadata(recording_mode="periodic", data_frequency="MS"), + series_metadata(data_frequency="1D", data_frequency_offset="23h"), + series_metadata(data_frequency="h", data_frequency_offset="59min"), + with_dataset_metadata( + series_metadata(), ref_data_frequency="1h", ref_data_frequency_offset="0s" + ), + with_dataset_metadata(series_metadata(), default_aggregation_interval="1MS"), + # intervals restarting at local midnight, e.g. compressed by the hetida platform + series_metadata( + recording_mode="periodic", + data_frequency="2h", + data_frequency_timezone="Europe/Berlin", + recording_aggregation_method="MEAN", + ), + series_metadata( + data_frequency="1D", data_frequency_offset="6h", data_frequency_timezone="+05:30" + ), + with_dataset_metadata( + series_metadata(recording_mode="periodic"), + ref_data_frequency="4h", + ref_data_frequency_timezone="America/New_York", + ), + ], +) +def test_consistent_recording_information(metadata: dict[str, Any]) -> None: + assert check_metadata_conventions(metadata, "SERIES") == [] + + +@pytest.mark.parametrize( + ("metadata", "location", "message_part"), + [ + (series_metadata(path="Plant A/Unit"), "metrics[0].path", "array"), + (series_metadata(path=["Plant A", 3]), "metrics[0].path[1]", "string"), + (series_metadata(recording_mode="continuous"), "metrics[0].recording_mode", "periodic"), + ( + series_metadata(recording_aggregation_method="mean"), + "metrics[0].recording_aggregation_method", + "MEAN", + ), + ( + series_metadata(value_dimensions=[value_dimension(deadband=-0.1)]), + "metrics[0].value_dimensions[0].deadband", + "greater than or equal to 0", + ), + ( + with_dataset_metadata(series_metadata(), query_timestamp="2026-10-19T02:05:00+02:00"), + "dataset_metadata.query_timestamp", + "UTC isoformat timestamp", + ), + ( + with_dataset_metadata(series_metadata(), query_timestamp="2026-10-18T24:00:00Z"), + "dataset_metadata.query_timestamp", + "UTC isoformat timestamp", + ), + ( + with_dataset_metadata(series_metadata(), default_aggregation_interval=60), + "dataset_metadata.default_aggregation_interval", + "string", + ), + ], +) +def test_recording_and_location_structure_errors( + metadata: Any, location: str, message_part: str +) -> None: + issues = check_metadata_conventions(metadata, "SERIES") + + assert issues_as_tuples(issues) == [(MetadataIssueSeverity.ERROR, location)] + assert message_part in issues[0].message + + +@pytest.mark.parametrize( + ("metadata", "location", "message_part"), + [ + ( + with_dataset_metadata(series_metadata(), ref_data_frequency="every 5 minutes"), + "dataset_metadata.ref_data_frequency", + "not a Pandas date offset alias", + ), + ( + with_dataset_metadata( + series_metadata(), ref_data_frequency="1h", ref_data_frequency_offset="" + ), + "dataset_metadata.ref_data_frequency_offset", + "not a Pandas date offset alias", + ), + ( + with_dataset_metadata(series_metadata(), ref_data_frequency_offset="5min"), + "dataset_metadata.ref_data_frequency_offset", + "Requires ref_data_frequency", + ), + # the offset of the dataset only applies together with its frequency + ( + with_dataset_metadata( + series_metadata(data_frequency_offset="5min"), ref_data_frequency="1h" + ), + "metrics[0].data_frequency_offset", + "Requires data_frequency", + ), + ( + series_metadata(data_frequency="15min", data_frequency_offset="15min"), + "metrics[0].data_frequency_offset", + 'not smaller than data_frequency "15min"', + ), + ( + series_metadata(data_frequency="1D", data_frequency_offset="-1h"), + "metrics[0].data_frequency_offset", + "not a non-negative timedelta", + ), + ( + series_metadata(data_frequency="1D", data_frequency_offset="MS"), + "metrics[0].data_frequency_offset", + "not a non-negative timedelta", + ), + ( + series_metadata(data_frequency="1D", data_frequency_offset="1D"), + "metrics[0].data_frequency_offset", + 'not smaller than data_frequency "1D"', + ), + ( + series_metadata(data_frequency="1"), + "metrics[0].data_frequency", + "not a Pandas date offset alias", + ), + (series_metadata(data_frequency="0s"), "metrics[0].data_frequency", "not positive"), + ( + series_metadata(data_frequency_timezone="Europe/Berlin"), + "metrics[0].data_frequency_timezone", + "Requires data_frequency", + ), + # the time zone of the dataset only applies together with its frequency + ( + with_dataset_metadata( + series_metadata(data_frequency_timezone="Europe/Berlin"), ref_data_frequency="1h" + ), + "metrics[0].data_frequency_timezone", + "Requires data_frequency", + ), + ( + with_dataset_metadata(series_metadata(), ref_data_frequency_timezone="Europe/Berlin"), + "dataset_metadata.ref_data_frequency_timezone", + "Requires ref_data_frequency", + ), + ( + series_metadata(data_frequency="2h", data_frequency_timezone="Europe/Berlinn"), + "metrics[0].data_frequency_timezone", + "IANA time zone name", + ), + ( + with_dataset_metadata( + series_metadata(), ref_data_frequency="2h", ref_data_frequency_timezone="CEST" + ), + "dataset_metadata.ref_data_frequency_timezone", + "IANA time zone name", + ), + (series_metadata(data_frequency="-15min"), "metrics[0].data_frequency", "not positive"), + ( + with_dataset_metadata(series_metadata(), default_aggregation_interval="-1MS"), + "dataset_metadata.default_aggregation_interval", + "not positive", + ), + ( + with_dataset_metadata(series_metadata(), default_aggregation_interval="1month"), + "dataset_metadata.default_aggregation_interval", + "not a Pandas date offset alias", + ), + ( + series_metadata(data_frequency="hourly"), + "metrics[0].data_frequency", + "not a Pandas date offset alias", + ), + ( + series_metadata(data_frequency="1h", data_frequency_offset="4 mins later"), + "metrics[0].data_frequency_offset", + "not a Pandas date offset alias", + ), + ( + series_metadata( + recording_mode="on_change", recording_aggregation_method="MEAN", data_frequency="1h" + ), + "metrics[0].recording_aggregation_method", + "Only meaningful for recording_mode periodic", + ), + ( + series_metadata(recording_aggregation_method="MEAN", data_frequency="1h"), + "metrics[0].recording_aggregation_method", + "Only meaningful for recording_mode periodic", + ), + ( + series_metadata(recording_mode="periodic", recording_aggregation_method="MEAN"), + "metrics[0].recording_mode", + "periodic requires data_frequency", + ), + ( + series_metadata(recording_mode="periodic"), + "metrics[0].recording_mode", + "periodic requires data_frequency", + ), + ], +) +def test_recording_consistency_errors( + metadata: dict[str, Any], location: str, message_part: str +) -> None: + issues = check_metadata_conventions(metadata, "SERIES") + + assert issues_as_tuples(issues) == [(MetadataIssueSeverity.ERROR, location)] + assert message_part in issues[0].message + + +@pytest.mark.parametrize("timezone", ["Europe/Berlin", "UTC", "Etc/GMT-2", "+02:00", "-05:30"]) +def test_default_timezones(timezone: str) -> None: + metadata = with_dataset_metadata( + series_metadata(default_timezone=timezone), default_timezone=timezone + ) + + assert check_metadata_conventions(metadata, "SERIES") == [] + + +@pytest.mark.parametrize( + "timezone", + [ + "Europe/Berln", + "europe/berlin", + "Europe", + "UTC+02:00", + "+2:00", + "+15:00", + "", + # files of the system's time zone database, which are no time zone names + "posixrules", + "right/UTC", + "localtime", + ], +) +@pytest.mark.parametrize("level", ["dataset_metadata", "metrics[0]"]) +def test_invalid_default_timezones(timezone: str, level: str) -> None: + metadata = ( + with_dataset_metadata(series_metadata(), default_timezone=timezone) + if level == "dataset_metadata" + else series_metadata(default_timezone=timezone) + ) + + issues = check_metadata_conventions(metadata, "SERIES") + + assert issues_as_tuples(issues) == [(MetadataIssueSeverity.ERROR, f"{level}.default_timezone")] + assert "IANA time zone name" in issues[0].message + + +def test_default_timezone_must_be_a_string() -> None: + issues = check_metadata_conventions(series_metadata(default_timezone=1), "SERIES") + + assert issues_as_tuples(issues) == [ + (MetadataIssueSeverity.ERROR, "metrics[0].default_timezone") + ] + + +@pytest.mark.parametrize("decimals", [0, 3, None]) +def test_decimals(decimals: Any) -> None: + metadata = series_metadata(value_dimensions=[value_dimension(decimals=decimals)]) | { + "value_dimensions_shared": [{"column": "value", "decimals": decimals}] + } + + assert check_metadata_conventions(metadata, "SERIES") == [] + + +@pytest.mark.parametrize( + ("decimals", "message_part"), + [(-1, "greater than or equal to 0"), (1.5, "integer"), ("2", "integer"), (True, "integer")], +) +def test_invalid_decimals(decimals: Any, message_part: str) -> None: + metadata = series_metadata(value_dimensions=[value_dimension(decimals=decimals)]) + + issues = check_metadata_conventions(metadata, "SERIES") + + assert issues_as_tuples(issues) == [ + (MetadataIssueSeverity.ERROR, "metrics[0].value_dimensions[0].decimals") + ] + assert message_part in issues[0].message + + +@pytest.mark.parametrize("recording_mode", [None, "periodic", "irregular"]) +def test_deadband_without_on_change_recording_warns(recording_mode: Any) -> None: + metadata = series_metadata( + recording_mode=recording_mode, + data_frequency="1h", + value_dimensions=[value_dimension(deadband=0.5)], + ) + + issues = check_metadata_conventions(metadata, "SERIES") + + assert issues_as_tuples(issues) == [ + (MetadataIssueSeverity.WARNING, "metrics[0].value_dimensions[0].deadband") + ] + + +@pytest.mark.parametrize( + "metadata", + [ + with_dataset_metadata( + series_metadata(), ref_interval_start_timestamp=pd.Timestamp("2026-01-01", tz="UTC") + ), + series_metadata(inherited={"elevation": float("nan")}), + ], +) +def test_metadata_not_json_serializable(metadata: dict[str, Any]) -> None: + issues = check_metadata_conventions(metadata, "SERIES") + + assert issues_as_tuples(issues) == [(MetadataIssueSeverity.ERROR, "")] + assert "JSON serializable" in issues[0].message + + +@pytest.mark.parametrize( + ("data_type", "metadata", "location"), + [ + ( + "MULTITSFRAME", + with_dataset_metadata(multitsframe_metadata("a"), metric_key="external_id"), + "metrics[0]", + ), + ("MULTITSFRAME", multitsframe_metadata("a", "b", "a"), "metrics"), + ( + "SERIES", + series_metadata(value_dimensions=[value_dimension(), value_dimension()]), + "metrics[0].value_dimensions", + ), + ( + "MULTITSFRAME", + multitsframe_metadata("a") + | {"value_dimensions_shared": [{"column": "value"}, {"column": "value"}]}, + "value_dimensions_shared", + ), + ( + "MULTITSFRAME", + with_dataset_metadata(multitsframe_metadata("a", "b"), single_metric="a"), + "dataset_metadata.single_metric", + ), + ( + "SERIES", + with_dataset_metadata(series_metadata(), single_metric="b"), + "dataset_metadata.single_metric", + ), + ( + "SERIES", + with_dataset_metadata( + series_metadata(), only_invalidate=True, invalidate_dataset=False + ), + "dataset_metadata.only_invalidate", + ), + ( + "SERIES", + with_dataset_metadata( + series_metadata(), invalidation_interval_end="2026-01-01T00:00:00Z" + ), + "dataset_metadata.invalidation_interval_start", + ), + ( + "SERIES", + with_dataset_metadata( + series_metadata(), ref_interval_start_timestamp="2026-10-20T00:00:00.000000000Z" + ), + "dataset_metadata.ref_interval_start_timestamp", + ), + ( + "SERIES", + series_metadata(comments=[platform_comment | {"end_date": None}]), + "metrics[0].comments[0].end_date", + ), + ( + "SERIES", + series_metadata(comments=[platform_comment | {"start_date": "2026-10-18T08:00:00Z"}]), + "metrics[0].comments[0].start_date", + ), + # a SERIES has the value dimension "value", which must be described + ( + "SERIES", + series_metadata(value_dimensions=[value_dimension("temperature")]), + "metrics[0].value_dimensions", + ), + ( + "SERIES", + with_dataset_metadata(series_metadata(), queried_metrics=["b"]), + "dataset_metadata.single_metric", + ), + ], +) +def test_consistency_errors(data_type: Any, metadata: dict[str, Any], location: str) -> None: + assert errors_of(check_metadata_conventions(metadata, data_type)) == [location] + + +@pytest.mark.parametrize( + ("metadata", "location"), + [ + ( + with_dataset_metadata( + series_metadata(), ref_interval_start_timestamp="2026-02-30T00:00:00Z" + ), + "dataset_metadata.ref_interval_start_timestamp", + ), + ( + with_dataset_metadata(series_metadata(), query_timestamp="2026-04-31T00:00:00Z"), + "dataset_metadata.query_timestamp", + ), + ( + series_metadata(comments=[platform_comment | {"end_date": "2026-02-29T00:00:00Z"}]), + "metrics[0].comments[0].end_date", + ), + ], +) +def test_timestamps_must_be_actual_points_in_time(metadata: dict[str, Any], location: str) -> None: + issues = check_metadata_conventions(metadata, "SERIES") + + assert issues_as_tuples(issues) == [(MetadataIssueSeverity.ERROR, location)] + assert "not an actual point in time" in issues[0].message + + +@pytest.mark.parametrize("value_data_type", ["float", "integer", "boolean", "string", None]) +def test_value_data_types(value_data_type: Any) -> None: + metadata = series_metadata(value_dimensions=[value_dimension(value_data_type=value_data_type)]) + + assert check_metadata_conventions(metadata, "SERIES") == [] + + +def test_unknown_value_data_type() -> None: + metadata = series_metadata(value_dimensions=[value_dimension(value_data_type="double")]) + + issues = check_metadata_conventions(metadata, "SERIES") + + assert issues_as_tuples(issues) == [ + (MetadataIssueSeverity.ERROR, "metrics[0].value_dimensions[0].value_data_type") + ] + assert "'float', 'integer', 'boolean' or 'string'" in issues[0].message + + +@pytest.mark.parametrize("with_data", [True, False]) +def test_queried_metrics_without_metadata_warn(with_data: bool) -> None: + metadata = with_dataset_metadata( + multitsframe_metadata("a"), queried_metrics=["a", "without data", "b"] + ) + obj = multitsframe(metadata, ["a", "b"]) if with_data else metadata + + issues = check_metadata_conventions(obj, "MULTITSFRAME") + + # with data, "b" is reported as metric of the data missing under metrics instead + assert issues_as_tuples(issues) == ( + [(MetadataIssueSeverity.ERROR, "metrics"), (MetadataIssueSeverity.WARNING, "metrics")] + if with_data + else [(MetadataIssueSeverity.WARNING, "metrics")] + ) + assert issues[-1].message.endswith('"without data"' if with_data else '"b", "without data"') + + +@pytest.mark.parametrize( + ("ref_interval", "in_ref_interval"), + [ + (("2026-10-18T06:00:00Z", "2026-10-18T07:00:00Z", "closed"), True), + (("2026-10-18T06:00:00Z", "2026-10-18T07:00:00Z", "right_open"), False), + (("2026-10-18T06:00:00Z", "2026-10-18T08:00:00Z", "right_open"), True), + (("2026-10-18T06:00:00Z", "2026-10-18T07:00:00Z", "left_open"), False), + (("2026-10-18T06:30:00Z", "2026-10-18T07:00:00Z", "closed"), False), + (("2026-10-18T06:00:00Z", "2026-10-18T06:30:00Z", "closed"), False), + ], +) +def test_data_outside_of_ref_interval_warns( + ref_interval: tuple[str, str, str], in_ref_interval: bool +) -> None: + start, end, interval_type = ref_interval + metadata = with_dataset_metadata( + series_metadata(), + ref_interval_start_timestamp=start, + ref_interval_end_timestamp=end, + ref_interval_type=interval_type, + ) + + # the series has values at 06:00 and 07:00 + issues = check_metadata_conventions(series(metadata)) + + assert issues_as_tuples(issues) == ( + [] if in_ref_interval else [(MetadataIssueSeverity.WARNING, "dataset_metadata")] + ) + + +def test_discrete_dataset_is_not_compared_with_ref_interval() -> None: + metadata = with_dataset_metadata(series_metadata(), ref_dataset_discrete=True) + for field in ("ref_interval_start_timestamp", "ref_interval_end_timestamp"): + metadata["dataset_metadata"][field] = "2026-01-01T00:00:00Z" + + assert check_metadata_conventions(series(metadata)) == [] + + +def test_consistency_is_only_checked_for_valid_structure() -> None: + metadata = with_dataset_metadata(multitsframe_metadata("a", "a"), ref_interval_type="half_open") + + assert issues_as_tuples(check_metadata_conventions(metadata, "MULTITSFRAME")) == [ + (MetadataIssueSeverity.ERROR, "dataset_metadata.ref_interval_type") + ] + + +def test_metadata_without_data_requires_data_type() -> None: + with pytest.raises(ValueError, match="data type is required"): + check_metadata_conventions(series_metadata()) + + +def test_multitsframe_inconsistent_with_metadata() -> None: + metadata = multitsframe_metadata("a", "b") + metadata["dataset_metadata"]["queried_metrics"] = ["a"] + metadata["metrics"][0]["value_dimensions"].append(value_dimension("quality")) + + issues = check_metadata_conventions(multitsframe(metadata, ["a", "b", "c"], state=[0, 1, 0])) + + assert issues_as_tuples(issues) == [ + (MetadataIssueSeverity.ERROR, "metrics"), + (MetadataIssueSeverity.ERROR, "dataset_metadata.queried_metrics"), + (MetadataIssueSeverity.ERROR, "metrics[0].value_dimensions"), + (MetadataIssueSeverity.ERROR, "metrics[1].value_dimensions"), + (MetadataIssueSeverity.WARNING, "metrics[0].value_dimensions[1].column"), + ] + assert '"c"' in issues[0].message + assert '"b", "c"' in issues[1].message + assert '"state"' in issues[2].message + assert '"quality"' in issues[4].message + + +def test_multitsframe_metric_listing_is_shortened() -> None: + metric_ids = [f"m{i:02}" for i in range(15)] + metadata = multitsframe_metadata("a") + metadata["dataset_metadata"]["queried_metrics"] = ["a", *metric_ids] + + issues = check_metadata_conventions(multitsframe(metadata, ["a", *metric_ids])) + + assert issues_as_tuples(issues) == [(MetadataIssueSeverity.ERROR, "metrics")] + assert '"m09" and 5 more' in issues[0].message + + +def test_singletsframe_inconsistent_with_metadata() -> None: + metadata = deepcopy(SINGLETSFRAME_METADATA_EXAMPLE) + del metadata["metrics"][0]["value_dimensions"][1] + metadata["value_dimensions_shared"] = [{"column": "quality"}] + + assert issues_as_tuples(check_metadata_conventions(singletsframe(metadata))) == [ + (MetadataIssueSeverity.ERROR, "metrics[0].value_dimensions"), + (MetadataIssueSeverity.WARNING, "value_dimensions_shared[0].column"), + ] + + +@pytest.mark.parametrize( + ("metadata", "location", "message_part"), + [ + ( + series_metadata(displayName="A"), + "metrics[0].displayName", + 'Did you mean "display_name"?', + ), + ( + series_metadata(value_dimensions=[value_dimension(measurment="level")]), + "metrics[0].value_dimensions[0].measurment", + 'Did you mean "measurement"?', + ), + (series_metadata(unit="m"), "metrics[0].unit", "under value_dimensions instead"), + ( + with_dataset_metadata(series_metadata(), ref_metrics=["a"]), + "dataset_metadata.ref_metrics", + "Additional fields are allowed", + ), + ( + series_metadata(comments=[platform_comment | {"author": "jdoe"}]), + "metrics[0].comments[0].author", + "Additional fields are allowed", + ), + (series_metadata() | {"anomalies": []}, "anomalies", "Additional fields are allowed"), + ], +) +def test_unknown_fields(metadata: dict[str, Any], location: str, message_part: str) -> None: + issues = check_metadata_conventions(metadata, "SERIES") + + assert issues_as_tuples(issues) == [(MetadataIssueSeverity.WARNING, location)] + assert message_part in issues[0].message + + +def test_custom_metric_key_field_is_not_unknown() -> None: + metadata = with_dataset_metadata( + series_metadata(externalTimeSeriesId="a"), metric_key="externalTimeSeriesId" + ) + + assert check_metadata_conventions(metadata, "SERIES") == [] + + +def test_unknown_fields_explain_missing_required_fields() -> None: + metadata = series_metadata() + metadata["metrics"][0]["valueDimensions"] = metadata["metrics"][0].pop("value_dimensions") + + issues = check_metadata_conventions(metadata, "SERIES") + + assert issues_as_tuples(issues) == [ + (MetadataIssueSeverity.ERROR, "metrics[0].value_dimensions"), + (MetadataIssueSeverity.WARNING, "metrics[0].valueDimensions"), + ] + assert 'Did you mean "value_dimensions"?' in issues[1].message + + +def test_deprecated_channel_id() -> None: + metadata = multitsframe_metadata("a", "b") + metadata["metrics"][0]["channel_id"] = "abc" + metadata["metrics"][1] |= {"signal_id": "def", "channel_id": None} + + issues = check_metadata_conventions(metadata, "MULTITSFRAME") + + assert issues_as_tuples(issues) == [(MetadataIssueSeverity.WARNING, "metrics[0].channel_id")] + assert '"signal_id"' in issues[0].message diff --git a/tests/metadata/test_conventions_immutable.py b/tests/metadata/test_conventions_immutable.py new file mode 100644 index 0000000..113bc7a --- /dev/null +++ b/tests/metadata/test_conventions_immutable.py @@ -0,0 +1,171 @@ +import copy +import pickle +from collections.abc import Mapping +from typing import Any + +import numpy as np +import pandas as pd +import pytest +from pydantic import ValidationError + +from hdhelpers.metadata.conventions import ( + METADATA_MODELS, + MULTITSFRAME_METADATA_EXAMPLE, + SERIES_METADATA_EXAMPLE, + SINGLETSFRAME_METADATA_EXAMPLE, + FrozenDict, + MultiTSFrameMetadata, +) + +EXAMPLES = { + "SERIES": SERIES_METADATA_EXAMPLE, + "SINGLETSFRAME": SINGLETSFRAME_METADATA_EXAMPLE, + "MULTITSFRAME": MULTITSFRAME_METADATA_EXAMPLE, +} + + +def multitsframe_metadata(**metric_fields: Any) -> MultiTSFrameMetadata: + attrs = copy.deepcopy(MULTITSFRAME_METADATA_EXAMPLE) + attrs["metrics"][0] |= metric_fields + return MultiTSFrameMetadata.model_validate(attrs) + + +@pytest.mark.parametrize("data_type", EXAMPLES) +def test_dump_reproduces_the_input(data_type: str) -> None: + meta = METADATA_MODELS[data_type].model_validate(EXAMPLES[data_type]) + + assert meta.model_dump(mode="json") == EXAMPLES[data_type] + + +def test_dump_leaves_out_optional_fields_with_none_or_default() -> None: + attrs = copy.deepcopy(MULTITSFRAME_METADATA_EXAMPLE) + attrs["dataset_metadata"] |= {"ref_dataset_discrete": False, "default_timezone": None} + attrs["metrics"][0]["display_name"] = None + attrs["metrics"][0]["value_dimensions"][0]["unit"] = None + + dumped = MultiTSFrameMetadata.model_validate(attrs).model_dump(mode="json") + + assert "ref_dataset_discrete" not in dumped["dataset_metadata"] + assert "default_timezone" not in dumped["dataset_metadata"] + assert "display_name" not in dumped["metrics"][0] + # required, but nullable + assert dumped["metrics"][0]["value_dimensions"][0]["unit"] is None + + +def test_dump_keeps_extra_fields_with_none() -> None: + meta = multitsframe_metadata(anomaly_score=None) + + assert meta.model_dump(mode="json")["metrics"][0]["anomaly_score"] is None + + +def test_lists_become_tuples() -> None: + meta = multitsframe_metadata(path=["site", "area"]) + + assert isinstance(meta.metrics, tuple) + assert isinstance(meta.metrics[0].value_dimensions, tuple) + assert meta.metrics[0].path == ("site", "area") + + +def test_fields_cannot_be_changed() -> None: + meta = multitsframe_metadata() + + with pytest.raises(ValidationError, match="frozen"): + meta.metrics[0].name = "other" # type: ignore[misc] + + +def test_extra_values_are_frozen() -> None: + meta = multitsframe_metadata( + inherited={"site": "Site A", "asset": {"type": "pump", "tags": ["a", "b"]}}, + anomaly_scores=[{"score": 0.5}], + ) + inherited = meta.metrics[0].inherited + assert inherited is not None + asset = inherited.asset # type: ignore[attr-defined] + + assert isinstance(asset, FrozenDict) + assert isinstance(asset, Mapping) + assert asset["tags"] == ("a", "b") + assert meta.metrics[0].anomaly_scores == (FrozenDict(score=0.5),) # type: ignore[attr-defined] + with pytest.raises(TypeError, match="immutable"): + asset["type"] = "valve" + with pytest.raises(TypeError, match="immutable"): + asset.update(type="valve") + + +@pytest.mark.parametrize("value", [True, 1, 1.5, "text", None]) +def test_scalar_extra_values_are_kept(value: Any) -> None: + meta = multitsframe_metadata(extra_value=value) + + assert meta.metrics[0].extra_value == value # type: ignore[attr-defined] + assert type(meta.metrics[0].extra_value) is type(value) # type: ignore[attr-defined] + + +@pytest.mark.parametrize( + ("field", "value"), [("decimals", 1.0), ("decimals", "1"), ("deadband", "0.5")] +) +def test_numbers_are_not_coerced(field: str, value: Any) -> None: + attrs = copy.deepcopy(MULTITSFRAME_METADATA_EXAMPLE) + attrs["metrics"][0]["value_dimensions"][0][field] = value + + with pytest.raises(ValidationError): + MultiTSFrameMetadata.model_validate(attrs) + + +def test_booleans_are_not_coerced() -> None: + attrs = copy.deepcopy(MULTITSFRAME_METADATA_EXAMPLE) + attrs["dataset_metadata"]["only_invalidate"] = "true" + + with pytest.raises(ValidationError): + MultiTSFrameMetadata.model_validate(attrs) + + +def test_copies_are_the_original() -> None: + meta = multitsframe_metadata(inherited={"asset": {"type": "pump"}}) + + assert copy.deepcopy(meta) is meta + assert copy.deepcopy(meta.metrics[0]) is meta.metrics[0] + assert copy.deepcopy(meta.metrics[0].inherited.asset) is meta.metrics[0].inherited.asset # type: ignore[union-attr] + + +def test_model_copy_never_changes_the_original() -> None: + meta = multitsframe_metadata() + + changed = meta.model_copy(update={"metrics": meta.metrics[:1]}, deep=True) + + assert len(meta.metrics) == 2 + assert len(changed.metrics) == 1 + assert changed.metrics[0] is meta.metrics[0] + + +def test_pickle_round_trip() -> None: + meta = multitsframe_metadata(inherited={"asset": {"tags": ["a"]}}) + + unpickled = pickle.loads(pickle.dumps(meta)) # noqa: S301 + + assert unpickled == meta + assert isinstance(unpickled.metrics[0].inherited.asset, FrozenDict) + + +def test_pandas_operations_share_the_model() -> None: + meta = multitsframe_metadata() + df = pd.DataFrame( + { + "timestamp": pd.date_range("2026-01-01", periods=4, freq="h", tz="UTC"), + "metric": ["plant_a.pressure", "plant_a.temperature"] * 2, + "value": np.arange(4.0), + } + ) + df.attrs = {"hd_metadata": meta} + + derived = [ + df[df["value"] > 0], + df.sort_values("value"), + df.reset_index(drop=True), + df["value"], + df.copy(deep=False), + df.copy(), + pd.concat([df, df.copy()]), + ] + + assert all(obj.attrs["hd_metadata"] is meta for obj in derived) + assert all(obj.attrs is not df.attrs for obj in derived) diff --git a/tests/metadata/test_conventions_rules.py b/tests/metadata/test_conventions_rules.py new file mode 100644 index 0000000..99dd4f0 --- /dev/null +++ b/tests/metadata/test_conventions_rules.py @@ -0,0 +1,233 @@ +"""The models reject everything check_metadata_conventions reports as error""" + +import copy +import datetime +from collections.abc import Callable +from typing import Any + +import pandas as pd +import pytest +from pydantic import ValidationError + +from hdhelpers.metadata.conventions import ( + METADATA_MODELS, + MULTITSFRAME_METADATA_EXAMPLE, + SERIES_METADATA_EXAMPLE, + SINGLETSFRAME_METADATA_EXAMPLE, + MetadataIssueSeverity, + MetricMetadata, + MultiTSFrameDatasetMetadata, + MultiTSFrameMetadata, + ValueDimensionMetadata, + check_metadata_conventions, +) + +EXAMPLES = { + "SERIES": SERIES_METADATA_EXAMPLE, + "SINGLETSFRAME": SINGLETSFRAME_METADATA_EXAMPLE, + "MULTITSFRAME": MULTITSFRAME_METADATA_EXAMPLE, +} + + +def changed(data_type: str, change: Callable[[dict[str, Any]], Any]) -> dict[str, Any]: + attrs = copy.deepcopy(EXAMPLES[data_type]) + change(attrs) + return attrs + + +def first_metric(attrs: dict[str, Any]) -> dict[str, Any]: + return attrs["metrics"][0] + + +# (data type, change of its example, location of the error) +RULE_VIOLATIONS = [ + ( + "MULTITSFRAME", + lambda a: first_metric(a).pop("external_id"), + "metrics[0]", + ), + ( + "MULTITSFRAME", + lambda a: a["metrics"][1].update(external_id=first_metric(a)["external_id"]), + "metrics", + ), + ( + "MULTITSFRAME", + lambda a: first_metric(a)["value_dimensions"].append( + first_metric(a)["value_dimensions"][0] + ), + "metrics[0].value_dimensions", + ), + ("MULTITSFRAME", lambda a: first_metric(a).update(data_frequency="often"), None), + ("MULTITSFRAME", lambda a: first_metric(a).update(data_frequency="-1h"), None), + ("MULTITSFRAME", lambda a: first_metric(a).update(data_frequency_offset="5min"), None), + ( + "MULTITSFRAME", + lambda a: first_metric(a).update(data_frequency="1h", data_frequency_offset="2h"), + None, + ), + ("MULTITSFRAME", lambda a: first_metric(a).update(default_timezone="Mars/Base"), None), + ("MULTITSFRAME", lambda a: a["dataset_metadata"].update(default_timezone="CEST"), None), + ( + "MULTITSFRAME", + lambda a: first_metric(a).update(recording_mode="periodic"), + "metrics[0].recording_mode", + ), + ( + "MULTITSFRAME", + lambda a: first_metric(a).update( + comments=[{"start_date": "2026-01-02T00:00:00Z", "end_date": "2026-01-01T00:00:00Z"}] + ), + None, + ), + ( + "MULTITSFRAME", + lambda a: a["dataset_metadata"].update(only_invalidate=True, invalidate_dataset=False), + "dataset_metadata.only_invalidate", + ), + ( + "MULTITSFRAME", + lambda a: a["dataset_metadata"].pop("ref_interval_start_timestamp"), + None, + ), + ( + "MULTITSFRAME", + lambda a: a["dataset_metadata"].update( + ref_interval_start_timestamp="2026-01-03T00:00:00+00:00" + ), + None, + ), + ( + "MULTITSFRAME", + lambda a: a["dataset_metadata"].update(default_aggregation_interval="sometimes"), + None, + ), + ( + "SINGLETSFRAME", + lambda a: a["dataset_metadata"].update(single_metric="another metric"), + "dataset_metadata.single_metric", + ), + ( + "SERIES", + lambda a: first_metric(a)["value_dimensions"][0].update(column="other"), + "metrics[0].value_dimensions", + ), +] + + +@pytest.mark.parametrize(("data_type", "change", "location"), RULE_VIOLATIONS) +def test_models_reject_rule_violations( + data_type: str, change: Callable[[dict[str, Any]], Any], location: str | None +) -> None: + attrs = changed(data_type, change) + errors = [ + issue + for issue in check_metadata_conventions(attrs, data_type) # type: ignore[arg-type] + if issue.severity == MetadataIssueSeverity.ERROR + ] + assert len(errors) > 0 + if location is not None: + assert location in [error.location for error in errors] + + with pytest.raises(ValidationError, match="Violates the metadata conventions") as raised: + METADATA_MODELS[data_type].model_validate(attrs) + assert errors[0].location in str(raised.value) + + +def test_warnings_do_not_make_metadata_invalid() -> None: + attrs = changed( + "MULTITSFRAME", + lambda a: first_metric(a).update(channel_id="deprecated", unknown_field=1), + ) + issues = check_metadata_conventions(attrs, "MULTITSFRAME") + assert len(issues) > 0 + assert all(issue.severity == MetadataIssueSeverity.WARNING for issue in issues) + + MultiTSFrameMetadata.model_validate(attrs) + + +def nested_models(default_timezone: str) -> MultiTSFrameMetadata: + return MultiTSFrameMetadata( + dataset_metadata=MultiTSFrameDatasetMetadata.model_validate( + MULTITSFRAME_METADATA_EXAMPLE["dataset_metadata"] + ), + metrics=( + MetricMetadata( + id="m1", + external_id="plant_a.pressure", + name="Pressure", + default_timezone=default_timezone, + value_dimensions=( + ValueDimensionMetadata(column="value", unit="bar", measurement=None), + ), + ), + ), + ) + + +def test_rules_apply_to_nested_models() -> None: + assert nested_models("UTC").metrics[0].default_timezone == "UTC" + with pytest.raises(ValidationError, match=r"metrics\[0\].default_timezone"): + nested_models("Mars/Base") + + +@pytest.mark.parametrize( + "value", + [ + datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone(datetime.timedelta(0))), + pd.Timestamp("2026-01-01", tz="UTC"), + ], +) +def test_timestamps_from_utc_datetimes(value: datetime.datetime) -> None: + attrs = changed( + "MULTITSFRAME", lambda a: a["dataset_metadata"].update(ref_interval_start_timestamp=value) + ) + + meta = MultiTSFrameMetadata.model_validate(attrs) + + assert meta.dataset_metadata.ref_interval_start_timestamp == "2026-01-01T00:00:00+00:00" + + +@pytest.mark.parametrize( + ("value", "message_part"), + [ + (datetime.datetime(2026, 1, 1), "naive"), # noqa: DTZ001 + (pd.Timestamp("2026-01-01T01:00", tz="Europe/Berlin"), "not in UTC"), + ], +) +def test_timestamps_from_other_datetimes_are_rejected(value: Any, message_part: str) -> None: + attrs = changed( + "MULTITSFRAME", lambda a: a["dataset_metadata"].update(ref_interval_start_timestamp=value) + ) + + with pytest.raises(ValidationError, match=message_part): + MultiTSFrameMetadata.model_validate(attrs) + + +@pytest.mark.parametrize( + "extra_value", + [ + datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + float("nan"), + float("inf"), + [1, float("nan")], + {1: "integer key"}, + {"nested": {"value": object()}}, + ], +) +def test_extra_values_must_be_json(extra_value: Any) -> None: + attrs = changed("MULTITSFRAME", lambda a: first_metric(a).update(extra=extra_value)) + + with pytest.raises(ValidationError): + MultiTSFrameMetadata.model_validate(attrs) + + +def test_deadband_must_be_finite() -> None: + attrs = changed( + "MULTITSFRAME", + lambda a: first_metric(a)["value_dimensions"][0].update(deadband=float("inf")), + ) + + with pytest.raises(ValidationError, match="deadband"): + MultiTSFrameMetadata.model_validate(attrs) diff --git a/tests/metadata/test_conventions_schema.py b/tests/metadata/test_conventions_schema.py new file mode 100644 index 0000000..8263a9a --- /dev/null +++ b/tests/metadata/test_conventions_schema.py @@ -0,0 +1,35 @@ +"""Snapshot of the JSON Schemas of the metadata conventions + +The hetida designer publishes these schemas in its openapi.json, so a change of the snapshot is a +change of the contract. To accept an intended change, run the tests with +UPDATE_SCHEMA_SNAPSHOT=1 and review the diff of the snapshot file. +""" + +import json +import os +from pathlib import Path +from typing import Any + +from pydantic.json_schema import models_json_schema + +from hdhelpers.metadata.conventions import METADATA_MODELS + +SNAPSHOT = Path(__file__).parent.parent / "data" / "convention_schemas.json" + + +def convention_schemas() -> dict[str, Any]: + _, schemas = models_json_schema( + [(model, "validation") for model in METADATA_MODELS.values()], + ref_template="#/$defs/{model}", + ) + return schemas + + +def test_json_schemas_match_snapshot() -> None: + schemas = convention_schemas() + if os.environ.get("UPDATE_SCHEMA_SNAPSHOT"): + SNAPSHOT.write_text( + json.dumps(schemas, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + ) + + assert schemas == json.loads(SNAPSHOT.read_text()) diff --git a/tests/metadata/test_helpers.py b/tests/metadata/test_helpers.py new file mode 100644 index 0000000..f955795 --- /dev/null +++ b/tests/metadata/test_helpers.py @@ -0,0 +1,476 @@ +"""The get_* functions, thin wrappers over the reading methods of the models""" + +import datetime +from copy import deepcopy +from typing import Any + +import numpy as np +import pandas as pd +import pytest + +from hdhelpers.metadata import ( + Recording, + attach_metadata, + get_default_aggregation_methods, + get_default_timezones, + get_display_names, + get_measurements, + get_metric_info, + get_names, + get_queried_interval, + get_recordings, + get_series_default_aggregation_method, + get_series_default_timezone, + get_series_display_name, + get_series_info, + get_series_measurement, + get_series_name, + get_series_recording, + get_series_short_display_name, + get_series_unit, + get_short_display_names, + get_singlets_default_aggregation_method, + get_singlets_default_timezone, + get_singlets_display_names, + get_singlets_info, + get_singlets_metric_info, + get_singlets_recording, + get_singlets_short_display_names, + get_singlets_units, + get_units, +) + +SIGNAL_ID = "7ab8c419-8745-4714-b588-57c0c772cc95" +REF_INTERVAL = { + "ref_interval_start_timestamp": "2026-10-18T00:00:00Z", + "ref_interval_end_timestamp": "2026-10-19T00:00:00Z", + "ref_interval_type": "closed", +} + + +def value_dimension(**fields: Any) -> dict[str, Any]: + """Value dimension "value" as the hetida platform sends it, with nulls for missing fields""" + return { + "column": "value", + "name": None, + "display_name": None, + "short_display_name": None, + "description": None, + "unit": None, + "measurement": None, + "value_data_type": "float", + } | fields + + +def metric(**fields: Any) -> dict[str, Any]: + """Metric as the hetida platform sends it, with nulls for missing fields""" + return { + "id": SIGNAL_ID, + "external_id": None, + "name": "signal", + "display_name": None, + "description": None, + "inherited": {}, + "comments": [], + "value_dimensions": [value_dimension()], + "default_aggregation_method": None, + } | fields + + +def dataset_metadata(**fields: Any) -> dict[str, Any]: + return {"metric_key": "id", "queried_metrics": []} | REF_INTERVAL | fields + + +# what the hetida platform sends for a timeseries of a SERIES read +platform_series_attrs = { + "dataset_metadata": dataset_metadata(single_metric=SIGNAL_ID, queried_metrics=[SIGNAL_ID]), + "metrics": [ + metric( + external_id="water.north.level", + name="Water-lvl-north", + value_dimensions=[ + value_dimension(name="Water-lvl-north", unit="m", measurement="level") + ], + default_aggregation_method="AVG", + ) + ], +} + +# what the hetida platform sends for a MULTITSFRAME read of a signal group: display_name of +# a metric is the display name configured for the member of the group +platform_multitsframe_attrs = { + "dataset_metadata": dataset_metadata(queried_metrics=["level", "flow"]), + "metrics": [ + metric( + id="level", + name="Water-lvl-north", + value_dimensions=[value_dimension(name="Water-lvl-north", unit="m")], + ), + metric( + id="flow", + name="Pump-flow", + display_name="Flow pump 1", + value_dimensions=[value_dimension(name="Pump-flow")], + ), + ], +} + +singletsframe_attrs = { + "dataset_metadata": dataset_metadata( + metric_key="external_id", single_metric="abc.temp", queried_metrics=["abc.temp"] + ), + "metrics": [ + metric( + external_id="abc.temp", + name="ABC temperature", + display_name="ABC temp.", + value_dimensions=[ + value_dimension(name="temperature", unit="°C"), + value_dimension(column="state", name="measurement state"), + ], + ) + ], +} + + +def series(attrs: dict[str, Any]) -> pd.Series: + ser = pd.Series([1.0], index=pd.to_datetime(["2026-10-18T06:00:00Z"])) + ser.attrs = deepcopy(attrs) + return ser + + +def frame(attrs: dict[str, Any]) -> pd.DataFrame: + df = pd.DataFrame() + df.attrs = deepcopy(attrs) + return df + + +def test_series_from_platform() -> None: + platform_series = series(platform_series_attrs) + + assert get_series_unit(platform_series) == "m" + assert get_series_measurement(platform_series) == "level" + assert get_series_name(platform_series) == "Water-lvl-north" + assert get_series_display_name(platform_series) == "Water-lvl-north" + assert get_series_short_display_name(platform_series) == "Water-lvl-north" + assert get_series_info(platform_series, "value_data_type") == "float" + + +def test_multitsframe_from_platform() -> None: + multitsframe = frame(platform_multitsframe_attrs) + + assert get_units(multitsframe)["level"]["value"] == "m" + assert get_units(multitsframe)["flow"]["value"] is None + assert get_names(multitsframe)["flow"]["value"] == "Pump-flow" + assert get_measurements(multitsframe)["level"]["value"] is None + # a null display name falls back to the name + assert get_display_names(multitsframe)["level"]["value"] == "Water-lvl-north" + assert get_short_display_names(multitsframe)["level"]["value"] == "Water-lvl-north" + # the display name of the metric wins over the name of its value dimension + assert get_display_names(multitsframe)["flow"]["value"] == "Flow pump 1" + assert get_short_display_names(multitsframe)["flow"]["value"] == "Flow pump 1" + assert get_metric_info(multitsframe, "name")["flow"] == "Pump-flow" + + +def test_each_field_is_looked_up_on_all_levels_before_the_next_field() -> None: + attrs = { + "dataset_metadata": dataset_metadata(), + "metrics": [ + metric( + id="a", + name="metric name", + value_dimensions=[ + value_dimension(name="value name"), + value_dimension(column="state", name="state name"), + ], + ), + metric(id="b", name="metric name", display_name="metric display name"), + metric( + id="c", + display_name="metric display name", + value_dimensions=[value_dimension(display_name="value display name")], + ), + ], + "value_dimensions_shared": [ + {"column": "state", "display_name": "shared display name"}, + ], + } + + display_names = get_display_names(frame(attrs)) + + assert display_names["a"]["value"] == "value name" + assert display_names["a"]["state"] == "shared display name" + assert display_names["b"]["value"] == "metric display name" + assert display_names["c"]["value"] == "value display name" + + +def test_singletsframe() -> None: + singletsframe = frame(singletsframe_attrs) + + assert get_singlets_units(singletsframe)["value"] == "°C" + # only the value dimension "value" falls back to the metric + assert get_singlets_display_names(singletsframe)["value"] == "ABC temp." + assert get_singlets_display_names(singletsframe)["state"] == "measurement state" + assert get_singlets_short_display_names(singletsframe)["state"] == "measurement state" + assert get_singlets_info(singletsframe, "name")["state"] == "measurement state" + assert get_singlets_info(singletsframe, "name")["unknown column"] is None + assert get_singlets_metric_info(singletsframe, "name") == "ABC temperature" + + +def test_series_display_name_of_metric_wins_over_value_dimension_name() -> None: + attrs = deepcopy(platform_series_attrs) + attrs["metrics"][0]["display_name"] = "Level north" + attrs["metrics"][0]["short_display_name"] = "north" + + assert get_series_display_name(series(attrs)) == "Level north" + assert get_series_short_display_name(series(attrs)) == "north" + + +def test_series_display_name_does_not_fall_back_to_short_display_name() -> None: + attrs = deepcopy(platform_series_attrs) + attrs["metrics"][0]["value_dimensions"][0]["name"] = None + attrs["metrics"][0]["short_display_name"] = "north" + + assert get_series_display_name(series(attrs)) == "Water-lvl-north" + assert get_series_short_display_name(series(attrs)) == "north" + + +def test_metadata_not_following_the_conventions_gives_no_information() -> None: + attrs = deepcopy(platform_series_attrs) + # the single metric is another one than the metric present + attrs["dataset_metadata"]["single_metric"] = "another signal" + + assert get_series_unit(series(attrs)) is None + assert get_singlets_units(frame(attrs))["value"] is None + assert get_singlets_metric_info(frame(attrs), "name") is None + assert get_units(frame(attrs))[SIGNAL_ID]["value"] is None + + +@pytest.mark.parametrize( + "attrs", [{}, {"dataset_metadata": {"metric_key": "id"}}, {"metrics": None}] +) +def test_without_metadata(attrs: dict[str, Any]) -> None: + """Components should not require metadata, so missing metadata must not raise""" + assert get_metric_info(frame(attrs), "name")["any metric"] is None + assert get_series_unit(series(attrs)) is None + assert get_singlets_units(frame(attrs))["value"] is None + assert get_default_timezones(frame(attrs))["any metric"] is None + assert get_recordings(frame(attrs))["any metric"] == Recording() + assert get_series_default_aggregation_method(series(attrs)) == "MEAN" + assert get_queried_interval(series(attrs)) == (None, None) + + +def multitsframe_with_data(attrs: dict[str, Any]) -> pd.DataFrame: + df = pd.DataFrame( + { + "timestamp": pd.date_range("2026-10-18T06:00", periods=2, freq="h", tz="UTC"), + "metric": ["level", "flow"], + "value": np.arange(2.0), + } + ) + df.attrs = deepcopy(attrs) + return df + + +def test_attached_metadata_of_another_data_type() -> None: + """E.g. a Series taken from a MultiTSFrame, whose metadata it carries""" + attrs = deepcopy(platform_multitsframe_attrs) + attrs["dataset_metadata"]["default_timezone"] = "UTC" + attrs["metrics"][0]["default_timezone"] = "Asia/Tokyo" + multitsframe = multitsframe_with_data(attrs) + attach_metadata(multitsframe) + values = multitsframe["value"] + + # several metrics, none of which is the single one + assert get_series_name(values) is None + assert get_series_default_timezone(values) == "UTC" + + one_metric = multitsframe[multitsframe["metric"] == "level"] + assert get_units(one_metric)["level"]["value"] == "m" + + +def test_attached_singletsframe_metadata_on_a_series() -> None: + singletsframe = pd.DataFrame( + { + "timestamp": pd.date_range("2026-10-18T06:00", periods=2, freq="h", tz="UTC"), + "value": [1.0, 2.0], + "state": ["ok", "ok"], + } + ) + singletsframe.attrs = deepcopy(singletsframe_attrs) + attach_metadata(singletsframe) + + assert get_series_unit(singletsframe["value"]) == "°C" + assert get_series_display_name(singletsframe["value"]) == "ABC temp." + + +@pytest.mark.parametrize("include_inherited", [False, True]) +def test_short_display_names_of_metrics_resolved_by_different_fields( + include_inherited: bool, +) -> None: + """Each metric gets its information from the first field providing it, independently""" + attrs = { + "dataset_metadata": dataset_metadata(), + "metrics": [ + metric( + id="a", + short_display_name="a short", + value_dimensions=[value_dimension(), value_dimension(column="state")], + ), + metric(id="b", display_name="b display", name="b name"), + metric(id="c", name="c name"), + ], + "value_dimensions_shared": [{"column": "state", "name": "shared state"}], + } + + short_display_names = get_short_display_names(frame(attrs), include_inherited=include_inherited) + + assert short_display_names["a"]["value"] == "a short" + assert short_display_names["a"]["state"] == "shared state" + assert short_display_names["b"]["value"] == "b display" + assert short_display_names["c"]["value"] == "c name" + assert short_display_names["c"]["state"] == "shared state" + + +def test_inherited_properties() -> None: + attrs = deepcopy(platform_multitsframe_attrs) + attrs["metrics"][1]["inherited"] = {"unit": "m³/h", "site": "north"} + + assert get_units(frame(attrs))["flow"]["value"] is None + assert get_units(frame(attrs), include_inherited=True)["flow"]["value"] == "m³/h" + assert get_metric_info(frame(attrs), "site")["flow"] == "north" + assert get_metric_info(frame(attrs), "site", include_inherited=False)["flow"] is None + + +def test_default_timezones_fall_back_to_the_dataset_metadata() -> None: + attrs = { + "dataset_metadata": dataset_metadata( + metric_key="external_id", default_timezone="Europe/Berlin" + ), + "metrics": [ + metric(external_id="new_york", default_timezone="America/New_York"), + metric(id="2", external_id="fixed_offset", default_timezone="+02:00"), + metric(id="3", external_id="without"), + ], + } + + timezones = get_default_timezones(frame(attrs)) + + assert timezones == { + "new_york": "America/New_York", + "fixed_offset": "+02:00", + "without": "Europe/Berlin", + } + assert timezones["metric without metadata"] == "Europe/Berlin" + + +def test_default_timezone_of_single_metric() -> None: + attrs = deepcopy(platform_series_attrs) + attrs["dataset_metadata"]["default_timezone"] = "Europe/Berlin" + + assert get_series_default_timezone(series(attrs)) == "Europe/Berlin" + attrs["metrics"][0]["default_timezone"] = "America/New_York" + assert get_series_default_timezone(series(attrs)) == "America/New_York" + assert get_singlets_default_timezone(frame(attrs)) == "America/New_York" + + +def test_recordings_fall_back_to_the_frequency_of_the_dataset_as_a_whole() -> None: + attrs = { + "dataset_metadata": dataset_metadata( + metric_key="external_id", + ref_data_frequency="1h", + ref_data_frequency_offset="30min", + ref_data_frequency_timezone="Europe/Berlin", + ), + "metrics": [ + metric( + id="1", + external_id="own_frequency", + recording_mode="periodic", + data_frequency="15min", + recording_aggregation_method="MEAN", + ), + metric(id="2", external_id="dataset_frequency", recording_mode="on_change"), + ], + } + dataset_frequency = { + "data_frequency": "1h", + "data_frequency_offset": "30min", + "data_frequency_timezone": "Europe/Berlin", + } + + recordings = get_recordings(frame(attrs)) + + assert recordings == { + "own_frequency": Recording( + recording_mode="periodic", data_frequency="15min", recording_aggregation_method="MEAN" + ), + "dataset_frequency": Recording(recording_mode="on_change", **dataset_frequency), + } + assert recordings["metric without metadata"] == Recording(**dataset_frequency) + + +def test_single_metric_recording() -> None: + attrs = deepcopy(platform_series_attrs) + attrs["metrics"][0] |= {"recording_mode": "periodic", "data_frequency": "15min"} + + expected = Recording(recording_mode="periodic", data_frequency="15min") + assert get_series_recording(series(attrs)) == expected + assert get_singlets_recording(frame(attrs)) == expected + + +def test_default_aggregation_methods() -> None: + attrs = deepcopy(platform_multitsframe_attrs) + attrs["metrics"][0]["default_aggregation_method"] = "SUM" + attrs["metrics"][1]["value_dimensions"][0]["value_data_type"] = "boolean" + + methods = get_default_aggregation_methods(frame(attrs)) + + assert methods == {"level": "SUM", "flow": "LAST"} + assert methods["metric without metadata"] == "MEAN" + assert get_series_default_aggregation_method(series(platform_series_attrs)) == "MEAN" + assert get_singlets_default_aggregation_method(frame(platform_series_attrs)) == "MEAN" + + +def test_queried_interval() -> None: + assert get_queried_interval(series(platform_series_attrs)) == ( + datetime.datetime(2026, 10, 18, tzinfo=datetime.UTC), + datetime.datetime(2026, 10, 19, tzinfo=datetime.UTC), + ) + + +def test_many_metrics() -> None: + attrs = { + "dataset_metadata": dataset_metadata(), + "metrics": [ + metric( + id=f"m{i}", + name=f"Metric {i}", + inherited={"site": "north"}, + value_dimensions=[value_dimension(name=f"Metric {i}", unit="m")], + ) + for i in range(1000) + ], + } + multitsframe = frame(attrs) + + assert get_short_display_names(multitsframe)["m999"]["value"] == "Metric 999" + assert get_units(multitsframe)["m0"]["value"] == "m" + assert get_metric_info(multitsframe, "unit")["m500"] == "m" + assert get_metric_info(multitsframe, "site")["m1"] == "north" + + +@pytest.mark.parametrize( + ("function", "data"), + [ + (get_units, pd.Series(dtype=float)), + (get_default_timezones, pd.Series(dtype=float)), + (get_singlets_units, pd.Series(dtype=float)), + (get_series_unit, pd.DataFrame()), + (get_series_recording, pd.DataFrame()), + (get_queried_interval, {}), + ], +) +def test_wrong_input_types_raise(function: Any, data: Any) -> None: + with pytest.raises(TypeError): + function(data) diff --git a/tests/metadata/test_metadata.py b/tests/metadata/test_metadata.py deleted file mode 100644 index 4e9db3d..0000000 --- a/tests/metadata/test_metadata.py +++ /dev/null @@ -1,625 +0,0 @@ -import datetime - -import pandas as pd -import pytest - -from hdhelpers.metadata import ( - # get_display_names, - # get_measurements, - # get_metric_info, - get_queried_interval, - get_series_display_name, - # get_series_measurement, - get_series_name, - get_series_short_display_name, - get_series_unit, - # get_units, - get_singlets_display_names, - get_singlets_info, - get_singlets_measurements, - get_singlets_metric_info, - get_singlets_names, - get_singlets_short_display_names, - get_singlets_units, -) - - -def test_get_queried_interval(empty_series_with_old_attr_real): - start, end = get_queried_interval(empty_series_with_old_attr_real) - assert start == datetime.datetime(2025, 11, 5, 13, 28, tzinfo=datetime.UTC) - assert end == datetime.datetime(2025, 11, 6, 13, 28, tzinfo=datetime.UTC) - - -def test_doctest_get_queried_interval(): - attr = { - "dataset_metadata": { - "ref_interval_start_timestamp": "2025-11-04T13:28:00Z", - "ref_interval_end_timestamp": "2025-11-07T13:28:00Z", - } - } - series = pd.Series() - series.attrs = attr - start, end = get_queried_interval(series) - - assert start == datetime.datetime(2025, 11, 4, 13, 28, tzinfo=datetime.UTC) - assert end == datetime.datetime(2025, 11, 7, 13, 28, tzinfo=datetime.UTC) - - -def test_get_queried_interval_not_given(): - start, end = get_queried_interval(pd.Series()) - - assert start is None - assert end is None - - -def test_get_series_unit(empty_series_with_old_attr_real): - unit = get_series_unit(empty_series_with_old_attr_real) - assert unit == "m³/s" - - -@pytest.mark.parametrize( - ("attr", "output"), - [ - pytest.param( - {"single_metric_metadata": {"structured_metadata": {"value_dimensions": {}}}}, - None, - id="value not given", - ), - pytest.param( - { - "single_metric_metadata": { - "structured_metadata": {"value_dimensions": {"value": {"unit": None}}} - } - }, - None, - id="unit not given", - ), - pytest.param( - { - "single_metric_metadata": { - "structured_metadata": {"value_dimensions": {"value": {"unit": "m/s"}}} - } - }, - "m/s", - id="given", - ), - ], -) -def test_doctest_get_series_unit(attr, output): - series = pd.Series() - series.attrs = attr - assert get_series_unit(series) == output - - -def test_get_series_unit_no_attr(): - assert get_series_unit(pd.Series()) is None - - -def test_get_series_name(empty_series_with_old_attr_real): - name = get_series_name(empty_series_with_old_attr_real) - assert name == "Wasserstand" - - -@pytest.mark.parametrize( - ("attr", "output"), - [ - pytest.param( - { - "single_metric_metadata": { - "structured_metadata": { - "value_dimensions": {"value": {"name": "value_name_of_series"}} - } - } - }, - "value_name_of_series", - id="value name", - ), - pytest.param( - { - "single_metric_metadata": { - "structured_metadata": {"metric": {"name": "name_of_series"}} - } - }, - "name_of_series", - id="name_of_series", - ), - pytest.param( - { - "single_metric_metadata": { - "structured_metadata": { - "metric": {"name": "name_of_series"}, - "value_dimensions": {"value": {"name": "value_name_of_series"}}, - } - } - }, - "name_of_series", - id="metric before value", - ), - ], -) -def test_doctest_get_series_name(attr, output): - series = pd.Series() - series.attrs = attr - assert get_series_name(series) == output - - -def test_get_series_name_no_attr(): - assert get_series_name(pd.Series()) is None - - -def test_get_display_name_series_name(empty_series_with_old_attr_real): - display_name = get_series_display_name(empty_series_with_old_attr_real) - assert display_name == "Wasserstand" - - -@pytest.mark.parametrize( - ("attr", "output"), - [ - pytest.param( - {"by_metric": {"series": {"metric": {"display_name": "metric_display_name"}}}}, - "metric_display_name", - id="metric_display_name", - ), - pytest.param( - { - "by_metric": { - "series": { - "value_dimensions": {"value": {"display_name": "value_display_name"}} - } - } - }, - "value_display_name", - id="value_display_name", - ), - pytest.param( - { - "by_metric": { - "series": { - "metric": {"display_name": "metric_display_name"}, - "value_dimensions": {"value": {"display_name": "value_display_name"}}, - } - } - }, - "metric_display_name", - id="metric before value", - ), - pytest.param( - { - "by_metric": { - "series": { - "metric": {"name": "metric_name"}, - "value_dimensions": {"value": {"display_name": "value_display_name"}}, - } - } - }, - "metric_name", - id="metric name before value display name", - ), - pytest.param( - { - "by_metric": { - "series": { - "metric": {"name": "metric_name"}, - "value_dimensions": {"value": {"name": "value_name"}}, - } - } - }, - "metric_name", - id="metric name before value name", - ), - pytest.param( - { - "single_metric_metadata": { - "structured_metadata": {"metric": {"display_name": "metric_display_name"}} - } - }, - "metric_display_name", - id="metric_display_name platform", - ), - pytest.param( - { - "single_metric_metadata": { - "structured_metadata": { - "value_dimensions": {"value": {"display_name": "value_display_name"}} - } - } - }, - "value_display_name", - id="value_display_name platform", - ), - pytest.param( - { - "single_metric_metadata": { - "structured_metadata": { - "metric": {"display_name": "metric_display_name"}, - "value_dimensions": {"value": {"display_name": "value_display_name"}}, - } - } - }, - "metric_display_name", - id="metric before value platform", - ), - pytest.param( - { - "single_metric_metadata": { - "structured_metadata": { - "metric": {"name": "metric_display_name"}, - "value_dimensions": {"value": {"display_name": "value_display_name"}}, - } - } - }, - "metric_display_name", - id="metric before value name platform", - ), - pytest.param( - { - "single_metric_metadata": { - "structured_metadata": { - "metric": {"name": "metric_name"}, - "value_dimensions": {"value": {"name": "value_name"}}, - } - } - }, - "metric_name", - id="metric name before value name platform", - ), - ], -) -def test_doctest_get_series_display_name(attr, output): - series = pd.Series() - series.attrs = attr - assert get_series_display_name(series) == output - - -def test_get_series_display_name_no_attr(): - assert get_series_display_name(pd.Series()) is None - - -def test_get_short_display_name_series_name(empty_series_with_old_attr_real): - display_name = get_series_short_display_name(empty_series_with_old_attr_real) - assert display_name == "W" - - -@pytest.mark.parametrize( - ("attr", "output"), - [ - pytest.param( - { - "single_metric_metadata": { - "structured_metadata": { - "metric": {"short_display_name": "short_display_name_of_series"} - } - } - }, - "short_display_name_of_series", - id="metric_short_display_name", - ), - pytest.param( - { - "single_metric_metadata": { - "structured_metadata": { - "value_dimensions": { - "value": {"short_display_name": "value_short_display_name"} - } - } - } - }, - "value_short_display_name", - id="value_short_display_name", - ), - pytest.param( - { - "single_metric_metadata": { - "structured_metadata": { - "metric": {"short_display_name": "metric_short_display_name"}, - "value_dimensions": { - "value": {"short_display_name": "value_short_display_name"} - }, - } - } - }, - "metric_short_display_name", - id="metric before value", - ), - pytest.param( - { - "single_metric_metadata": { - "structured_metadata": { - "metric": {"name": "metric_name"}, - "value_dimensions": {"value": {"display_name": "value_display_name"}}, - } - } - }, - "metric_name", - id="metric display name before value name", - ), - pytest.param( - { - "single_metric_metadata": { - "structured_metadata": { - "metric": {"name": "metric_name"}, - "value_dimensions": {"value": {"name": "value_name"}}, - } - } - }, - "metric_name", - id="metric name before value name", - ), - ], -) -def test_doctest_get_series_short_display_name(attr, output): - series = pd.Series() - series.attrs = attr - assert get_series_short_display_name(series) == output - - -def test_get_names_for_mts_with_metric_info_fallback_option(): - from hdhelpers.metadata import get_display_names, get_names, get_short_display_names - - attr = { - "by_metric": { - "metric1": {"metric": {"name": "name_of_metric1"}}, - "metric2": {"metric": {"name": None}}, - } - } - - attr = { - "by_metric": { - "metric1": {"metric": {"short_display_name": "short_display_name_of_metric1"}}, - "metric2": {"metric": {"name": "name_of_metric2"}}, - "metric3": {}, - } - } - - attr = { - "dataset_metadata": {"metric_key": "column"}, - "metrics": [ - { - "external_id": "column_name", - "value_dimensions": [{"column": "temp", "measurement": "temperature"}], - } - ], - } - - dataframe = pd.DataFrame() - dataframe.attrs = attr - result_name = get_names(dataframe) - result_display_name = get_display_names(dataframe) - result_short_display_name = get_short_display_names(dataframe) - - assert result_name == result_display_name == result_short_display_name - - -def test_get_singlets_info_is_keyed_by_value_dimension_only(empty_singletsframe_with_attr): - """A SingleTSFrame has one metric, so no metric level in the result""" - units = get_singlets_units(empty_singletsframe_with_attr) - - assert units["value"] == "°C" - assert units["state"] == "UNKNOWN" # falls back to value_dimensions_shared - assert units["not-given"] is None - - -def test_get_singlets_names(empty_singletsframe_with_attr): - names = get_singlets_names(empty_singletsframe_with_attr) - - assert names["value"] == "temperature" - assert names["state"] == "measurement state" - assert names["not-given"] is None - - -def test_get_singlets_display_names_fall_back_to_name(empty_singletsframe_with_attr): - empty_singletsframe_with_attr.attrs["metrics"][0]["value_dimensions"][0]["display_name"] = ( - "temp" - ) - - display_names = get_singlets_display_names(empty_singletsframe_with_attr) - - assert display_names["value"] == "temp" - assert display_names["state"] == "measurement state" - - -def test_get_singlets_short_display_names_fall_back(empty_singletsframe_with_attr): - empty_singletsframe_with_attr.attrs["metrics"][0]["value_dimensions"][0][ - "short_display_name" - ] = "T" - empty_singletsframe_with_attr.attrs["metrics"][0]["value_dimensions"][1]["display_name"] = ( - "state" - ) - - short_display_names = get_singlets_short_display_names(empty_singletsframe_with_attr) - - assert short_display_names["value"] == "T" - assert short_display_names["state"] == "state" - - -def test_get_singlets_measurements(empty_singletsframe_with_attr): - empty_singletsframe_with_attr.attrs["metrics"][0]["value_dimensions"][0]["measurement"] = ( - "temperature" - ) - - measurements = get_singlets_measurements(empty_singletsframe_with_attr) - - assert measurements["value"] == "temperature" - assert measurements["state"] is None - - -def test_get_singlets_info_with_arbitrary_spec(empty_singletsframe_with_attr): - assert get_singlets_info(empty_singletsframe_with_attr, "unit")["value"] == "°C" - assert get_singlets_info(empty_singletsframe_with_attr, "not-given")["value"] is None - - -def test_get_singlets_metric_info(empty_singletsframe_with_attr): - assert get_singlets_metric_info(empty_singletsframe_with_attr, "name") == "ABC temperature" - assert get_singlets_metric_info(empty_singletsframe_with_attr, "external_id") == "abc.temp" - assert get_singlets_metric_info(empty_singletsframe_with_attr, "not-given") is None - - -def test_get_singlets_metric_key_defaults_to_id(): - """metric_key is optional per convention and defaults to "id" """ - empty_singletsframe_with_attr = pd.DataFrame() - empty_singletsframe_with_attr.attrs = { - "dataset_metadata": {"single_metric": "abc.temp"}, - "metrics": [{"id": "abc.temp", "value_dimensions": [{"column": "value", "unit": "m"}]}], - } - - assert get_singlets_units(empty_singletsframe_with_attr)["value"] == "m" - assert get_singlets_metric_info(empty_singletsframe_with_attr, "id") == "abc.temp" - - -def test_get_singlets_falls_back_to_only_metric_without_single_metric( - empty_singletsframe_with_attr, -): - """If single_metric is missing but there is exactly one metric, use that one""" - del empty_singletsframe_with_attr.attrs["dataset_metadata"]["single_metric"] - - assert get_singlets_units(empty_singletsframe_with_attr)["value"] == "°C" - assert get_singlets_metric_info(empty_singletsframe_with_attr, "name") == "ABC temperature" - - -def test_get_singlets_without_metadata(): - """Components should not require metadata, so no metadata must not raise""" - empty_singletsframe_with_attr = pd.DataFrame() - - assert get_singlets_units(empty_singletsframe_with_attr)["value"] is None - assert get_singlets_names(empty_singletsframe_with_attr)["value"] is None - assert get_singlets_display_names(empty_singletsframe_with_attr)["value"] is None - assert get_singlets_short_display_names(empty_singletsframe_with_attr)["value"] is None - assert get_singlets_measurements(empty_singletsframe_with_attr)["value"] is None - assert get_singlets_metric_info(empty_singletsframe_with_attr, "name") is None - - -@pytest.mark.parametrize( - "func", - [ - get_singlets_units, - get_singlets_names, - get_singlets_display_names, - get_singlets_short_display_names, - get_singlets_measurements, - ], -) -def test_get_singlets_requires_a_dataframe(func): - with pytest.raises(TypeError): - func(pd.Series()) - - -@pytest.fixture -def attrs_with_inherited() -> dict: - return { - "dataset_metadata": {"metric_key": "id", "single_metric": "sensor"}, - "value_dimensions_shared": [{"column": "value", "unit": "shared unit"}], - "metrics": [ - { - "id": "sensor", - "name": "sensor name", - "Latitude": None, - "Longitude": 7.58, - "inherited": { - "Latitude": 51.43, - "Longitude": 8.28, - "display_name": "plant display name", - "unit": "inherited unit", - }, - "value_dimensions": [{"column": "value"}, {"column": "state"}], - }, - {"id": "other", "value_dimensions": [{"column": "value"}]}, - ], - } - - -def test_get_metric_info_falls_back_to_inherited(attrs_with_inherited): - from hdhelpers.metadata import get_metric_info - - dataframe = pd.DataFrame() - dataframe.attrs = attrs_with_inherited - - assert get_metric_info(dataframe, "Latitude")["sensor"] == 51.43 # own None -> inherited - assert get_metric_info(dataframe, "Longitude")["sensor"] == 7.58 # own beats inherited - assert get_metric_info(dataframe, "Latitude")["other"] is None - assert get_metric_info(dataframe, "Latitude", include_inherited=False)["sensor"] is None - assert get_singlets_metric_info(dataframe, "Latitude") == 51.43 - assert get_singlets_metric_info(dataframe, "Latitude", include_inherited=False) is None - - -def test_get_value_dimension_info_falls_back_to_inherited_for_value_only( - attrs_with_inherited, -): - from hdhelpers.metadata import get_units - - dataframe = pd.DataFrame() - dataframe.attrs = attrs_with_inherited - - # like the metric metadata itself, inherited metadata only applies to the "value" column - assert get_singlets_info(dataframe, "Latitude")["value"] == 51.43 - assert get_singlets_info(dataframe, "Latitude")["state"] is None - assert get_singlets_info(dataframe, "Latitude", include_inherited=False)["value"] is None - - # inherited metadata is the last resort, after the dataset wide value_dimensions_shared - assert get_units(dataframe, include_inherited=True)["sensor"]["value"] == "shared unit" - assert get_units(dataframe, include_inherited=True)["other"]["value"] == "shared unit" - assert get_singlets_units(dataframe, include_inherited=True)["value"] == "shared unit" - - del dataframe.attrs["value_dimensions_shared"] - assert get_units(dataframe, include_inherited=True)["sensor"]["value"] == "inherited unit" - assert get_singlets_units(dataframe, include_inherited=True)["value"] == "inherited unit" - - # the functions for dedicated information do not fall back to inherited by default - assert get_units(dataframe)["sensor"]["value"] is None - assert get_singlets_units(dataframe)["value"] is None - - -def test_get_series_info_falls_back_to_inherited(attrs_with_inherited): - from hdhelpers.metadata import get_series_info - - series = pd.Series() - series.attrs = attrs_with_inherited - - assert get_series_info(series, "Latitude") == 51.43 - assert get_series_info(series, "Longitude") == 7.58 - assert get_series_info(series, "Latitude", include_inherited=False) is None - assert get_series_unit(series) == "shared unit" - assert get_series_unit(series, include_inherited=True) == "shared unit" - - del series.attrs["value_dimensions_shared"] - assert get_series_unit(series) is None - assert get_series_unit(series, include_inherited=True) == "inherited unit" - - -def test_own_metadata_beats_inherited_metadata_in_fallback_chains(attrs_with_inherited): - """The whole spec is applied to the metric itself before trying its inherited metadata""" - series = pd.Series() - series.attrs = attrs_with_inherited - - # own "name" beats inherited "display_name" - assert get_series_display_name(series, include_inherited=True) == "sensor name" - assert get_series_short_display_name(series, include_inherited=True) == "sensor name" - - del series.attrs["metrics"][0]["name"] - assert get_series_display_name(series, include_inherited=True) == "plant display name" - assert get_series_display_name(series) is None - - -def test_get_metric_info_prefers_value_dimension_and_shared_over_inherited( - attrs_with_inherited, -): - """Order of get_metric_info: metric, value dimension "value", shared, inherited""" - from hdhelpers.metadata import get_metric_info, get_units - - dataframe = pd.DataFrame() - dataframe.attrs = attrs_with_inherited - - assert get_metric_info(dataframe, "unit")["sensor"] == "shared unit" - - dataframe.attrs["metrics"][0]["value_dimensions"][0]["unit"] = "m" - assert get_metric_info(dataframe, "unit")["sensor"] == "m" - assert get_units(dataframe)["sensor"]["value"] == "m" - - dataframe.attrs["metrics"][0]["unit"] = "kg" - assert get_metric_info(dataframe, "unit")["sensor"] == "kg" - - del dataframe.attrs["value_dimensions_shared"] - del dataframe.attrs["metrics"][0]["unit"] - dataframe.attrs["metrics"][0]["value_dimensions"][0]["unit"] = None - assert get_metric_info(dataframe, "unit")["sensor"] == "inherited unit" - assert get_metric_info(dataframe, "unit", include_inherited=False)["sensor"] is None - assert get_singlets_metric_info(dataframe, "unit") == "inherited unit" diff --git a/tests/metadata/test_metadata_current_convention.py b/tests/metadata/test_metadata_current_convention.py deleted file mode 100644 index f425d86..0000000 --- a/tests/metadata/test_metadata_current_convention.py +++ /dev/null @@ -1,179 +0,0 @@ -"""Extraction of metadata following the current metadata convention in a single pass""" - -import pandas as pd -import pytest -from glom import Coalesce, Path - -import hdhelpers.metadata._current_convention as _current_convention -from hdhelpers.metadata import ( - get_display_names, - get_metric_info, - get_series_info, - get_short_display_names, - get_singlets_info, - get_singlets_metric_info, - get_singlets_units, - get_units, -) - - -def frame(attrs): - df = pd.DataFrame() - df.attrs = attrs - return df - - -def many_metrics_attrs(number): - return { - "dataset_metadata": {"metric_key": "id"}, - "metrics": [ - { - "id": f"m{i}", - "name": f"Metric {i}", - "display_name": None, - "inherited": {"site": "north"}, - "value_dimensions": [{"column": "value", "name": f"Metric {i}", "unit": "m"}], - } - for i in range(number) - ], - } - - -def test_entries_not_adhering_to_the_conventions_are_skipped(): - attrs = { - "metrics": [ - "no object", - {"name": "without metric key"}, - {"id": None, "name": "null metric key"}, - {"id": 42, "name": "metric key no string"}, - { - "id": "a", - "value_dimensions": [ - {"column": "value", "unit": "m"}, - {"unit": "without column"}, - {"column": None, "unit": "null column"}, - {"column": ["no", "string"], "unit": "column no string"}, - "no object", - ], - }, - {"id": "b", "unit": "m³/h", "value_dimensions": None}, - ], - "value_dimensions_shared": [ - {"column": "state", "unit": "shared unit"}, - {"unit": "without column"}, - "no object", - ], - } - - units = get_units(frame(attrs)) - - assert set(units) == {"a", "b"} - assert units["a"] == {"state": "shared unit", "value": "m"} - assert units["b"] == {"state": "shared unit", "value": "m³/h"} - assert set(get_metric_info(frame(attrs), "unit")) == {"a", "b"} - - -@pytest.mark.parametrize("shared", [None, "no list", {"column": "value", "unit": "m"}]) -def test_value_dimensions_shared_which_is_no_list_counts_as_empty(shared): - attrs = {"metrics": [{"id": "a"}], "value_dimensions_shared": shared} - - assert get_units(frame(attrs)) == {"a": {"value": None}} - - -@pytest.mark.parametrize("metric_key", [None, 42]) -def test_metric_key_which_is_no_string_defaults_to_id(metric_key): - attrs = { - "dataset_metadata": {"metric_key": metric_key, "single_metric": "a"}, - "metrics": [{"id": "a", "external_id": "ext", "unit": "m"}], - } - - assert get_metric_info(frame(attrs), "external_id") == {"a": "ext"} - assert get_singlets_metric_info(frame(attrs), "unit") == "m" - assert get_singlets_units(frame(attrs))["value"] == "m" - - -@pytest.mark.parametrize( - ("spec", "expected"), - [ - ("name", "n"), - ("missing", None), - ("items", None), # glom does not fall back to attributes of dicts - (Coalesce("name", default=None), "n"), - (Coalesce("display_name", "name", default=None), None), # null display_name is present - ("sensor.manufacturer", "Acme"), - ("sensor.missing", None), - ("display_name.missing", None), # path through null, left to glom - ("comments.0.content", "c"), # list index, left to glom - ("operator.name", None), # string paths are split at dots - (Path("operator.name"), "Ruhrverband"), - (("comments", ["content"]), ["c"]), - ], -) -def test_getter_applies_spec_like_glom(spec, expected): - obj = { - "name": "n", - "display_name": None, - "sensor": {"manufacturer": "Acme"}, - "operator.name": "Ruhrverband", - "comments": [{"content": "c"}], - } - - assert _current_convention.getter(spec)(obj) == expected - - -def test_spec_is_only_applied_to_levels_needed(): - attrs = { - "metrics": [ - {"id": "a", "name": "Sensor", "value_dimensions": [{"column": "value", "name": None}]} - ], - "value_dimensions_shared": [{"column": "value", "name": None}], - } - - assert get_metric_info(frame(attrs), ("name", str.upper))["a"] == "SENSOR" - - -def test_unused_inherited_properties_are_not_evaluated(): - attrs = { - "dataset_metadata": {"single_metric": "a"}, - "metrics": [ - { - "id": "a", - "name": "Sensor", - "inherited": {"name": None}, - "value_dimensions": [{"column": "value", "name": "Value"}], - } - ], - } - series = pd.Series([1.0]) - series.attrs = attrs - - assert get_series_info(series, ("name", str.upper)) == "VALUE" - assert get_singlets_info(frame(attrs), ("name", str.upper))["value"] == "VALUE" - - -def test_no_glom_evaluation_per_metric_for_field_names(monkeypatch): - """glom costs a lot of time per call, which must not add up for frames with many metrics""" - calls = [] - original_glom = _current_convention.glom - monkeypatch.setattr( - _current_convention, - "glom", - lambda *args, **kwargs: calls.append(1) or original_glom(*args, **kwargs), - ) - multitsframe = frame(many_metrics_attrs(100)) - - get_units(multitsframe) - get_display_names(multitsframe, include_inherited=True) - get_short_display_names(multitsframe, include_inherited=True) - get_metric_info(multitsframe, "unit") - - assert calls == [] - - -def test_many_metrics(): - multitsframe = frame(many_metrics_attrs(1000)) - - assert get_short_display_names(multitsframe)["m999"]["value"] == "Metric 999" - assert get_units(multitsframe)["m0"]["value"] == "m" - assert get_metric_info(multitsframe, "unit")["m500"] == "m" - assert get_metric_info(multitsframe, "site")["m1"] == "north" diff --git a/tests/metadata/test_metadata_default_timezone.py b/tests/metadata/test_metadata_default_timezone.py deleted file mode 100644 index dc3504c..0000000 --- a/tests/metadata/test_metadata_default_timezone.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Default time zone of the metrics, falling back to the one of the dataset metadata""" - -import pandas as pd -import pytest - -from hdhelpers.helpers import modify_timezone -from hdhelpers.metadata import ( - get_default_timezones, - get_series_default_timezone, - get_singlets_default_timezone, -) - - -def frame(attrs): - df = pd.DataFrame() - df.attrs = attrs - return df - - -def series(attrs, values=(1.0,), timestamps=("2026-03-29T12:00:00Z",)): - ser = pd.Series(list(values), index=pd.to_datetime(list(timestamps))) - ser.attrs = attrs - return ser - - -multitsframe_attrs = { - "dataset_metadata": {"metric_key": "external_id", "default_timezone": "Europe/Berlin"}, - "metrics": [ - {"external_id": "new_york", "default_timezone": "America/New_York"}, - {"external_id": "fixed_offset", "default_timezone": "+02:00"}, - {"external_id": "without"}, - {"external_id": "null", "default_timezone": None}, - {"external_id": "no_string", "default_timezone": 2}, - {"external_id": "empty", "default_timezone": ""}, - ], -} - - -def test_default_timezones_of_metrics_fall_back_to_dataset_metadata(): - timezones = get_default_timezones(frame(multitsframe_attrs)) - - assert timezones == { - "new_york": "America/New_York", - "fixed_offset": "+02:00", - "without": "Europe/Berlin", - "null": "Europe/Berlin", - "no_string": "Europe/Berlin", - "empty": "Europe/Berlin", - } - # the default time zone of the dataset metadata applies to all metrics without their own - assert timezones["metric without metadata"] == "Europe/Berlin" - - -def test_default_timezones_without_dataset_default_timezone(): - attrs = {"metrics": [{"id": "a", "default_timezone": "UTC"}, {"id": "b"}]} - - timezones = get_default_timezones(frame(attrs)) - - assert timezones == {"a": "UTC", "b": None} - assert timezones["metric without metadata"] is None - - -def test_default_timezone_does_not_fall_back_to_other_levels(): - """The conventions specify only the dataset metadata as fallback""" - attrs = { - "dataset_metadata": {"single_metric": "a"}, - "metrics": [ - { - "id": "a", - "inherited": {"default_timezone": "Asia/Tokyo"}, - "value_dimensions": [{"column": "value", "default_timezone": "Asia/Tokyo"}], - } - ], - "value_dimensions_shared": [{"column": "value", "default_timezone": "Asia/Tokyo"}], - } - - assert get_default_timezones(frame(attrs))["a"] is None - assert get_series_default_timezone(series(attrs)) is None - assert get_singlets_default_timezone(frame(attrs)) is None - - -@pytest.mark.parametrize( - ("attrs", "expected"), - [ - ( - { - "dataset_metadata": {"single_metric": "a", "default_timezone": "Europe/Berlin"}, - "metrics": [{"id": "a", "default_timezone": "America/New_York"}], - }, - "America/New_York", - ), - ( - { - "dataset_metadata": {"single_metric": "a", "default_timezone": "Europe/Berlin"}, - "metrics": [{"id": "a", "default_timezone": None}], - }, - "Europe/Berlin", - ), - # without single_metric the only metric is used - ({"metrics": [{"id": "a", "default_timezone": "UTC"}]}, "UTC"), - # but not if it is another one than the single metric - ( - { - "dataset_metadata": {"single_metric": "a", "default_timezone": "Europe/Berlin"}, - "metrics": [{"id": "b", "default_timezone": "America/New_York"}], - }, - "Europe/Berlin", - ), - ( - { - "dataset_metadata": {"single_metric": "a"}, - "metrics": [{"id": "b", "default_timezone": "America/New_York"}], - }, - None, - ), - # a single_metric which is no string counts as missing - ( - { - "dataset_metadata": {"single_metric": ["a"]}, - "metrics": [{"id": "a", "default_timezone": "UTC"}], - }, - "UTC", - ), - # without metric metadata the dataset metadata still applies - ({"dataset_metadata": {"default_timezone": "Europe/Berlin"}}, "Europe/Berlin"), - ( - {"dataset_metadata": {"default_timezone": "Europe/Berlin"}, "metrics": []}, - "Europe/Berlin", - ), - ({}, None), - # older metadata structures do not provide default time zones of the metrics - ({"by_metric": {"series": {"metric": {"default_timezone": "UTC"}}}}, None), - ], -) -def test_default_timezone_of_single_metric(attrs, expected): - assert get_series_default_timezone(series(attrs)) == expected - assert get_singlets_default_timezone(frame(attrs)) == expected - - -def test_wrong_types_raise(): - with pytest.raises(TypeError): - get_default_timezones(series({})) - with pytest.raises(TypeError): - get_series_default_timezone(frame({})) - with pytest.raises(TypeError): - get_singlets_default_timezone(series({})) - - -def test_aggregating_days_in_default_timezone(): - """As in the first steps: days start at midnight of the default time zone, which may - differ from UTC days and changes its UTC offset with daylight saving time""" - attrs = { - "dataset_metadata": {"single_metric": "a"}, - "metrics": [{"id": "a", "default_timezone": "Europe/Berlin"}], - } - # 2026-03-29 is the day daylight saving time starts in Germany, it has 23 hours - hourly = series( - attrs, - values=[1.0] * 26, - timestamps=pd.date_range("2026-03-28T23:00:00Z", periods=26, freq="1h"), - ) - - timezone = get_series_default_timezone(hourly) or "UTC" - daily_sums = modify_timezone(hourly, to_timezone=timezone).resample("1D").sum() - - assert daily_sums.to_dict() == { - pd.Timestamp("2026-03-29", tz="Europe/Berlin"): 23.0, - pd.Timestamp("2026-03-30", tz="Europe/Berlin"): 3.0, - } diff --git a/tests/metadata/test_metadata_fallbacks.py b/tests/metadata/test_metadata_fallbacks.py deleted file mode 100644 index 3455339..0000000 --- a/tests/metadata/test_metadata_fallbacks.py +++ /dev/null @@ -1,341 +0,0 @@ -"""Fallbacks for metadata following the current metadata convention - -Metadata providers like the hetida platform send fields without value explicitly as null. -These count as missing: The accessors fall back to the next level or field as if the field -was not provided at all. -""" - -from copy import deepcopy - -import pandas as pd -import pytest - -from hdhelpers.metadata import ( - get_display_names, - get_measurements, - get_metric_info, - get_names, - get_series_default_timezone, - get_series_display_name, - get_series_info, - get_series_measurement, - get_series_name, - get_series_short_display_name, - get_series_unit, - get_short_display_names, - get_singlets_display_names, - get_singlets_metric_info, - get_singlets_short_display_names, - get_singlets_units, - get_units, -) - -SIGNAL_ID = "7ab8c419-8745-4714-b588-57c0c772cc95" - - -def value_dimension(**fields): - """Value dimension "value" as the hetida platform sends it, with nulls for missing fields""" - return { - "column": "value", - "name": None, - "display_name": None, - "short_display_name": None, - "description": None, - "unit": None, - "measurement": None, - "value_data_type": "float", - } | fields - - -def metric(**fields): - """Metric as the hetida platform sends it, with nulls for missing fields""" - return { - "id": SIGNAL_ID, - "external_id": None, - "name": None, - "display_name": None, - "description": None, - "inherited": {}, - "comments": [], - "value_dimensions": [value_dimension()], - "default_aggregation_method": None, - } | fields - - -# what the hetida platform sends for a timeseries of a SERIES read -platform_series_attrs = { - "dataset_metadata": { - "metric_key": "id", - "single_metric": SIGNAL_ID, - "queried_metrics": [SIGNAL_ID], - "ref_interval_start_timestamp": "2026-10-18T00:00:00Z", - "ref_interval_end_timestamp": "2026-10-19T00:00:00Z", - "ref_interval_type": "closed", - }, - "metrics": [ - metric( - external_id="water.north.level", - name="Water-lvl-north", - value_dimensions=[ - value_dimension(name="Water-lvl-north", unit="m", measurement="level") - ], - default_aggregation_method="AVG", - ) - ], -} - -# what the hetida platform sends for a MULTITSFRAME read of a signal group: display_name of -# a metric is the display name configured for the member of the group -platform_multitsframe_attrs = { - "dataset_metadata": {"metric_key": "id", "queried_metrics": ["level", "flow"]}, - "metrics": [ - metric( - id="level", - name="Water-lvl-north", - value_dimensions=[value_dimension(name="Water-lvl-north", unit="m")], - ), - metric( - id="flow", - name="Pump-flow", - display_name="Flow pump 1", - value_dimensions=[value_dimension(name="Pump-flow")], - ), - ], -} - - -def series(attrs): - ser = pd.Series([1.0], index=pd.to_datetime(["2026-10-18T06:00:00Z"])) - ser.attrs = deepcopy(attrs) - return ser - - -def frame(attrs): - df = pd.DataFrame() - df.attrs = deepcopy(attrs) - return df - - -def test_series_from_platform(): - platform_series = series(platform_series_attrs) - - assert get_series_unit(platform_series) == "m" - assert get_series_measurement(platform_series) == "level" - assert get_series_name(platform_series) == "Water-lvl-north" - assert get_series_display_name(platform_series) == "Water-lvl-north" - assert get_series_short_display_name(platform_series) == "Water-lvl-north" - - -def test_multitsframe_from_platform(): - multitsframe = frame(platform_multitsframe_attrs) - - assert get_units(multitsframe)["level"]["value"] == "m" - assert get_units(multitsframe)["flow"]["value"] is None - assert get_names(multitsframe)["flow"]["value"] == "Pump-flow" - assert get_measurements(multitsframe)["level"]["value"] is None - # a null display name falls back to the name - assert get_display_names(multitsframe)["level"]["value"] == "Water-lvl-north" - assert get_short_display_names(multitsframe)["level"]["value"] == "Water-lvl-north" - # the display name of the metric wins over the name of its value dimension - assert get_display_names(multitsframe)["flow"]["value"] == "Flow pump 1" - assert get_short_display_names(multitsframe)["flow"]["value"] == "Flow pump 1" - - -def test_each_field_is_looked_up_on_all_levels_before_the_next_field(): - attrs = { - "metrics": [ - metric( - id="a", - name="metric name", - value_dimensions=[ - value_dimension(name="value name"), - value_dimension(column="state", name="state name"), - ], - ), - metric(id="b", name="metric name", display_name="metric display name"), - metric( - id="c", - display_name="metric display name", - value_dimensions=[value_dimension(display_name="value display name")], - ), - ], - "value_dimensions_shared": [{"column": "state", "display_name": "shared display name"}], - } - - display_names = get_display_names(frame(attrs)) - - assert display_names["a"]["value"] == "value name" - assert display_names["a"]["state"] == "shared display name" - assert display_names["b"]["value"] == "metric display name" - assert display_names["c"]["value"] == "value display name" - - -def test_singletsframe_display_names(): - attrs = { - "dataset_metadata": {"single_metric": "abc.temp", "metric_key": "external_id"}, - "metrics": [ - metric( - external_id="abc.temp", - name="ABC temperature", - display_name="ABC temp.", - value_dimensions=[ - value_dimension(name="temperature", unit="°C"), - value_dimension(column="state", name="measurement state"), - ], - ) - ], - } - singletsframe = frame(attrs) - - assert get_singlets_units(singletsframe)["value"] == "°C" - # only the value dimension "value" falls back to the metric - assert get_singlets_display_names(singletsframe)["value"] == "ABC temp." - assert get_singlets_display_names(singletsframe)["state"] == "measurement state" - assert get_singlets_short_display_names(singletsframe)["state"] == "measurement state" - - -def test_series_display_name_of_metric_wins_over_value_dimension_name(): - attrs = deepcopy(platform_series_attrs) - attrs["metrics"][0]["display_name"] = "Level north" - attrs["metrics"][0]["short_display_name"] = "north" - - assert get_series_display_name(series(attrs)) == "Level north" - assert get_series_short_display_name(series(attrs)) == "north" - - -def test_series_without_single_metric_uses_its_only_metric(): - attrs = deepcopy(platform_series_attrs) - del attrs["dataset_metadata"]["single_metric"] - - assert get_series_unit(series(attrs)) == "m" - assert get_series_display_name(series(attrs)) == "Water-lvl-north" - - -def test_single_metric_is_not_replaced_by_another_only_metric(): - """If the metadata lacks the single metric, the only metric present is another one""" - attrs = deepcopy(platform_series_attrs) - attrs["dataset_metadata"]["single_metric"] = "another signal" - - assert get_series_unit(series(attrs)) is None - assert get_series_display_name(series(attrs)) is None - assert get_singlets_units(frame(attrs))["value"] is None - assert get_singlets_display_names(frame(attrs))["value"] is None - assert get_singlets_metric_info(frame(attrs), "name") is None - - -@pytest.mark.parametrize("single_metric", [None, 1, [SIGNAL_ID]]) -def test_single_metric_which_is_no_string_counts_as_missing(single_metric): - attrs = deepcopy(platform_series_attrs) - attrs["dataset_metadata"]["single_metric"] = single_metric - - assert get_series_unit(series(attrs)) == "m" - assert get_singlets_units(frame(attrs))["value"] == "m" - assert get_singlets_metric_info(frame(attrs), "name") == "Water-lvl-north" - - -def test_series_with_metadata_of_several_metrics_has_no_metric(): - """E.g. a Series taken from a MultiTSFrame, whose metadata it carries - - A metric named "series" is only the metric of a Series in older metadata structures. - """ - attrs = { - "dataset_metadata": {"default_timezone": "UTC"}, - "metrics": [ - metric(id="series", name="Series", default_timezone="Asia/Tokyo"), - metric(id="other", name="Other"), - ], - } - - assert get_series_name(series(attrs)) is None - assert get_series_default_timezone(series(attrs)) == "UTC" - - -def test_only_the_single_metric_is_resolved(): - """The spec is not applied to other metrics, so failing on their null names does not matter""" - attrs = deepcopy(platform_series_attrs) - attrs["metrics"].append(metric(id="other")) - - assert get_series_info(series(attrs), ("name", str.upper)) == "WATER-LVL-NORTH" - assert get_singlets_metric_info(frame(attrs), ("name", str.upper)) == "WATER-LVL-NORTH" - - -def test_series_display_name_does_not_fall_back_to_short_display_name(): - """Like the display names of MultiTSFrames and SingleTSFrames""" - attrs = deepcopy(platform_series_attrs) - attrs["metrics"][0]["name"] = None - attrs["metrics"][0]["value_dimensions"][0]["name"] = None - attrs["metrics"][0]["short_display_name"] = "north" - - assert get_series_display_name(series(attrs)) is None - assert get_singlets_display_names(frame(attrs))["value"] is None - assert get_series_short_display_name(series(attrs)) == "north" - - -@pytest.mark.parametrize( - "attrs", - [{}, {"dataset_metadata": {"metric_key": "id"}}, {"metrics": None}], -) -def test_metric_info_without_metric_metadata(attrs): - """Components should not require metadata, so missing metric metadata must not raise""" - assert get_metric_info(frame(attrs), "name")["any metric"] is None - - -def test_metric_info_from_platform(): - multitsframe = frame(platform_multitsframe_attrs) - multitsframe.attrs["metrics"][0]["default_aggregation_method"] = "SUM" - - aggregation_methods = get_metric_info(multitsframe, "default_aggregation_method") - - assert aggregation_methods["level"] == "SUM" - assert aggregation_methods["flow"] is None - - -def test_metric_info_falls_back_only_for_metrics_lacking_it(): - """A spec transforming the information is not applied to fallback levels needlessly - - The null name of the value dimension of metric "a" would let str.upper fail. - """ - metric_b = metric(id="b", value_dimensions=[value_dimension(name="value name")]) - del metric_b["name"] - multitsframe = frame({"metrics": [metric(id="a", name="Sensor"), metric_b]}) - - names = get_metric_info(multitsframe, ("name", str.upper)) - - assert names["a"] == "SENSOR" - assert names["b"] == "VALUE NAME" - assert get_singlets_metric_info(frame(platform_series_attrs), ("name", str.upper)) == ( - "WATER-LVL-NORTH" - ) - - -def test_metric_info_raises_errors_of_the_spec(): - """Errors of the spec must not be mistaken for missing metric metadata""" - with pytest.raises(TypeError): - get_metric_info(frame(platform_multitsframe_attrs), ("display_name", str.upper)) - - -@pytest.mark.parametrize("include_inherited", [False, True]) -def test_short_display_names_of_metrics_resolved_by_different_fields(include_inherited): - """Each metric gets its information from the first field providing it, independently""" - attrs = { - "metrics": [ - metric( - id="a", - short_display_name="a short", - value_dimensions=[value_dimension(), value_dimension(column="state")], - ), - metric(id="b", display_name="b display", name="b name"), - metric(id="c", name="c name"), - metric(id="d", inherited={"display_name": "d inherited"}), - ], - "value_dimensions_shared": [{"column": "state", "name": "shared state"}], - } - - short_display_names = get_short_display_names(frame(attrs), include_inherited=include_inherited) - - assert short_display_names["a"]["value"] == "a short" - assert short_display_names["a"]["state"] == "shared state" - assert short_display_names["b"]["value"] == "b display" - assert short_display_names["c"]["value"] == "c name" - assert short_display_names["c"]["state"] == "shared state" - assert short_display_names["d"]["value"] == ("d inherited" if include_inherited else None) diff --git a/tests/metadata/test_metadata_migration.py b/tests/metadata/test_metadata_migration.py deleted file mode 100644 index 0363b50..0000000 --- a/tests/metadata/test_metadata_migration.py +++ /dev/null @@ -1,195 +0,0 @@ -import pandas as pd - -from hdhelpers.metadata import ( - get_display_names, - get_measurements, - get_metric_info, - get_series_display_name, - get_series_measurement, - get_series_name, - get_series_short_display_name, - get_series_unit, - get_units, -) - - -def test_get_units_mts_old_format_by_metric(empty_mts_with_old_attr_real): - ##### NEW ##### - units_by_metric_by_value_dimension = get_units(empty_mts_with_old_attr_real) - - # get values - assert units_by_metric_by_value_dimension["metric1"]["value_dim1"] == "m³/s" - assert units_by_metric_by_value_dimension["metric1"]["value_dim2"] == "m³/s" - assert units_by_metric_by_value_dimension["metric2"]["value_dim1"] == "m³/h" - - # ensure that default dict is used in case of missing entry - assert units_by_metric_by_value_dimension["SOME"]["SOME"] is None # is a default dict - - -def test_get_units_mts_old_format(empty_mts_with_old_attr): - units_by_metric_by_value_dimension = get_units(empty_mts_with_old_attr) - - # get values - assert units_by_metric_by_value_dimension["test_channel"]["value"] == "l/min" - assert units_by_metric_by_value_dimension["some_other_metric"]["value"] == "l" - - # ensure that default dict is used in case of missing entry - assert units_by_metric_by_value_dimension["SOME"]["SOME"] is None # is a default dict - - -def test_get_units_mts_new_format_1(empty_mts_with_attr): - measurements_by_metric_by_value_dimension = get_measurements(empty_mts_with_attr) - - assert measurements_by_metric_by_value_dimension["first"]["temp"] == "temperature" - assert measurements_by_metric_by_value_dimension["first"]["value"] is None - assert measurements_by_metric_by_value_dimension["first"]["pressure"] == "pressure" - assert measurements_by_metric_by_value_dimension["first"]["NOT OCCURING"] is None - assert measurements_by_metric_by_value_dimension["NOT OCCURING"]["value"] is None - - assert measurements_by_metric_by_value_dimension["second"]["temp"] == "temperature" - - assert measurements_by_metric_by_value_dimension["third"]["value"] == "height" - assert measurements_by_metric_by_value_dimension["third"]["pressure"] == "pressure" - - -def test_get_units_mts_new_format_2(empty_mts_with_attr): - units_by_metric_by_value_dimension = get_units(empty_mts_with_attr) - - assert units_by_metric_by_value_dimension["first"]["temp"] == "C" - assert units_by_metric_by_value_dimension["first"]["value"] == "m" - assert units_by_metric_by_value_dimension["first"]["NOT OCCURING"] is None - assert units_by_metric_by_value_dimension["NOT OCCURING"]["value"] is None - - assert units_by_metric_by_value_dimension["second"]["temp"] == "C" - assert units_by_metric_by_value_dimension["second"]["value"] is None - - assert units_by_metric_by_value_dimension["third"]["pressure"] == "bar" - assert units_by_metric_by_value_dimension["third"]["value"] is None - - assert units_by_metric_by_value_dimension["fourth"]["pressure"] == "Pa" - assert units_by_metric_by_value_dimension["fourth"]["value"] == "l" - - assert units_by_metric_by_value_dimension["fifth"]["pressure"] == "bar" - assert units_by_metric_by_value_dimension["fifth"]["value"] == "m^3" - - -def test_get_units_mts_new_format_3(empty_mts_with_attr): - empty_mts_with_attr.attrs["dataset_metadata"]["metric_key"] = "external_id" - units_by_metric_by_value_dimension = get_units(empty_mts_with_attr) - - assert units_by_metric_by_value_dimension["external_first"]["temp"] == "C" - assert units_by_metric_by_value_dimension["external_first"]["value"] == "m" - assert units_by_metric_by_value_dimension["external_first"]["NOT OCCURING"] is None - assert units_by_metric_by_value_dimension["NOT OCCURING"]["value"] is None - - assert units_by_metric_by_value_dimension["external_second"]["temp"] == "C" - assert units_by_metric_by_value_dimension["external_second"]["value"] is None - - assert units_by_metric_by_value_dimension["external_third"]["pressure"] == "bar" - assert units_by_metric_by_value_dimension["external_third"]["value"] is None - - assert units_by_metric_by_value_dimension["external_fourth"]["pressure"] == "Pa" - assert units_by_metric_by_value_dimension["external_fourth"]["value"] == "l" - - assert units_by_metric_by_value_dimension["external_fifth"]["pressure"] == "bar" - assert units_by_metric_by_value_dimension["external_fifth"]["value"] == "m^3" - - -def test_get_multitsframe_display_names_from_metadata_with_value_dimensions(empty_mts_with_attr): - display_names_by_metric_by_value_dimension = get_display_names(empty_mts_with_attr) - - assert display_names_by_metric_by_value_dimension["first"]["temp"] is None - assert display_names_by_metric_by_value_dimension["first"]["value"] == "first display name" - assert ( - display_names_by_metric_by_value_dimension["first"]["pressure"] - == "shared value_dimension pressure name" - ) - - assert display_names_by_metric_by_value_dimension["first"]["NOT OCCURING"] is None - assert display_names_by_metric_by_value_dimension["NOT OCCURING"]["value"] is None - - assert display_names_by_metric_by_value_dimension["second"]["value"] == "second name" - assert display_names_by_metric_by_value_dimension["second"]["temp"] is None - - assert display_names_by_metric_by_value_dimension["third"]["pressure"] == "thirds's pressure" - assert display_names_by_metric_by_value_dimension["third"]["temp"] is None - - -def test_get_metric_info(empty_mts_with_attr): - external_ids_by_metric = get_metric_info(empty_mts_with_attr, "external_id") - - assert external_ids_by_metric["UNKNOWN"] is None - assert external_ids_by_metric["first"] == "external_first" - assert external_ids_by_metric["second"] == "external_second" - - -def test_series_unit(empty_series_with_attr): - assert get_series_unit(empty_series_with_attr) == "m" - - assert get_series_name(empty_series_with_attr) == "first name" - assert get_series_display_name(empty_series_with_attr) == "first name" - assert get_series_short_display_name(empty_series_with_attr) == "first name" - - assert get_series_measurement(empty_series_with_attr) is None - - -def test_series_unit_old(empty_series_with_old_attr): - assert get_series_unit(empty_series_with_old_attr) == "C°" - - assert get_series_name(empty_series_with_old_attr) == "Muster Channel" - assert get_series_display_name(empty_series_with_old_attr) == "Muster Channel" - assert get_series_short_display_name(empty_series_with_old_attr) == "muster" - - assert get_series_measurement(empty_series_with_old_attr) is None - - -def test_metric_key_defaults_to_id(): - """ "metric_key" is optional per the conventions and defaults to "id" - - It therefore must not act as the discriminator between the metadata conventions - - see test_by_metric_convention_stays_reachable_without_metric_key. - """ - mts = pd.DataFrame() - mts.attrs = { - "dataset_metadata": {}, - "metrics": [{"id": "metric1", "value_dimensions": [{"column": "value", "unit": "m³/s"}]}], - } - - assert get_units(mts)["metric1"]["value"] == "m³/s" - assert get_metric_info(mts, "id")["metric1"] == "metric1" - - # ... also with no "dataset_metadata" at all - mts.attrs = { - "metrics": [{"id": "metric1", "value_dimensions": [{"column": "value", "unit": "m³/s"}]}] - } - - assert get_units(mts)["metric1"]["value"] == "m³/s" - - -def test_by_metric_convention_stays_reachable_without_metric_key(): - """The older "by_metric" convention must stay reachable - - "metrics" being a list is what discriminates the current convention from the older ones, - so metadata without a "metrics" list still resolves via "by_metric". - """ - mts = pd.DataFrame() - mts.attrs = { - "dataset_metadata": {}, - "by_metric": {"metric1": {"value_dimensions": {"value": {"unit": "m³/s"}}}}, - } - - assert get_units(mts)["metric1"]["value"] == "m³/s" - - -def test_metrics_list_takes_precedence_over_by_metric(): - """If both are present the current convention wins""" - mts = pd.DataFrame() - mts.attrs = { - "dataset_metadata": {}, - "metrics": [ - {"id": "metric1", "value_dimensions": [{"column": "value", "unit": "FROM_NEW"}]} - ], - "by_metric": {"metric1": {"value_dimensions": {"value": {"unit": "FROM_OLD"}}}}, - } - - assert get_units(mts)["metric1"]["value"] == "FROM_NEW" diff --git a/tests/metadata/test_metadata_recording.py b/tests/metadata/test_metadata_recording.py deleted file mode 100644 index 527bac6..0000000 --- a/tests/metadata/test_metadata_recording.py +++ /dev/null @@ -1,183 +0,0 @@ -"""Recording of the metrics and their default aggregation methods""" - -import pandas as pd -import pytest - -from hdhelpers.metadata import ( - Recording, - get_default_aggregation_methods, - get_recordings, - get_series_default_aggregation_method, - get_series_recording, - get_singlets_default_aggregation_method, - get_singlets_recording, -) - - -def frame(attrs): - df = pd.DataFrame() - df.attrs = attrs - return df - - -def series(attrs): - ser = pd.Series(dtype=float) - ser.attrs = attrs - return ser - - -def value_data_type(data_type): - return [{"column": "value", "unit": None, "measurement": None, "value_data_type": data_type}] - - -multitsframe_attrs = { - "dataset_metadata": { - "metric_key": "external_id", - "ref_data_frequency": "1h", - "ref_data_frequency_offset": "30min", - "ref_data_frequency_timezone": "Europe/Berlin", - }, - "metrics": [ - { - "external_id": "own_frequency", - "recording_mode": "periodic", - "data_frequency": "15min", - "recording_aggregation_method": "MEAN", - }, - # the offset and time zone of the metric only apply together with its frequency - { - "external_id": "without_frequency", - "recording_mode": "on_change", - "data_frequency_offset": "5min", - "data_frequency_timezone": "UTC", - }, - {"external_id": "null_frequency", "data_frequency": None}, - {"external_id": "no_string", "data_frequency": 15, "recording_mode": 1}, - ], -} - -dataset_frequency = { - "data_frequency": "1h", - "data_frequency_offset": "30min", - "data_frequency_timezone": "Europe/Berlin", -} - - -def test_recordings_fall_back_to_the_frequency_of_the_dataset_as_a_whole(): - recordings = get_recordings(frame(multitsframe_attrs)) - - assert recordings == { - "own_frequency": Recording( - recording_mode="periodic", - data_frequency="15min", - recording_aggregation_method="MEAN", - ), - "without_frequency": Recording(recording_mode="on_change", **dataset_frequency), - "null_frequency": Recording(**dataset_frequency), - "no_string": Recording(**dataset_frequency), - } - assert recordings["metric without metadata"] == Recording(**dataset_frequency) - - -@pytest.mark.parametrize( - "attrs", - [ - {}, - {"metrics": "no list"}, - # an older metadata structure, whose metrics are not considered - {"dataset_metadata": {}, "metrics": {"a": {"data_frequency": "1h"}}}, - ], -) -def test_recordings_without_metric_metadata(attrs): - recordings = get_recordings(frame(attrs)) - - assert recordings == {} - assert recordings["a"] == Recording() - - -def test_series_recording(): - attrs = { - "dataset_metadata": {"single_metric": "b", "ref_data_frequency": "1D"}, - "metrics": [{"id": "b", "recording_mode": "periodic", "data_frequency_offset": "1h"}], - } - - assert get_series_recording(series(attrs)) == Recording( - recording_mode="periodic", data_frequency="1D" - ) - - -def test_singlets_recording_falls_back_to_the_dataset_if_the_metric_is_not_found(): - attrs = { - "dataset_metadata": {"single_metric": "missing", "ref_data_frequency": "5min"}, - "metrics": [{"id": "a", "data_frequency": "1h"}, {"id": "b"}], - } - - assert get_singlets_recording(frame(attrs)) == Recording(data_frequency="5min") - - -def test_recording_functions_check_their_input(): - with pytest.raises(TypeError): - get_recordings(series({})) - with pytest.raises(TypeError): - get_series_recording(frame({})) - with pytest.raises(TypeError): - get_singlets_recording(series({})) - - -def test_default_aggregation_methods(): - attrs = { - "dataset_metadata": {"metric_key": "id"}, - "metrics": [ - {"id": "avg", "default_aggregation_method": "AVG"}, - {"id": "sum", "default_aggregation_method": "SUM"}, - {"id": "float", "value_dimensions": value_data_type("float")}, - {"id": "integer", "value_dimensions": value_data_type("integer")}, - {"id": "boolean", "value_dimensions": value_data_type("boolean")}, - {"id": "string", "value_dimensions": value_data_type("string")}, - # the method of the metric takes precedence over the default of its data type - { - "id": "boolean_max", - "default_aggregation_method": "MAX", - "value_dimensions": value_data_type("boolean"), - }, - {"id": "null", "default_aggregation_method": None}, - {"id": "shared_data_type", "value_dimensions": []}, - ], - "value_dimensions_shared": [{"column": "value", "value_data_type": "string"}], - } - - methods = get_default_aggregation_methods(frame(attrs)) - - assert methods == { - "avg": "MEAN", - "sum": "SUM", - "float": "MEAN", - "integer": "MEAN", - "boolean": "LAST", - "string": "LAST", - "boolean_max": "MAX", - # the value data type of the shared value dimension "value" applies - "null": "LAST", - "shared_data_type": "LAST", - } - assert methods["metric without metadata"] == "MEAN" - - -@pytest.mark.parametrize( - ("attrs", "method"), - [ - ({}, "MEAN"), - ({"metrics": {"a": {"default_aggregation_method": "SUM"}}}, "MEAN"), - ( - { - "dataset_metadata": {"single_metric": "a"}, - "metrics": [{"id": "a", "value_dimensions": value_data_type("boolean")}], - }, - "LAST", - ), - ({"metrics": [{"id": "a", "default_aggregation_method": "AVG"}]}, "MEAN"), - ], -) -def test_single_metric_default_aggregation_method(attrs, method): - assert get_series_default_aggregation_method(series(attrs)) == method - assert get_singlets_default_aggregation_method(frame(attrs)) == method diff --git a/tests/metadata/test_reading.py b/tests/metadata/test_reading.py new file mode 100644 index 0000000..1565425 --- /dev/null +++ b/tests/metadata/test_reading.py @@ -0,0 +1,412 @@ +"""Lookups with the fallbacks of the conventions and changed versions of the models""" + +import datetime +from typing import Any + +import pytest +from pydantic import ValidationError + +from hdhelpers.metadata._reading import Recording +from hdhelpers.metadata.conventions import ( + MetadataConventionError, + MultiTSFrameMetadata, + SeriesMetadata, + SingleTSFrameMetadata, +) + + +def value_dimension(column: str = "value", **fields: Any) -> dict[str, Any]: + return {"column": column, "unit": None, "measurement": None} | fields + + +def metric(metric_id: str, **fields: Any) -> dict[str, Any]: + return { + "id": metric_id, + "name": f"name of {metric_id}", + "value_dimensions": [value_dimension()], + } | fields + + +def dataset_metadata(**fields: Any) -> dict[str, Any]: + return { + "metric_key": "id", + "queried_metrics": [], + "ref_interval_start_timestamp": "2026-10-18T00:00:00Z", + "ref_interval_end_timestamp": "2026-10-19T00:00:00+00:00", + "ref_interval_type": "closed", + } | fields + + +def multitsframe(*metrics: dict[str, Any], **fields: Any) -> MultiTSFrameMetadata: + return MultiTSFrameMetadata.model_validate( + {"dataset_metadata": dataset_metadata(), "metrics": list(metrics)} | fields + ) + + +def series(metric_fields: dict[str, Any], **dataset_fields: Any) -> SeriesMetadata: + return SeriesMetadata.model_validate( + { + "dataset_metadata": dataset_metadata( + single_metric="s", queried_metrics=["s"], **dataset_fields + ), + "metrics": [metric("s", **metric_fields)], + } + ) + + +class TestMetrics: + def test_metric_ids_in_order(self) -> None: + meta = multitsframe(metric("b"), metric("a")) + + assert meta.metric_ids() == ["b", "a"] + assert meta.metric("a").name == "name of a" + + def test_metric_key_names_the_id(self) -> None: + meta = MultiTSFrameMetadata.model_validate( + { + "dataset_metadata": dataset_metadata(metric_key="tag"), + "metrics": [metric("m1", tag="plant.temperature")], + } + ) + + assert meta.metric_ids() == ["plant.temperature"] + + def test_unknown_metric(self) -> None: + meta = multitsframe(metric("a")) + + with pytest.raises(KeyError): + meta.metric("b") + assert meta.unit("b") is None + assert meta.metric_info("name", "b") is None + + def test_multitsframe_requires_metric_id(self) -> None: + with pytest.raises(TypeError, match="metric_id"): + multitsframe(metric("a")).unit() + + def test_single_metric_by_default(self) -> None: + meta = series({"value_dimensions": [value_dimension(unit="m")]}) + + assert meta.metric().id == "s" + assert meta.unit() == "m" + assert meta.unit("s") == "m" + + def test_singletsframe_columns(self) -> None: + meta = SingleTSFrameMetadata.model_validate( + { + "dataset_metadata": dataset_metadata(single_metric="f", queried_metrics=["f"]), + "metrics": [ + metric( + "f", + value_dimensions=[ + value_dimension("temperature", unit="°C"), + value_dimension("pressure", unit="bar"), + ], + ) + ], + } + ) + + assert meta.unit(column="pressure") == "bar" + assert meta.units() == {"f": {"value": None, "temperature": "°C", "pressure": "bar"}} + + +class TestValueDimensionInformation: + def test_value_dimension_first(self) -> None: + meta = series({"unit": "metric unit", "value_dimensions": [value_dimension(unit="m")]}) + + assert meta.unit() == "m" + + def test_metric_for_value(self) -> None: + meta = series({"unit": "m"}) + + assert meta.unit() == "m" + + def test_metric_not_for_other_columns(self) -> None: + meta = multitsframe( + metric("a", unit="m", value_dimensions=[value_dimension("other")]), + ) + + assert meta.units() == {"a": {"value": "m", "other": None}} + + def test_shared_value_dimensions_last(self) -> None: + meta = multitsframe( + metric("a", value_dimensions=[value_dimension("temperature", unit="K")]), + metric("b", value_dimensions=[value_dimension("temperature")]), + value_dimensions_shared=[ + value_dimension("temperature", unit="°C"), + value_dimension("pressure", unit="bar"), + ], + ) + + assert meta.units() == { + "a": {"temperature": "K", "pressure": "bar", "value": None}, + "b": {"temperature": "°C", "pressure": "bar", "value": None}, + } + assert meta.unit("b", "temperature") == "°C" + + def test_each_field_on_all_levels_before_the_next(self) -> None: + meta = series({"display_name": "metric display name"}) + + # the display name of the metric wins over the name of the value dimension + assert meta.display_name() == "metric display name" + assert meta.short_display_name() == "metric display name" + assert meta.name() == "name of s" + + def test_display_name_falls_back_to_name(self) -> None: + meta = series({"value_dimensions": [value_dimension(name="level")]}) + + assert meta.display_name() == "level" + assert meta.short_display_name() == "level" + + def test_null_counts_as_missing(self) -> None: + meta = series( + {"display_name": None, "value_dimensions": [value_dimension(display_name=None)]} + ) + + assert meta.display_name() == "name of s" + + def test_inherited_properties_only_on_request(self) -> None: + meta = series({"inherited": {"unit": "m", "measurement": "level"}}) + + assert meta.unit() is None + assert meta.unit(include_inherited=True) == "m" + assert meta.measurements(include_inherited=True) == {"s": {"value": "level"}} + + def test_inherited_properties_only_for_value(self) -> None: + meta = multitsframe( + metric("a", inherited={"unit": "m"}, value_dimensions=[value_dimension("other")]) + ) + + assert meta.units(include_inherited=True) == {"a": {"value": "m", "other": None}} + + def test_extra_fields_and_dotted_paths(self) -> None: + meta = series({"value_dimensions": [value_dimension(sensor={"model": "X1"})]}) + + assert meta.value_dimension_info("sensor.model") == "X1" + assert meta.value_dimension_info("sensor.serial") is None + assert meta.value_dimension_info(["sensor.serial", "sensor.model"]) == "X1" + + +class TestMetricInformation: + def test_fallbacks(self) -> None: + meta = multitsframe( + metric("own", description="own"), + metric("value", value_dimensions=[value_dimension(description="value")]), + metric("shared"), + metric("inherited", inherited={"description": "inherited"}), + value_dimensions_shared=[value_dimension(description="shared")], + ) + + assert meta.metric_infos("description") == { + "own": "own", + "value": "value", + "shared": "shared", + "inherited": "shared", + } + + def test_inherited_properties_by_default(self) -> None: + meta = series({"inherited": {"site": "north"}}) + + assert meta.metric_info("site") == "north" + assert meta.metric_info("site", include_inherited=False) is None + + +class TestTimezonesRecordingAggregation: + def test_default_timezone(self) -> None: + meta = MultiTSFrameMetadata.model_validate( + { + "dataset_metadata": dataset_metadata(default_timezone="UTC"), + "metrics": [metric("a", default_timezone="Europe/Berlin"), metric("b")], + } + ) + + assert meta.default_timezones() == {"a": "Europe/Berlin", "b": "UTC"} + assert meta.default_timezone("unknown") == "UTC" + + def test_recording_falls_back_as_a_whole(self) -> None: + meta = MultiTSFrameMetadata.model_validate( + { + "dataset_metadata": dataset_metadata( + ref_data_frequency="1h", + ref_data_frequency_offset="5min", + ref_data_frequency_timezone="Europe/Berlin", + ), + "metrics": [ + metric( + "own", + recording_mode="periodic", + data_frequency="15min", + recording_aggregation_method="MAX", + ), + metric("dataset", recording_mode="on_change"), + ], + } + ) + + assert meta.recordings() == { + "own": Recording("periodic", "15min", None, None, "MAX"), + "dataset": Recording("on_change", "1h", "5min", "Europe/Berlin", None), + } + assert meta.recording("unknown") == Recording(None, "1h", "5min", "Europe/Berlin", None) + assert type(meta.recording("own").recording_mode) is str + + @pytest.mark.parametrize( + ("metric_fields", "shared", "expected"), + [ + ({"default_aggregation_method": "MAX"}, None, "MAX"), + ({"default_aggregation_method": "AVG"}, None, "MEAN"), + ({"value_dimensions": [value_dimension(value_data_type="boolean")]}, None, "LAST"), + ({"value_data_type": "string"}, None, "LAST"), + ({}, [value_dimension(value_data_type="boolean")], "LAST"), + ({}, None, "MEAN"), + ], + ) + def test_default_aggregation_method( + self, metric_fields: dict[str, Any], shared: list[dict] | None, expected: str + ) -> None: + meta = multitsframe(metric("a", **metric_fields), value_dimensions_shared=shared) + + assert meta.default_aggregation_method("a") == expected + assert meta.default_aggregation_methods() == {"a": expected} + assert meta.default_aggregation_method("unknown") == "MEAN" + + def test_queried_interval(self) -> None: + start, end = series({}).queried_interval() + + assert start == datetime.datetime(2026, 10, 18, tzinfo=datetime.UTC) + assert end == datetime.datetime(2026, 10, 19, tzinfo=datetime.UTC) + + def test_no_queried_interval_for_discrete_datasets(self) -> None: + meta = SeriesMetadata.model_validate( + { + "dataset_metadata": { + "metric_key": "id", + "queried_metrics": ["s"], + "single_metric": "s", + "ref_dataset_discrete": True, + }, + "metrics": [metric("s")], + } + ) + + assert meta.queried_interval() == (None, None) + + +class TestChangedVersions: + def test_with_metric_shares_the_other_metrics(self) -> None: + meta = multitsframe(metric("a"), metric("b")) + + changed = meta.with_metric("a", display_name="A") + + assert changed.display_name("a") == "A" + assert meta.display_name("a") == "name of a" + assert changed.metrics[1] is meta.metrics[1] + assert changed.dataset_metadata is meta.dataset_metadata + + def test_with_metric_validates_the_changes(self) -> None: + meta = multitsframe(metric("a"), metric("b")) + + with pytest.raises(ValidationError): + meta.with_metric("a", name=5) + with pytest.raises(MetadataConventionError, match="default_timezone"): + meta.with_metric("a", default_timezone="Mars/Base") + with pytest.raises(MetadataConventionError, match="more than once"): + meta.with_metric("a", id="b") + + def test_with_metric_changing_the_id(self) -> None: + changed = multitsframe(metric("a"), metric("b")).with_metric("a", id="c") + + assert changed.metric_ids() == ["c", "b"] + + def test_with_value_dimension(self) -> None: + meta = series({"value_dimensions": [value_dimension(unit="m")]}) + + changed = meta.with_value_dimension(None, "value", unit="cm") + + assert changed.unit() == "cm" + assert meta.unit() == "m" + with pytest.raises(KeyError): + meta.with_value_dimension(None, "other", unit="cm") + + def test_with_dataset_metadata_checks_all_rules(self) -> None: + meta = multitsframe(metric("a")) + + changed = meta.with_dataset_metadata(default_timezone="Europe/Berlin") + + assert changed.default_timezone("a") == "Europe/Berlin" + assert changed.metrics is meta.metrics + with pytest.raises(MetadataConventionError, match="ref_interval_start_timestamp"): + meta.with_dataset_metadata(ref_interval_start_timestamp="2026-10-20T00:00:00Z") + + +class TestConversions: + def test_narrowed_to_a_metric(self) -> None: + meta = multitsframe( + metric("a"), metric("b"), value_dimensions_shared=[value_dimension(unit="m")] + ) + + narrowed = meta.narrowed_to("b") + + assert isinstance(narrowed, SingleTSFrameMetadata) + assert narrowed.metrics[0] is meta.metrics[1] + assert narrowed.dataset_metadata.single_metric == "b" + assert narrowed.dataset_metadata.queried_metrics == ("b",) + assert narrowed.unit() == "m" + with pytest.raises(KeyError): + meta.narrowed_to("c") + + def singletsframe(self) -> SingleTSFrameMetadata: + return SingleTSFrameMetadata.model_validate( + { + "dataset_metadata": dataset_metadata(single_metric="f", queried_metrics=["f"]), + "metrics": [ + metric( + "f", + value_dimensions=[ + value_dimension("temperature", unit="°C"), + value_dimension("pressure", unit="bar"), + ], + ) + ], + "value_dimensions_shared": [value_dimension("pressure", measurement="pressure")], + } + ) + + def test_singletsframe_as_series(self) -> None: + series_meta = self.singletsframe().as_series("pressure") + + assert isinstance(series_meta, SeriesMetadata) + assert series_meta.units() == {"f": {"value": "bar"}} + assert series_meta.measurement() == "pressure" + with pytest.raises(KeyError): + self.singletsframe().as_series("humidity") + + def test_series_as_singletsframe(self) -> None: + series_meta = self.singletsframe().as_series("pressure") + + singletsframe_meta = series_meta.as_singletsframe("p") + + assert singletsframe_meta.units() == {"f": {"p": "bar", "value": None}} + assert singletsframe_meta.measurement(column="p") == "pressure" + + def test_singletsframe_as_multitsframe(self) -> None: + meta = self.singletsframe().as_multitsframe() + + assert isinstance(meta, MultiTSFrameMetadata) + assert meta.metric_ids() == ["f"] + assert meta.dataset_metadata.queried_metrics == ("f",) + + def test_singletsframe_as_multitsframe_with_another_id(self) -> None: + meta = self.singletsframe().as_multitsframe("plant.f") + + assert meta.metric_ids() == ["plant.f"] + assert meta.dataset_metadata.metric_key == "metric" + assert meta.metric("plant.f").id == "f" + assert meta.dataset_metadata.single_metric == "plant.f" + + +def test_enum_members_are_given_as_values() -> None: + meta = series({"recording_mode": "irregular", "default_aggregation_method": "MAX"}) + + assert type(meta.metric_info("recording_mode")) is str + assert meta.metric_info("default_aggregation_method") == "MAX" diff --git a/tests/test_optional_plotting.py b/tests/test_optional_plotting.py new file mode 100644 index 0000000..3017c87 --- /dev/null +++ b/tests/test_optional_plotting.py @@ -0,0 +1,40 @@ +import os +import subprocess +import sys + +# None in sys.modules makes every import of plotly fail, as if it was not installed +WITHOUT_PLOTLY = "import sys; sys.modules['plotly'] = None\n" + + +def run_without_plotly(code: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-c", WITHOUT_PLOTLY + code], + capture_output=True, + text=True, + env=os.environ | {"PYTHONPATH": os.pathsep.join(sys.path)}, + check=False, + ) + + +def test_hdhelpers_works_without_plotly() -> None: + result = run_without_plotly( + "import hdhelpers, hdhelpers.metadata\n" + "from hdhelpers import StatusColors, get_locale, modify_timezone\n" + "from hdhelpers.metadata.conventions import check_metadata_conventions\n" + ) + assert result.returncode == 0, result.stderr + + +def test_plotting_names_need_the_plotting_extra() -> None: + result = run_without_plotly("from hdhelpers import set_dt_ticks\n") + + assert result.returncode != 0 + assert "hdhelpers[plotting]" in result.stderr + + +def test_plotting_names_are_available_with_plotly() -> None: + import hdhelpers # noqa: PLC0415 + from hdhelpers.plotting.timestamp_ticks import set_dt_ticks # noqa: PLC0415 + + assert hdhelpers.set_dt_ticks is set_dt_ticks + assert "set_dt_ticks" in dir(hdhelpers) diff --git a/uv.lock b/uv.lock index 815ed9d..1216f54 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'win32'", @@ -31,15 +31,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] -[[package]] -name = "attrs" -version = "26.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, -] - [[package]] name = "babel" version = "2.18.0" @@ -49,15 +40,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, ] -[[package]] -name = "boltons" -version = "25.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/63/54/71a94d8e02da9a865587fb3fff100cb0fc7aa9f4d5ed9ed3a591216ddcc7/boltons-25.0.0.tar.gz", hash = "sha256:e110fbdc30b7b9868cb604e3f71d4722dd8f4dcb4a5ddd06028ba8f1ab0b5ace", size = 246294, upload-time = "2025-02-03T05:57:59.129Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/45/7f/0e961cf3908bc4c1c3e027de2794f867c6c89fb4916fc7dba295a0e80a2d/boltons-25.0.0-py3-none-any.whl", hash = "sha256:dc9fb38bf28985715497d1b54d00b62ea866eca3938938ea9043e254a3a6ca62", size = 194210, upload-time = "2025-02-03T05:57:56.705Z" }, -] - [[package]] name = "certifi" version = "2026.4.22" @@ -242,45 +224,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" }, ] -[[package]] -name = "face" -version = "26.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "boltons" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/24/4e/0e106b0ba486cc38c858fb5efe899002f2ec4765e0808b298d8e19a16efb/face-26.0.0.tar.gz", hash = "sha256:ae12136ff0052f124811f5319670a8d9d29b7d2caaaabe542813690967cc6bca", size = 49862, upload-time = "2026-02-14T00:17:12.576Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/63/1d/c2f7a4334f7501a3474766b5bc0948e8e0b0916217a54d092dd700a5ed3c/face-26.0.0-py3-none-any.whl", hash = "sha256:6ec9cf271d8ee2447f04b14264209a09ec9cbe8252255e61fb7ab6b154e300f9", size = 54825, upload-time = "2026-02-14T00:17:11.519Z" }, -] - -[[package]] -name = "glom" -version = "25.12.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "boltons" }, - { name = "face" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/78/74/8387f95565ba7c30cd152a585b275ebb9a834d1d32782425c5d2fe0a102c/glom-25.12.0.tar.gz", hash = "sha256:1ae7da88be3693df40ad27bdf57a765a55c075c86c971bcddd67927403eb0069", size = 196128, upload-time = "2025-12-29T06:29:07.274Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/e6/4129d9a3baa72d747533bb33376543ccadd9a7f9944e5a6e3ae2e245f5d6/glom-25.12.0-py3-none-any.whl", hash = "sha256:b9f21e77f71a6576a43864e85066b8cc3f0f778d0d50961563f8981377a6dcb1", size = 103295, upload-time = "2025-12-29T06:29:06.074Z" }, -] - [[package]] name = "hdhelpers" version = "0.0.9" source = { editable = "." } dependencies = [ - { name = "glom" }, { name = "numpy" }, { name = "pandas" }, - { name = "plotly" }, { name = "pydantic" }, { name = "pytz" }, ] +[package.optional-dependencies] +plotting = [ + { name = "plotly" }, +] + [package.dev-dependencies] dev = [ { name = "mypy" }, @@ -295,13 +254,13 @@ dev = [ [package.metadata] requires-dist = [ - { name = "glom", specifier = ">25,<26" }, { name = "numpy" }, { name = "pandas", specifier = ">=3" }, - { name = "plotly" }, + { name = "plotly", marker = "extra == 'plotting'" }, { name = "pydantic" }, { name = "pytz" }, ] +provides-extras = ["plotting"] [package.metadata.requires-dev] dev = [ From 76e14174e2ea6b13ad46b067ba8a14bab9454874 Mon Sep 17 00:00:00 2001 From: stewit <> Date: Mon, 26 Oct 2026 21:47:58 +0100 Subject: [PATCH 2/8] fix timezone test --- pyproject.toml | 2 ++ src/hdhelpers/metadata/conventions.py | 13 +++++++++---- uv.lock | 2 ++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 07440df..826bd74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,8 @@ dependencies = [ "pydantic", "numpy", "pytz", + # the time zone names the metadata conventions accept, independent of the system + "tzdata", ] license = "MIT" license-files = ["LICENSE"] diff --git a/src/hdhelpers/metadata/conventions.py b/src/hdhelpers/metadata/conventions.py index d2b6caa..cc9631a 100644 --- a/src/hdhelpers/metadata/conventions.py +++ b/src/hdhelpers/metadata/conventions.py @@ -27,8 +27,8 @@ from collections.abc import Mapping from enum import StrEnum from functools import cache, lru_cache +from importlib import resources from typing import Annotated, Any, ClassVar, Literal, NoReturn, Self -from zoneinfo import available_timezones import pandas as pd from pydantic import ( @@ -1249,9 +1249,14 @@ def _check_frequency_with_alignment( @cache -def _timezone_names() -> set[str]: - # Independent of the system, unlike ZoneInfo, which also accepts e.g. "posixrules" - return available_timezones() +def _timezone_names() -> frozenset[str]: + """The IANA time zone names of the tzdata package, independent of the system + + Not zoneinfo.available_timezones, which also lists the files of the system's time zone + database, e.g. "localtime" on Debian or Ubuntu, which is no time zone name. + """ + with resources.files("tzdata").joinpath("zones").open(encoding="utf-8") as zones: + return frozenset(line.strip() for line in zones if line.strip() != "") def _is_timezone(value: str) -> bool: diff --git a/uv.lock b/uv.lock index 1216f54..7f4a8c2 100644 --- a/uv.lock +++ b/uv.lock @@ -233,6 +233,7 @@ dependencies = [ { name = "pandas" }, { name = "pydantic" }, { name = "pytz" }, + { name = "tzdata" }, ] [package.optional-dependencies] @@ -259,6 +260,7 @@ requires-dist = [ { name = "plotly", marker = "extra == 'plotting'" }, { name = "pydantic" }, { name = "pytz" }, + { name = "tzdata" }, ] provides-extras = ["plotting"] From 3a881e4bd8cf4a1fedf5f6b22b656ac17c45578d Mon Sep 17 00:00:00 2001 From: stewit <> Date: Tue, 27 Oct 2026 09:41:02 +0100 Subject: [PATCH 3/8] fixes and improvements --- src/hdhelpers/metadata/_reading.py | 28 +++++---- src/hdhelpers/metadata/conventions.py | 64 ++++++++++---------- tests/metadata/test_conventions_immutable.py | 21 ++++++- tests/metadata/test_conventions_rules.py | 7 +++ tests/metadata/test_helpers.py | 9 +++ tests/metadata/test_reading.py | 12 ++++ 6 files changed, 96 insertions(+), 45 deletions(-) diff --git a/src/hdhelpers/metadata/_reading.py b/src/hdhelpers/metadata/_reading.py index b2efe24..dcab2fd 100644 --- a/src/hdhelpers/metadata/_reading.py +++ b/src/hdhelpers/metadata/_reading.py @@ -28,6 +28,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from enum import Enum +from functools import cached_property from typing import TYPE_CHECKING, Any, ClassVar, Literal from pydantic import BaseModel @@ -158,13 +159,12 @@ class MetadataReading: metrics: tuple[MetricMetadata, ...] value_dimensions_shared: tuple[SharedValueDimensionMetadata, ...] | None _data_type: ClassVar[Literal["SERIES", "SINGLETSFRAME", "MULTITSFRAME"]] - _by_id: dict[str, MetricMetadata] | None # metrics def metric_ids(self) -> list[str]: """The ids of the metrics, i.e. their values of the field named by the metric key""" - return list(self._metrics_by_id()) + return list(self._metrics_by_id) def metric(self, metric_id: str | None = None) -> "MetricMetadata": """The metadata of a metric, KeyError if there is none""" @@ -173,18 +173,22 @@ def metric(self, metric_id: str | None = None) -> "MetricMetadata": raise KeyError(metric_id) return metric + @cached_property def _metrics_by_id(self) -> "dict[str, MetricMetadata]": - if self._by_id is None: - metric_key = self.dataset_metadata.metric_key - self._by_id = {lookup(metric, metric_key): metric for metric in self.metrics} - return self._by_id + """The metrics by id, built on first use + + Cached in __dict__ apart from the fields, which pydantic ignores when comparing models: + Reading must not change the equality, since pandas keeps only equal attrs, e.g. on concat. + """ + metric_key = self.dataset_metadata.metric_key + return {lookup(metric, metric_key): metric for metric in self.metrics} def _metric_or_none(self, metric_id: str | None) -> "MetricMetadata | None": if metric_id is None: if self._data_type == "MULTITSFRAME": raise TypeError("The metric_id is required for MULTITSFRAME metadata.") return self.metrics[0] - return self._metrics_by_id().get(metric_id) + return self._metrics_by_id.get(metric_id) def _shared_by_column(self) -> "dict[str, SharedValueDimensionMetadata]": return _by_column(self.value_dimensions_shared or ()) @@ -222,7 +226,7 @@ def value_dimension_infos( fields = _fields(fields) shared = self._shared_by_column() result = {} - for metric_id, metric in self._metrics_by_id().items(): + for metric_id, metric in self._metrics_by_id.items(): value_dimensions = _by_column(metric.value_dimensions) columns = dict.fromkeys([*shared, "value", *value_dimensions]) result[metric_id] = { @@ -324,7 +328,7 @@ def metric_infos(self, field: str, *, include_inherited: bool = True) -> dict[st shared = self._shared_by_column() return { metric_id: self._metric_info(field, metric, shared, include_inherited) - for metric_id, metric in self._metrics_by_id().items() + for metric_id, metric in self._metrics_by_id.items() } @staticmethod @@ -352,7 +356,7 @@ def default_timezone(self, metric_id: str | None = None) -> str | None: return own or dataset_default_timezone(self) def default_timezones(self) -> dict[str, str | None]: - return {metric_id: self.default_timezone(metric_id) for metric_id in self._metrics_by_id()} + return {metric_id: self.default_timezone(metric_id) for metric_id in self._metrics_by_id} def recording(self, metric_id: str | None = None) -> Recording: """How the values of the metric are recorded @@ -374,7 +378,7 @@ def recording(self, metric_id: str | None = None) -> Recording: ) def recordings(self) -> dict[str, Recording]: - return {metric_id: self.recording(metric_id) for metric_id in self._metrics_by_id()} + return {metric_id: self.recording(metric_id) for metric_id in self._metrics_by_id} def default_aggregation_method(self, metric_id: str | None = None) -> str: """The default aggregation method of the metric, see the module docstring""" @@ -401,7 +405,7 @@ def default_aggregation_method(self, metric_id: str | None = None) -> str: def default_aggregation_methods(self) -> dict[str, str]: return { metric_id: self.default_aggregation_method(metric_id) - for metric_id in self._metrics_by_id() + for metric_id in self._metrics_by_id } def queried_interval(self) -> tuple[datetime.datetime | None, datetime.datetime | None]: diff --git a/src/hdhelpers/metadata/conventions.py b/src/hdhelpers/metadata/conventions.py index cc9631a..4ff65c4 100644 --- a/src/hdhelpers/metadata/conventions.py +++ b/src/hdhelpers/metadata/conventions.py @@ -39,7 +39,6 @@ Field, GetJsonSchemaHandler, ModelWrapValidatorHandler, - PrivateAttr, StrictBool, StrictFloat, StrictInt, @@ -699,8 +698,6 @@ class _TimeseriesMetadataBase(MetadataReading, ConventionModel): """ _data_type: ClassVar[TimeseriesDataType] - # the metrics by id, built on first use - _by_id: dict[str, MetricMetadata] | None = PrivateAttr(default=None) value_dimensions_shared: tuple[SharedValueDimensionMetadata, ...] | None = optional( description="Metadata of value dimensions that have the same meaning for all metrics." @@ -727,9 +724,13 @@ def _conforms( def model_copy(self, *, update: Mapping[str, Any] | None = None, deep: bool = False) -> Self: copied = super().model_copy(update=update, deep=deep) - copied._by_id = None # the metrics may differ + copied.__dict__.pop("_metrics_by_id", None) # the metrics may differ return copied + def _converted[T: "_TimeseriesMetadataBase"](self, model: type[T], **fields: Any) -> T: + """The metadata as model with the given fields, keeping the extra fields""" + return model.model_validate(self.__pydantic_extra__ | fields) + def with_metric(self, metric_id: str | None = None, /, **changes: Any) -> Self: """A version with fields of a metric changed, the other metrics are shared @@ -920,12 +921,11 @@ def as_singletsframe(self, column: str = "value") -> "SingleTSFrameMetadata": Renames the value dimension "value" of the metric and the shared one to column. """ - return SingleTSFrameMetadata.model_validate( - { - "dataset_metadata": self.dataset_metadata, - "metrics": (_with_value_column(self.metric(), "value", column),), - "value_dimensions_shared": _renamed(self.value_dimensions_shared, "value", column), - } + return self._converted( + SingleTSFrameMetadata, + dataset_metadata=self.dataset_metadata, + metrics=(_with_value_column(self.metric(), "value", column),), + value_dimensions_shared=_renamed(self.value_dimensions_shared, "value", column), ) @@ -960,12 +960,11 @@ def as_series(self, column: str = "value") -> SeriesMetadata: """ if column not in {vd.column for vd in self.metric().value_dimensions}: raise KeyError(column) - return SeriesMetadata.model_validate( - { - "dataset_metadata": self.dataset_metadata, - "metrics": (_with_value_column(self.metric(), column, "value"),), - "value_dimensions_shared": _renamed(self.value_dimensions_shared, column, "value"), - } + return self._converted( + SeriesMetadata, + dataset_metadata=self.dataset_metadata, + metrics=(_with_value_column(self.metric(), column, "value"),), + value_dimensions_shared=_renamed(self.value_dimensions_shared, column, "value"), ) def as_multitsframe(self, metric_id: str | None = None) -> "MultiTSFrameMetadata": @@ -983,13 +982,12 @@ def as_multitsframe(self, metric_id: str | None = None) -> "MultiTSFrameMetadata metric_key = dataset_metadata["metric_key"] = _METRIC_NAME_FIELD metric = type(metric).model_validate(metric.model_dump() | {metric_key: metric_id}) metric_id = lookup(metric, dataset_metadata["metric_key"]) - return MultiTSFrameMetadata.model_validate( - { - "dataset_metadata": dataset_metadata - | {"single_metric": metric_id, "queried_metrics": [metric_id]}, - "metrics": (metric,), - "value_dimensions_shared": self.value_dimensions_shared, - } + return self._converted( + MultiTSFrameMetadata, + dataset_metadata=dataset_metadata + | {"single_metric": metric_id, "queried_metrics": [metric_id]}, + metrics=(metric,), + value_dimensions_shared=self.value_dimensions_shared, ) @@ -1022,13 +1020,12 @@ def narrowed_to(self, metric_id: str) -> SingleTSFrameMetadata: The metric is shared, not copied. It becomes the single and only queried metric. """ - return SingleTSFrameMetadata.model_validate( - { - "dataset_metadata": self.dataset_metadata.model_dump() - | {"single_metric": metric_id, "queried_metrics": [metric_id]}, - "metrics": (self.metric(metric_id),), - "value_dimensions_shared": self.value_dimensions_shared, - } + return self._converted( + SingleTSFrameMetadata, + dataset_metadata=self.dataset_metadata.model_dump() + | {"single_metric": metric_id, "queried_metrics": [metric_id]}, + metrics=(self.metric(metric_id),), + value_dimensions_shared=self.value_dimensions_shared, ) @@ -1595,7 +1592,12 @@ def _check_metrics_of_data( """Check that the metrics of the data and the queried metrics are described""" issues: list[MetadataConventionIssue] = [] dataset_metadata = attrs["dataset_metadata"] - metric_ids = {metric.get(dataset_metadata["metric_key"]) for metric in attrs["metrics"]} + # metrics without a string id are reported by _check_consistency + metric_ids = { + metric_id + for metric in attrs["metrics"] + if isinstance(metric_id := metric.get(dataset_metadata["metric_key"]), str) + } data_metrics: list[str] = [] if data_type == "MULTITSFRAME" and isinstance(data, pd.DataFrame) and "metric" in data.columns: data_metrics = [str(metric) for metric in data["metric"].unique()] diff --git a/tests/metadata/test_conventions_immutable.py b/tests/metadata/test_conventions_immutable.py index 113bc7a..0472ba0 100644 --- a/tests/metadata/test_conventions_immutable.py +++ b/tests/metadata/test_conventions_immutable.py @@ -146,8 +146,7 @@ def test_pickle_round_trip() -> None: assert isinstance(unpickled.metrics[0].inherited.asset, FrozenDict) -def test_pandas_operations_share_the_model() -> None: - meta = multitsframe_metadata() +def multitsframe(meta: MultiTSFrameMetadata) -> pd.DataFrame: df = pd.DataFrame( { "timestamp": pd.date_range("2026-01-01", periods=4, freq="h", tz="UTC"), @@ -156,6 +155,12 @@ def test_pandas_operations_share_the_model() -> None: } ) df.attrs = {"hd_metadata": meta} + return df + + +def test_pandas_operations_share_the_model() -> None: + meta = multitsframe_metadata() + df = multitsframe(meta) derived = [ df[df["value"] > 0], @@ -169,3 +174,15 @@ def test_pandas_operations_share_the_model() -> None: assert all(obj.attrs["hd_metadata"] is meta for obj in derived) assert all(obj.attrs is not df.attrs for obj in derived) + + +def test_reading_does_not_change_the_equality() -> None: + meta = multitsframe_metadata() + other = multitsframe_metadata() + + meta.unit("plant_a.pressure") + + assert meta == other + assert meta != meta.with_metric("plant_a.pressure", name="changed") + # pandas keeps attrs only if they are equal + assert pd.concat([multitsframe(meta), multitsframe(other)]).attrs["hd_metadata"] is meta diff --git a/tests/metadata/test_conventions_rules.py b/tests/metadata/test_conventions_rules.py index 99dd4f0..dccea28 100644 --- a/tests/metadata/test_conventions_rules.py +++ b/tests/metadata/test_conventions_rules.py @@ -39,8 +39,15 @@ def first_metric(attrs: dict[str, Any]) -> dict[str, Any]: return attrs["metrics"][0] +def unhashable_metric_ids(attrs: dict[str, Any]) -> None: + attrs["dataset_metadata"]["metric_key"] = "custom_key" + for metric in attrs["metrics"]: + metric["custom_key"] = [metric["external_id"]] + + # (data type, change of its example, location of the error) RULE_VIOLATIONS = [ + ("MULTITSFRAME", unhashable_metric_ids, "metrics[0]"), ( "MULTITSFRAME", lambda a: first_metric(a).pop("external_id"), diff --git a/tests/metadata/test_helpers.py b/tests/metadata/test_helpers.py index f955795..3821559 100644 --- a/tests/metadata/test_helpers.py +++ b/tests/metadata/test_helpers.py @@ -246,6 +246,15 @@ def test_metadata_not_following_the_conventions_gives_no_information() -> None: assert get_units(frame(attrs))[SIGNAL_ID]["value"] is None +def test_metric_ids_which_are_no_strings_give_no_information() -> None: + attrs = deepcopy(platform_series_attrs) + attrs["dataset_metadata"]["metric_key"] = "custom_key" + attrs["metrics"][0]["custom_key"] = [SIGNAL_ID] + + assert get_series_unit(series(attrs)) is None + assert get_units(frame(attrs))[SIGNAL_ID]["value"] is None + + @pytest.mark.parametrize( "attrs", [{}, {"dataset_metadata": {"metric_key": "id"}}, {"metrics": None}] ) diff --git a/tests/metadata/test_reading.py b/tests/metadata/test_reading.py index 1565425..2aa9cad 100644 --- a/tests/metadata/test_reading.py +++ b/tests/metadata/test_reading.py @@ -404,6 +404,18 @@ def test_singletsframe_as_multitsframe_with_another_id(self) -> None: assert meta.metric("plant.f").id == "f" assert meta.dataset_metadata.single_metric == "plant.f" + def test_conversions_keep_extra_fields(self) -> None: + extra = {"custom_root": {"source": "adapter"}} + meta = SingleTSFrameMetadata.model_validate(self.singletsframe().model_dump() | extra) + + series_meta = meta.as_series("pressure") + multitsframe_meta = meta.as_multitsframe() + + assert series_meta.model_extra == extra + assert series_meta.as_singletsframe().model_extra == extra + assert multitsframe_meta.model_extra == extra + assert multitsframe_meta.narrowed_to("f").model_extra == extra + def test_enum_members_are_given_as_values() -> None: meta = series({"recording_mode": "irregular", "default_aggregation_method": "MAX"}) From b8f5c929bf054309dfd07e80a47a457af28ffa17 Mon Sep 17 00:00:00 2001 From: stewit <> Date: Tue, 27 Oct 2026 12:54:49 +0100 Subject: [PATCH 4/8] review fixes and refactoring --- pyproject.toml | 14 +++--- src/hdhelpers/__init__.py | 20 ++++----- src/hdhelpers/helpers/locale.py | 2 +- src/hdhelpers/helpers/timezone_handling.py | 7 +-- src/hdhelpers/metadata/_reading.py | 27 +++++++---- src/hdhelpers/metadata/attached.py | 3 +- src/hdhelpers/metadata/conventions.py | 19 +++++--- src/hdhelpers/metadata/helpers.py | 6 +-- src/hdhelpers/plot_target_settings.py | 2 +- src/hdhelpers/plotting/plotly_theme.py | 52 +++++++++++----------- src/hdhelpers/plotting/timestamp_ticks.py | 16 +++---- tests/data/fixtures/helpers.py | 8 ++-- tests/metadata/test_reading.py | 23 ++++++++++ tests/test_optional_plotting.py | 16 +++++++ 14 files changed, 131 insertions(+), 84 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4f1c522..3d898ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,19 +83,17 @@ explicit = true # allow longer lines than the default (88, same as black) line-length = 100 +lint.select = ["E", "F", "B", "W", "PL", "ERA", "G", "I", "UP", "YTT", "S", "BLE", "A", "COM", "C4", "DTZ", "T10", "ISC", "ICN", "INP", "PIE", "T20", "PT", "Q", "RET", "SIM", "ARG", "PD"] + +lint.ignore = ["E713", "E714", "B008", "PLR2004", "COM812", "RET504", "PLR0913", "PLR0917", "UP017", "ISC001"] +# the aliases ruff expects by default for the libraries used here, without "datetime" as "dt" +lint.flake8-import-conventions.aliases = { numpy = "np", "numpy.typing" = "npt", pandas = "pd", "plotly.express" = "px" } -[pycodestyle] +[tool.ruff.lint.pycodestyle] # Ensures E501 (line-too-long) uses the same limit max-line-length = 100 -# Assume python314 -target-version = "py314" - -lint.select = ["E", "F", "B", "W", "PL", "ERA", "G", "I", "UP", "YTT", "S", "BLE", "A", "COM", "C4", "DTZ", "T10", "ISC", "ICN", "INP", "PIE", "T20", "PT", "Q", "RET", "SIM", "ARG", "PD"] - -lint.ignore = ["E713", "E714", "B008", "PLR2004", "COM812", "RET504", "PLR0913", "UP017", "ISC001"] - [dependency-groups] dev = [ "certifi>=2026.10.19", diff --git a/src/hdhelpers/__init__.py b/src/hdhelpers/__init__.py index 59501b3..736baca 100644 --- a/src/hdhelpers/__init__.py +++ b/src/hdhelpers/__init__.py @@ -20,29 +20,25 @@ # The plotting helpers need plotly, which comes with the optional extra hdhelpers[plotting]. They # are imported on first access, so that everything else works without plotly. -_PLOTTING_ATTRIBUTES: dict[str, tuple[str, str | None]] = { - "plotting": ("hdhelpers.plotting", None), - "FUSEKI_COLOR_CYCLE": ("hdhelpers.plotting.color", "FUSEKI_COLOR_CYCLE"), - "resolve_color": ("hdhelpers.plotting.color", "resolve_color"), - "set_agnostic_theme": ("hdhelpers.plotting.plotly_theme", "set_agnostic_theme"), - "TICKFORMATSTOPS": ("hdhelpers.plotting.timestamp_ticks", "TICKFORMATSTOPS"), - "set_dt_ticks": ("hdhelpers.plotting.timestamp_ticks", "set_dt_ticks"), -} +_PLOTTING_NAMES = frozenset( + {"FUSEKI_COLOR_CYCLE", "resolve_color", "set_agnostic_theme", "TICKFORMATSTOPS", "set_dt_ticks"} +) def __getattr__(name: str) -> Any: - if name not in _PLOTTING_ATTRIBUTES: + if name != "plotting" and name not in _PLOTTING_NAMES: raise AttributeError(f"module 'hdhelpers' has no attribute {name!r}") - module_name, attribute = _PLOTTING_ATTRIBUTES[name] + # always via the package, which needs plotly: A failed import of the package keeps the + # submodules imported until then, e.g. hdhelpers.plotting.color, which does not need plotly. try: - module = importlib.import_module(module_name) + plotting = importlib.import_module("hdhelpers.plotting") except ImportError as e: if (e.name or "").partition(".")[0] != "plotly": raise raise ImportError( f"hdhelpers.{name} needs plotly, which is installed with the extra hdhelpers[plotting]" ) from e - value = module if attribute is None else getattr(module, attribute) + value = plotting if name == "plotting" else getattr(plotting, name) globals()[name] = value return value diff --git a/src/hdhelpers/helpers/locale.py b/src/hdhelpers/helpers/locale.py index 8422b3b..af88a25 100644 --- a/src/hdhelpers/helpers/locale.py +++ b/src/hdhelpers/helpers/locale.py @@ -24,7 +24,7 @@ def get_locale(explicit_locale: str | None = None) -> str | None: >>> print(get_locale()) # returns locale from plot target settings of execution or None None - """ + """ # noqa: E501 if explicit_locale: return explicit_locale.replace("_", "-") diff --git a/src/hdhelpers/helpers/timezone_handling.py b/src/hdhelpers/helpers/timezone_handling.py index 35c4097..7050740 100644 --- a/src/hdhelpers/helpers/timezone_handling.py +++ b/src/hdhelpers/helpers/timezone_handling.py @@ -12,7 +12,7 @@ @singledispatch -def _convert_to_optional_timezone(object_to_convert, to_timezone: str | None): +def _convert_to_optional_timezone(object_to_convert, to_timezone: str | None): # noqa: ARG001 """Convert object_to_convert to to_timezone if not None, or to its own timezone if aware or to UTC otherwise""" @@ -77,13 +77,14 @@ def modify_timezone[T: (pd.Timestamp, datetime.datetime, pd.Series, pd.DataFrame >>> modified_timezone = modify_timezone(pd.to_datetime("2025-01-01T01:00:00+05:00"), to_timezone="Europe/Berlin") >>> int(modified_timezone.utcoffset().total_seconds()) 3600 - """ + """ # noqa: E501 if not isinstance( object_to_convert, pd.Timestamp | pd.Series | pd.DataFrame | datetime.datetime ): raise TypeError( - f"object_to_convert is {type(object_to_convert)} not pd.Timestamp | pd.Series | pd.DataFrame | datetime.datetime" + f"object_to_convert is {type(object_to_convert)} not pd.Timestamp | pd.Series |" + " pd.DataFrame | datetime.datetime" ) if column_names is None: column_names = [] diff --git a/src/hdhelpers/metadata/_reading.py b/src/hdhelpers/metadata/_reading.py index dcab2fd..50396d7 100644 --- a/src/hdhelpers/metadata/_reading.py +++ b/src/hdhelpers/metadata/_reading.py @@ -86,19 +86,28 @@ def lookup(obj: Any, path: str) -> Any: the JSON form of the metadata. """ for key in path.split("."): - if isinstance(obj, BaseModel): - # __dict__ holds the fields, without warning about deprecated ones like getattr - value = obj.__dict__.get(key, _MISSING) - obj = (obj.__pydantic_extra__ or {}).get(key) if value is _MISSING else value - elif isinstance(obj, Mapping): - obj = obj.get(key) - else: - return None + obj = _field(obj, key) if obj is None: return None return _plain(obj) +def id_of(metric: "MetricMetadata", metric_key: str) -> Any: + """The id of a metric: its field named metric_key, which may contain dots, None if missing""" + return _plain(_field(metric, metric_key)) + + +def _field(obj: Any, key: str) -> Any: + """A field or extra field of a model or a key of a mapping, None if missing""" + if isinstance(obj, BaseModel): + # __dict__ holds the fields, without warning about deprecated ones like getattr + value = obj.__dict__.get(key, _MISSING) + return (obj.__pydantic_extra__ or {}).get(key) if value is _MISSING else value + if isinstance(obj, Mapping): + return obj.get(key) + return None + + def _plain(value: Any) -> Any: """The value of an enum member, e.g. "MEAN" instead of AggregationMethod.MEAN""" return value.value if isinstance(value, Enum) else value @@ -181,7 +190,7 @@ def _metrics_by_id(self) -> "dict[str, MetricMetadata]": Reading must not change the equality, since pandas keeps only equal attrs, e.g. on concat. """ metric_key = self.dataset_metadata.metric_key - return {lookup(metric, metric_key): metric for metric in self.metrics} + return {id_of(metric, metric_key): metric for metric in self.metrics} def _metric_or_none(self, metric_id: str | None) -> "MetricMetadata | None": if metric_id is None: diff --git a/src/hdhelpers/metadata/attached.py b/src/hdhelpers/metadata/attached.py index 17f1fe1..63f4cff 100644 --- a/src/hdhelpers/metadata/attached.py +++ b/src/hdhelpers/metadata/attached.py @@ -43,7 +43,7 @@ def infer_data_type(obj: pd.Series | pd.DataFrame) -> TimeseriesDataType: - """SERIES for a Series, MULTITSFRAME for a DataFrame with a "metric" column, else SINGLETSFRAME""" + """SERIES for a Series, MULTITSFRAME for a frame with a "metric" column, else SINGLETSFRAME""" if isinstance(obj, pd.Series): return "SERIES" return "MULTITSFRAME" if "metric" in obj.columns else "SINGLETSFRAME" @@ -166,7 +166,6 @@ def _no_metadata(required: bool, message: str) -> None: ) ] ) - return None def set_metadata(obj: pd.Series | pd.DataFrame, meta: AnyMetadata) -> None: diff --git a/src/hdhelpers/metadata/conventions.py b/src/hdhelpers/metadata/conventions.py index 4ff65c4..d34a3d8 100644 --- a/src/hdhelpers/metadata/conventions.py +++ b/src/hdhelpers/metadata/conventions.py @@ -50,7 +50,7 @@ from pydantic.json_schema import JsonSchemaValue from pydantic_core import CoreSchema, PydanticCustomError -from hdhelpers.metadata._reading import MetadataReading, lookup +from hdhelpers.metadata._reading import MetadataReading, id_of TimeseriesDataType = Literal["SERIES", "SINGLETSFRAME", "MULTITSFRAME"] @@ -183,7 +183,7 @@ class RecordingMode(StrEnum): ) -def _immutable(self: Any, *args: Any, **kwargs: Any) -> NoReturn: +def _immutable(self: Any, *_args: Any, **_kwargs: Any) -> NoReturn: raise TypeError(f"{type(self).__name__} is immutable") @@ -290,7 +290,12 @@ class ConventionModel(BaseModel): def __deepcopy__(self, memo: dict[int, Any] | None = None) -> Self: return self - def model_copy(self, *, update: Mapping[str, Any] | None = None, deep: bool = False) -> Self: + def model_copy( + self, + *, + update: Mapping[str, Any] | None = None, + deep: bool = False, # noqa: ARG002 + ) -> Self: """Copy with the given fields replaced, without validating them Always shallow, since the model is immutable. A deep copy would apply the update to the @@ -739,8 +744,8 @@ def with_metric(self, metric_id: str | None = None, /, **changes: Any) -> Self: old = self.metric(metric_id) new = type(old).model_validate(old.model_dump() | changes) errors = _metric_rule_errors(self, new) - new_id = lookup(new, self.dataset_metadata.metric_key) - if new_id != lookup(old, self.dataset_metadata.metric_key) and new_id in self.metric_ids(): + new_id = id_of(new, self.dataset_metadata.metric_key) + if new_id != id_of(old, self.dataset_metadata.metric_key) and new_id in self.metric_ids(): errors.append(_error("metrics", f'Metric "{new_id}" occurs more than once.')) if len(errors) > 0: raise MetadataConventionError(errors) @@ -978,10 +983,10 @@ def as_multitsframe(self, metric_id: str | None = None) -> "MultiTSFrameMetadata dataset_metadata = self.dataset_metadata.model_dump() if metric_id is not None and metric_id != self.metric_ids()[0]: metric_key = dataset_metadata["metric_key"] - if lookup(metric, metric_key) is not None: + if id_of(metric, metric_key) is not None: metric_key = dataset_metadata["metric_key"] = _METRIC_NAME_FIELD metric = type(metric).model_validate(metric.model_dump() | {metric_key: metric_id}) - metric_id = lookup(metric, dataset_metadata["metric_key"]) + metric_id = id_of(metric, dataset_metadata["metric_key"]) return self._converted( MultiTSFrameMetadata, dataset_metadata=dataset_metadata diff --git a/src/hdhelpers/metadata/helpers.py b/src/hdhelpers/metadata/helpers.py index 9d62ad2..1c3df88 100644 --- a/src/hdhelpers/metadata/helpers.py +++ b/src/hdhelpers/metadata/helpers.py @@ -389,7 +389,7 @@ def get_singlets_metric_info( lambda meta, metric: meta.metric_info( metric_info, metric, include_inherited=include_inherited ), - lambda meta: None, + lambda _meta: None, empty=None, ) @@ -541,7 +541,7 @@ def get_series_default_aggregation_method(series: pd.Series) -> str: series, "SERIES", lambda meta, metric: meta.default_aggregation_method(metric), - lambda meta: _DEFAULT_AGGREGATION_METHOD, + lambda _meta: _DEFAULT_AGGREGATION_METHOD, empty=_DEFAULT_AGGREGATION_METHOD, ) @@ -553,7 +553,7 @@ def get_singlets_default_aggregation_method(singletsframe: pd.DataFrame) -> str: singletsframe, "SINGLETSFRAME", lambda meta, metric: meta.default_aggregation_method(metric), - lambda meta: _DEFAULT_AGGREGATION_METHOD, + lambda _meta: _DEFAULT_AGGREGATION_METHOD, empty=_DEFAULT_AGGREGATION_METHOD, ) diff --git a/src/hdhelpers/plot_target_settings.py b/src/hdhelpers/plot_target_settings.py index 08a4623..eeb2f1a 100644 --- a/src/hdhelpers/plot_target_settings.py +++ b/src/hdhelpers/plot_target_settings.py @@ -101,7 +101,7 @@ def get_plot_target_settings() -> PlotTargetSettings: ) plot_target_settings_in_context: BaseModel = get_runtime_exec_context().plot_target_settings - # Ensure that plot_target_settings is compatible with hdhelpers version of PlotTargetSettings + # Ensure that plot_target_settings is compatible with the PlotTargetSettings of hdhelpers plot_target_settings_as_dict = plot_target_settings_in_context.model_dump(mode="json") plot_target_settings = PlotTargetSettings(**plot_target_settings_as_dict) return plot_target_settings diff --git a/src/hdhelpers/plotting/plotly_theme.py b/src/hdhelpers/plotting/plotly_theme.py index 17d8278..3b9d93b 100644 --- a/src/hdhelpers/plotting/plotly_theme.py +++ b/src/hdhelpers/plotting/plotly_theme.py @@ -11,32 +11,32 @@ plot_bgcolor="rgba(128, 128, 128, 0)", # plot_bg_color not 0,0,0,0 because zoom highlighting then does not work properly! # Must be >=128 to actuall work in light mode! - font=dict(color=NEUTRAL_TEXT, family="Inter, Helvetica, sans-serif", size=12), - xaxis=dict( - showgrid=True, - gridcolor=NEUTRAL_GRID, - zerolinecolor=NEUTRAL, - linecolor=NEUTRAL, - tickfont=dict(color=NEUTRAL_TEXT), - ), - yaxis=dict( - showgrid=True, - gridcolor=NEUTRAL_GRID, - zerolinecolor=NEUTRAL, - linecolor=NEUTRAL, - tickfont=dict(color=NEUTRAL_TEXT), - ), - hoverlabel=dict( - bgcolor="rgba(128,128,128,0.85)", - bordercolor="rgba(0,0,0,0.08)", - font=dict(color="#fff", size=12, family="Inter, Helvetica, sans-serif"), - namelength=0, - ), - modebar=dict( - bgcolor="rgba(0,0,0,0)", - color=NEUTRAL_TEXT, - activecolor="rgba(128,128,128,1)", - ), + font={"color": NEUTRAL_TEXT, "family": "Inter, Helvetica, sans-serif", "size": 12}, + xaxis={ + "showgrid": True, + "gridcolor": NEUTRAL_GRID, + "zerolinecolor": NEUTRAL, + "linecolor": NEUTRAL, + "tickfont": {"color": NEUTRAL_TEXT}, + }, + yaxis={ + "showgrid": True, + "gridcolor": NEUTRAL_GRID, + "zerolinecolor": NEUTRAL, + "linecolor": NEUTRAL, + "tickfont": {"color": NEUTRAL_TEXT}, + }, + hoverlabel={ + "bgcolor": "rgba(128,128,128,0.85)", + "bordercolor": "rgba(0,0,0,0.08)", + "font": {"color": "#fff", "size": 12, "family": "Inter, Helvetica, sans-serif"}, + "namelength": 0, + }, + modebar={ + "bgcolor": "rgba(0,0,0,0)", + "color": NEUTRAL_TEXT, + "activecolor": "rgba(128,128,128,1)", + }, ) ) diff --git a/src/hdhelpers/plotting/timestamp_ticks.py b/src/hdhelpers/plotting/timestamp_ticks.py index 4fe55dd..5ebd8c3 100644 --- a/src/hdhelpers/plotting/timestamp_ticks.py +++ b/src/hdhelpers/plotting/timestamp_ticks.py @@ -13,14 +13,14 @@ # language agnostic, isoformat inspired, plotl x axis timestamp formatting # defined for multiple zoom levels # always showing a tick with the current day somewhere for reference - dict(dtickrange=[None, 1000], value="%H:%M:%S.%L\n%Y-%m-%d"), - dict(dtickrange=[1000, 60000], value="%H:%M:%S\n%Y-%m-%d"), - dict(dtickrange=[60000, 3600000], value="%H:%M\n%Y-%m-%d"), - dict(dtickrange=[3600000, 86400000], value="%H:%M\n%Y-%m-%d"), - dict(dtickrange=[86400000, 604800000], value="%H:%M\n%Y-%m-%d"), - dict(dtickrange=[604800000, "M1"], value="%Y-%m-%d"), - dict(dtickrange=["M1", "M12"], value="%Y-%m"), - dict(dtickrange=["M12", None], value="%Y"), + {"dtickrange": [None, 1000], "value": "%H:%M:%S.%L\n%Y-%m-%d"}, + {"dtickrange": [1000, 60000], "value": "%H:%M:%S\n%Y-%m-%d"}, + {"dtickrange": [60000, 3600000], "value": "%H:%M\n%Y-%m-%d"}, + {"dtickrange": [3600000, 86400000], "value": "%H:%M\n%Y-%m-%d"}, + {"dtickrange": [86400000, 604800000], "value": "%H:%M\n%Y-%m-%d"}, + {"dtickrange": [604800000, "M1"], "value": "%Y-%m-%d"}, + {"dtickrange": ["M1", "M12"], "value": "%Y-%m"}, + {"dtickrange": ["M12", None], "value": "%Y"}, ] diff --git a/tests/data/fixtures/helpers.py b/tests/data/fixtures/helpers.py index 104eb23..98cef5c 100644 --- a/tests/data/fixtures/helpers.py +++ b/tests/data/fixtures/helpers.py @@ -2,7 +2,7 @@ import pytest -@pytest.fixture(scope="function") +@pytest.fixture() def series_winter() -> pd.Series: winter = pd.Series( [0, 1, 2, 3], @@ -17,7 +17,7 @@ def series_winter() -> pd.Series: return winter -@pytest.fixture(scope="function") +@pytest.fixture() def series_summer() -> pd.Series: summer = pd.Series( [0, 1, 2, 3], @@ -31,7 +31,7 @@ def series_summer() -> pd.Series: return summer -@pytest.fixture(scope="function") +@pytest.fixture() def dataframe() -> pd.DataFrame: values = [1.0, 1.2, 1.2] timestamps = pd.to_datetime( @@ -49,7 +49,7 @@ def dataframe() -> pd.DataFrame: return ts_df -@pytest.fixture(scope="function") +@pytest.fixture() def multicolumn_frame() -> pd.DataFrame: values = [1.0, 1.2, 1.2] index = pd.to_datetime( diff --git a/tests/metadata/test_reading.py b/tests/metadata/test_reading.py index 2aa9cad..ff96930 100644 --- a/tests/metadata/test_reading.py +++ b/tests/metadata/test_reading.py @@ -71,6 +71,29 @@ def test_metric_key_names_the_id(self) -> None: assert meta.metric_ids() == ["plant.temperature"] + def test_metric_key_with_dots_names_a_field(self) -> None: + meta = MultiTSFrameMetadata.model_validate( + { + "dataset_metadata": dataset_metadata(metric_key="custom.id"), + "metrics": [ + metric( + "m1", value_dimensions=[value_dimension(unit="m")], **{"custom.id": "a"} + ), + metric( + "m2", value_dimensions=[value_dimension(unit="s")], **{"custom.id": "b"} + ), + ], + } + ) + + assert meta.metric_ids() == ["a", "b"] + assert meta.units() == {"a": {"value": "m"}, "b": {"value": "s"}} + assert meta.with_metric("a", **{"custom.id": "c"}).metric_ids() == ["c", "b"] + with pytest.raises(MetadataConventionError, match="more than once"): + meta.with_metric("a", **{"custom.id": "b"}) + assert meta.narrowed_to("b").as_multitsframe().metric_ids() == ["b"] + assert meta.narrowed_to("b").as_multitsframe("x").metric_ids() == ["x"] + def test_unknown_metric(self) -> None: meta = multitsframe(metric("a")) diff --git a/tests/test_optional_plotting.py b/tests/test_optional_plotting.py index 3017c87..bd45700 100644 --- a/tests/test_optional_plotting.py +++ b/tests/test_optional_plotting.py @@ -32,6 +32,22 @@ def test_plotting_names_need_the_plotting_extra() -> None: assert "hdhelpers[plotting]" in result.stderr +def test_plotting_names_need_the_plotting_extra_on_every_access() -> None: + # twice, since a failed import keeps the modules imported until then, e.g. the colors + result = run_without_plotly( + "import hdhelpers\n" + "for name in sorted(hdhelpers._PLOTTING_NAMES | {'plotting'}) * 2:\n" + " try:\n" + " getattr(hdhelpers, name)\n" + " except ImportError as e:\n" + " assert 'hdhelpers[plotting]' in str(e), e\n" + " else:\n" + " raise SystemExit(f'hdhelpers.{name} is available without plotly')\n" + ) + + assert result.returncode == 0, result.stderr + + def test_plotting_names_are_available_with_plotly() -> None: import hdhelpers # noqa: PLC0415 from hdhelpers.plotting.timestamp_ticks import set_dt_ticks # noqa: PLC0415 From 80d030a7d45d4fa4ca172bcc4191019bbb4e6e06 Mon Sep 17 00:00:00 2001 From: jenny kupzig Date: Tue, 27 Oct 2026 15:07:30 +0100 Subject: [PATCH 5/8] rename without_metadata, shorten Changelog, update docs, enhance pipeline --- .github/workflows/check_pull_request.yml | 56 + CHANGELOG.md | 26 +- docs/_sources/first_steps.rst.txt | 106 +- docs/first_steps.html | 181 ++- docs/genindex.html | 47 +- docs/index.html | 1365 +++++------------ docs/objects.inv | Bin 663 -> 731 bytes docs/py-modindex.html | 2 +- docs/search.html | 2 +- docs/searchindex.js | 2 +- src/hdhelpers/metadata/__init__.py | 4 +- src/hdhelpers/metadata/helpers.py | 6 +- tests/metadata/test_conventions_schema.py | 5 +- ...adata.py => test_copy_without_metadata.py} | 12 +- 14 files changed, 704 insertions(+), 1110 deletions(-) rename tests/metadata/{test_metadata_without_metadata.py => test_copy_without_metadata.py} (81%) diff --git a/.github/workflows/check_pull_request.yml b/.github/workflows/check_pull_request.yml index a2b716d..02b0ef4 100644 --- a/.github/workflows/check_pull_request.yml +++ b/.github/workflows/check_pull_request.yml @@ -46,6 +46,62 @@ jobs: ./run test + test_without_plotly: + name: Core installation without Plotly (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.12", "3.15"] + + steps: + - uses: actions/checkout@v6 + + - name: Install uv and set the Python version + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + version: ">=0.13.4" + python-version: ${{ matrix.python-version }} + + - name: Install the core package without extras or development dependencies + run: uv sync --locked --no-dev --no-editable + + - name: Check core functionality and optional plotting exports + run: | + uv run --no-sync python -I - <<'PY' + import importlib.util + + assert importlib.util.find_spec("plotly") is None, "Plotly must not be installed" + + import pandas as pd + import hdhelpers + from hdhelpers.metadata import attach_metadata, attrs_to_json, get_series_unit, read_metadata + from hdhelpers.metadata.conventions import SERIES_METADATA_EXAMPLE + + series = pd.Series([1.0], index=pd.to_datetime(["2026-01-01T00:00:00Z"])) + series.attrs = SERIES_METADATA_EXAMPLE + meta = attach_metadata(series) + assert meta is not None + assert read_metadata(series) is meta + assert get_series_unit(series) == "m" + assert attrs_to_json(series.attrs)["metrics"][0]["value_dimensions"][0]["unit"] == "m" + converted = hdhelpers.modify_timezone(series, to_timezone="Europe/Berlin") + assert str(converted.index.tz) == "Europe/Berlin" + assert read_metadata(converted) is meta + + # Repeat every export to catch modules cached by an earlier failed import. + names = ( + "plotting", "FUSEKI_COLOR_CYCLE", "resolve_color", + "set_agnostic_theme", "TICKFORMATSTOPS", "set_dt_ticks", + ) + for name in names * 2: + try: + getattr(hdhelpers, name) + except ImportError as error: + assert "hdhelpers[plotting]" in str(error), (name, error) + else: + raise AssertionError(f"hdhelpers.{name} is available without Plotly") + PY + check_lint_format_mypy: runs-on: ubuntu-latest steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index 635658d..ff95727 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,22 +1,12 @@ ## [Unreleased] -- Update python versions to fix CA certificate issue (https://emudhra.we-have-a-problem.com/) -- metadata accessors can fall back to the properties a metric inherits from objects higher up in an asset hierarchy (under "inherited" in the metric metadata of the current convention), only as last resort after "value_dimensions_shared". This is the default for get_metric_info, get_singlets_metric_info, get_series_info and get_singlets_info. The functions for dedicated information (units, names, display names, short display names and measurements) only do so with keyword argument include_inherited=True, since an inherited property with the same label usually does not describe the metric itself -- get_metric_info and get_singlets_metric_info fall back to the value dimension "value" and to "value_dimensions_shared" if the metric itself does not provide the information -- fix: (short) display names of metadata following the current convention treat null as missing and look up each field on the value dimension, the metric and "value_dimensions_shared" before falling back to the next field. So e.g. a display name of the metric wins over the name of its value dimension -- fix: get_metric_info returns None entries instead of raising if the metadata contains no metric information -- the series accessors determine their metric like the SingleTSFrame accessors: the one given by "single_metric" in the dataset metadata or, if that is missing, the only metric present. The metric named "series" is only used as default for older metadata structures -- a "single_metric" in the dataset metadata which is no string counts as missing. Before, e.g. a list let all accessors of Series and SingleTSFrames raise an error -- fix: get_series_display_name no longer falls back to the short display name, as documented and like get_display_names and get_singlets_display_names -- fix: the SingleTSFrame accessors no longer use the only metric present if it is another one than the metric given by "single_metric" in the dataset metadata. The information counts as missing then -- metadata following the current convention is extracted in a single pass in plain Python instead of evaluating glom specs per level and field. The accessors are considerably faster for frames with many metrics, e.g. about 100 times for 1000 metrics when requesting field names or dotted paths. Other glom specs are still evaluated by glom, but only on the levels needed -- the accessors of Series and SingleTSFrames only resolve the information of their single metric. So they are considerably faster for a Series carrying the metadata of many metrics, e.g. taken from a MultiTSFrame, and a spec is not applied to the other metrics. Also get_series_unit and get_series_name no longer evaluate glom specs for metadata following the current convention. Both are about 90 times faster for the metadata of 1000 metrics -- the requested information is only looked up on the levels needed, so e.g. a spec transforming the information is no longer applied to null values of levels which are not used anyway -- entries of metadata following the current convention which do not adhere to the conventions are skipped: metrics which are no objects or whose field named by the metric key is no string, and value dimensions which are no objects or whose column is no string. "value_dimensions" and "value_dimensions_shared" which are no lists count as empty, a metric key which is no string as missing. Before, a single such entry could make all information missing or raise an error -- new functions get_default_timezones, get_series_default_timezone and get_singlets_default_timezone provide the default time zone of the metrics ("default_timezone" of the current metadata convention), falling back to the one of the dataset metadata. It is meant for computations, e.g. bucketing days when aggregating, not for displaying timestamps, which is what the time zone of the plot target settings is for. New section "Which time zone to compute in?" in the documentation -- documentation: the information requested from get_metric_info, get_singlets_metric_info, get_series_info and get_singlets_info is interpreted as a glom spec. New section "How to query nested information?" with examples for dotted paths into sub dictionaries, other glom specs and keys containing dots -- new class Recording and functions get_recordings, get_series_recording and get_singlets_recording provide how the values of the metrics are recorded: "recording_mode", "data_frequency" with "data_frequency_offset" and "data_frequency_timezone", and "recording_aggregation_method" of the current metadata convention. The frequency falls back to "ref_data_frequency", "ref_data_frequency_offset" and "ref_data_frequency_timezone" of the dataset metadata as a whole, i.e. offset and time zone only apply together with the frequency of the same level. The values are provided as given in the metadata, the documentation of Recording explains what they mean -- new functions get_default_aggregation_methods, get_series_default_aggregation_method and get_singlets_default_aggregation_method provide the default aggregation method of the metrics ("default_aggregation_method"), normalizing "AVG" to "MEAN". Without one, it is "LAST" for metrics whose value dimension "value" has the value data type "boolean" or "string", otherwise "MEAN" -- new function without_metadata provides a copy of a Series or DataFrame without its metadata (attrs) to compute on. pandas deep-copies the attrs to every object an operation creates, which makes e.g. loops over the metrics of a MultiTSFrame with the metadata of many metrics slow. The copy shares the data until it is changed + +- Add validated, immutable metadata models with helpers to read, attach, update, and serialize them. Pandas operations share attached models instead of repeatedly copying metadata. +- Improve metadata lookups with consistent fallbacks for missing values, display names, and inherited properties (`include_inherited=True` for dedicated unit/name helpers). +- Add helpers for default time zones, recording information, default aggregation methods, and copying pandas objects without metadata (`copy_without_metadata`). +- **Breaking:** Only the current metadata conventions are supported. Invalid metadata returns no information. Legacy formats and arbitrary glom specs are removed; queries accept field names and dotted paths. +- Make Plotly optional: install `hdhelpers[plotting]` for plotting helpers. Core metadata and time zone functionality works without it. +- Add Python 3.15 support and update Python/tooling requirements to address the CA certificate security issue. +- Expand validation and regression tests, correct lint configuration, and add CI checks of actual installations without Plotly. ## [0.0.9] - 2026-09-10 - singletsframe support diff --git a/docs/_sources/first_steps.rst.txt b/docs/_sources/first_steps.rst.txt index 24dab44..dc5324e 100644 --- a/docs/_sources/first_steps.rst.txt +++ b/docs/_sources/first_steps.rst.txt @@ -17,16 +17,18 @@ In hetida designer this series can be represented as json for *direct provisioni "dataset_metadata": { "metric_key": "external_id", "single_metric": "water.north.level", + "queried_metrics": ["water.north.level"], "ref_interval_start_timestamp": "2020-01-01T08:10:00+00:00", "ref_interval_end_timestamp": "2020-01-01T08:20:00+00:00", "ref_interval_type": "closed" }, "metrics": [ { + "id": "4c1f0e2a-7b3d-4e5f-9a8b-1c2d3e4f5a6b", "external_id": "water.north.level", "name": "Water level north", "short_display_name": "Water Level", - "value_dimensions": [{"column": "value", "unit": "cm"}], + "value_dimensions": [{"column": "value", "unit": "cm", "measurement": "level"}], "inherited": {"latitude": 51.43, "longitude": 7.58} } ] @@ -55,8 +57,65 @@ We can retrieve the name, unit and latitude of the series with the following cod ... +or, equivalently, via the metadata as model: + +.. code-block:: python + + from hdhelpers.metadata import read_metadata + + def main(*, series): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + + meta = read_metadata(series) # None if the series has no valid metadata + if meta is not None: + name = meta.name() # "Water level north" + unit = meta.unit() # "cm" + latitude = meta.metric_info("latitude") # 51.43 + + ... + .. _hetida designer metadata conventions: https://hetida.github.io/hetida-designer/user_guide/attached_metadata/ +Metadata as model +================= + +``hdhelpers.metadata.conventions`` holds the models of the metadata conventions, one per data +type: ``SeriesMetadata``, ``SingleTSFrameMetadata`` and ``MultiTSFrameMetadata``. They are the +contract of the conventions: Parsing fails for metadata which does not follow them. +``check_metadata_conventions`` reports the issues of such metadata at their locations. + +``read_metadata(obj)`` gives the metadata of a Series or DataFrame as model, or None if there is +no valid metadata. With ``required=True`` it raises a ``MetadataConventionError`` listing the +issues instead, for components which cannot work without metadata. The methods of the models +resolve the fallbacks described below, e.g. ``unit()``, ``display_name()``, ``metric_info()``, +``default_timezone()``, ``recording()`` and ``queried_interval()``. For a SERIES or +SINGLETSFRAME, the metric can be omitted. For a MULTITSFRAME, the methods take the metric, and +methods in plural like ``units()`` return the information of all metrics. + +The hetida designer runtime parses the metadata of the workflow inputs once and attaches the +model to them (``attach_metadata``). The models are immutable, so that all objects pandas +derives from an input share the model instead of copying the metadata, which saves a lot of +time for metadata of many metrics. To change metadata, create a changed version, which shares +all unchanged parts, and set it on your result: + +.. code-block:: python + + from hdhelpers.metadata import read_metadata, set_metadata + + def main(*, series): + # entrypoint function for this component + # ***** DO NOT EDIT LINES ABOVE ***** + + result = series * 100 + meta = read_metadata(series, required=True) + set_metadata(result, meta.with_value_dimension(None, "value", unit="m")) + + return {"result": result} + +Since the model is no JSON value, ``json.dumps(obj.attrs)`` or ``obj.to_parquet()`` fail for +objects with attached metadata. Use ``attrs_to_json(obj.attrs)`` to get attrs in their JSON form. + Where does the information come from? ===================================== @@ -91,43 +150,36 @@ In contrast, the functions for dedicated information do not use them by default, inherited property with the same label usually does not describe the metric itself, e.g. a property "unit" of a plant. Pass ``include_inherited=True`` to use them as last resort anyway. -Entries which do not adhere to the conventions are skipped: metrics which are no objects or -whose field named by the metric key is no string, and value dimensions which are no objects or -whose column is no string. ``"value_dimensions"`` and ``"value_dimensions_shared"`` which are no -lists count as empty, a metric key which is no string as missing, i.e. ``"id"`` is used. +Metadata which does not follow the conventions provides no information at all: The functions +return None or empty results for it. How to query nested information? ================================ The information requested from ``get_metric_info``, ``get_singlets_metric_info``, -``get_series_info`` and ``get_singlets_info`` is interpreted as a `glom`_ spec. A string is a -path in which dots separate the keys, so it also reaches into sub dictionaries. Let's say the -metric of the series above additionally provides comments and information on its sensor (the -conventions allow additional fields at every level), and the label of one of its inherited -properties contains a dot: +``get_series_info`` and ``get_singlets_info`` (and from the corresponding methods of the models) +is a field or a path of fields separated by dots, which reaches into sub objects. Let's say the +metric of the series above additionally provides information on its sensor (the conventions +allow additional fields at every level): .. code-block:: json { + "id": "4c1f0e2a-7b3d-4e5f-9a8b-1c2d3e4f5a6b", "external_id": "water.north.level", "name": "Water level north", "sensor": { "manufacturer": "Acme", "calibration": {"date": "2026-03-01", "offset": 0.02} }, - "comments": [ - {"content": "Gauge maintenance"}, - {"content": "Sensor replaced"} - ], - "value_dimensions": [{"column": "value", "unit": "cm"}], - "inherited": {"latitude": 51.43, "operator.name": "Ruhrverband"} + "value_dimensions": [{"column": "value", "unit": "cm", "measurement": "level"}], + "inherited": {"latitude": 51.43} } We can retrieve the nested information with the following code .. code-block:: python - from glom import Path from hdhelpers.metadata import get_series_info def main(*, series): @@ -139,13 +191,6 @@ We can retrieve the nested information with the following code latitude = get_series_info(series, "inherited.latitude") # 51.43, never the metric's own firmware = get_series_info(series, "sensor.firmware") # None, not provided - # any other glom spec, e.g. the content of each comment - comments = get_series_info(series, ("comments", ["content"])) - # ["Gauge maintenance", "Sensor replaced"] - - # a key which itself contains dots requires a glom Path - operator = get_series_info(series, Path("operator.name")) # "Ruhrverband" - ... For a MULTITSFRAME, ``get_metric_info(multitsframe, "sensor.manufacturer")`` provides the @@ -153,16 +198,9 @@ manufacturer keyed by metric accordingly. The fallbacks described above apply to the whole path, e.g. ``"sensor.manufacturer"`` is looked up in the metric first, then in its value dimension ``"value"`` and so on. A path which is not -present yields None instead of an error. Note that a plain string like ``"operator.name"`` is -always split at its dots, so it does not find the inherited property ``"operator.name"``. - -A spec may also transform the information, e.g. ``("name", str.upper)``. Since information -provided as null counts as missing, such a spec may receive null: on the metric itself and, if -the metric does not provide the information, on the levels it falls back to. Errors raised by -the spec are not caught. So either let it handle None, or rather transform the information -after retrieving it. - -.. _glom: https://glom.readthedocs.io/ +present yields None instead of an error. Paths cannot address list items or keys which contain +dots themselves. Such information, e.g. the comments of a metric, is available via the models: +``read_metadata(series).metric().comments``. Which time zone to compute in? ============================== diff --git a/docs/first_steps.html b/docs/first_steps.html index a6623f6..57e2888 100644 --- a/docs/first_steps.html +++ b/docs/first_steps.html @@ -48,30 +48,32 @@

How to get metadata with hdhelpers?hetida designer metadata conventions. In hetida designer this series can be represented as json for direct provisioning

{
-    "__hd_wrapped_data_object__": "SERIES",
-    "__metadata__": {
-        "dataset_metadata": {
-            "metric_key": "external_id",
-            "single_metric": "water.north.level",
-            "ref_interval_start_timestamp": "2020-01-01T08:10:00+00:00",
-            "ref_interval_end_timestamp": "2020-01-01T08:20:00+00:00",
-            "ref_interval_type": "closed"
+    "__hd_wrapped_data_object__": "SERIES",
+    "__metadata__": {
+        "dataset_metadata": {
+            "metric_key": "external_id",
+            "single_metric": "water.north.level",
+            "queried_metrics": ["water.north.level"],
+            "ref_interval_start_timestamp": "2020-01-01T08:10:00+00:00",
+            "ref_interval_end_timestamp": "2020-01-01T08:20:00+00:00",
+            "ref_interval_type": "closed"
         },
-        "metrics": [
+        "metrics": [
             {
-                "external_id": "water.north.level",
-                "name": "Water level north",
-                "short_display_name": "Water Level",
-                "value_dimensions": [{"column": "value", "unit": "cm"}],
-                "inherited": {"latitude": 51.43, "longitude": 7.58}
+                "id": "4c1f0e2a-7b3d-4e5f-9a8b-1c2d3e4f5a6b",
+                "external_id": "water.north.level",
+                "name": "Water level north",
+                "short_display_name": "Water Level",
+                "value_dimensions": [{"column": "value", "unit": "cm", "measurement": "level"}],
+                "inherited": {"latitude": 51.43, "longitude": 7.58}
             }
         ]
     },
-    "__data__": {
-        "2020-01-01T08:10:00+00:00": 1,
-        "2020-01-01T08:15:00+00:00": 2,
-        "2020-01-01T08:16:00+00:00": 3,
-        "2020-01-01T08:17:00+00:00": 4
+    "__data__": {
+        "2020-01-01T08:10:00+00:00": 1,
+        "2020-01-01T08:15:00+00:00": 2,
+        "2020-01-01T08:16:00+00:00": 3,
+        "2020-01-01T08:17:00+00:00": 4
     }
 }
@@ -83,13 +85,63 @@

How to get metadata with hdhelpers?# entrypoint function for this component # ***** DO NOT EDIT LINES ABOVE ***** - name = get_series_name(series) # "Water level north" - unit = get_series_unit(series) # "cm" - latitude = get_series_info(series, "latitude") # 51.43, inherited from the hierarchy + name = get_series_name(series) # "Water level north" + unit = get_series_unit(series) # "cm" + latitude = get_series_info(series, "latitude") # 51.43, inherited from the hierarchy ...

+

or, equivalently, via the metadata as model:

+
from hdhelpers.metadata import read_metadata
+
+def main(*, series):
+    # entrypoint function for this component
+    # ***** DO NOT EDIT LINES ABOVE *****
+
+    meta = read_metadata(series)  # None if the series has no valid metadata
+    if meta is not None:
+        name = meta.name()  # "Water level north"
+        unit = meta.unit()  # "cm"
+        latitude = meta.metric_info("latitude")  # 51.43
+
+    ...
+
+
+ +
+

Metadata as model

+

hdhelpers.metadata.conventions holds the models of the metadata conventions, one per data +type: SeriesMetadata, SingleTSFrameMetadata and MultiTSFrameMetadata. They are the +contract of the conventions: Parsing fails for metadata which does not follow them. +check_metadata_conventions reports the issues of such metadata at their locations.

+

read_metadata(obj) gives the metadata of a Series or DataFrame as model, or None if there is +no valid metadata. With required=True it raises a MetadataConventionError listing the +issues instead, for components which cannot work without metadata. The methods of the models +resolve the fallbacks described below, e.g. unit(), display_name(), metric_info(), +default_timezone(), recording() and queried_interval(). For a SERIES or +SINGLETSFRAME, the metric can be omitted. For a MULTITSFRAME, the methods take the metric, and +methods in plural like units() return the information of all metrics.

+

The hetida designer runtime parses the metadata of the workflow inputs once and attaches the +model to them (attach_metadata). The models are immutable, so that all objects pandas +derives from an input share the model instead of copying the metadata, which saves a lot of +time for metadata of many metrics. To change metadata, create a changed version, which shares +all unchanged parts, and set it on your result:

+
from hdhelpers.metadata import read_metadata, set_metadata
+
+def main(*, series):
+    # entrypoint function for this component
+    # ***** DO NOT EDIT LINES ABOVE *****
+
+    result = series * 100
+    meta = read_metadata(series, required=True)
+    set_metadata(result, meta.with_value_dimension(None, "value", unit="m"))
+
+    return {"result": result}
+
+
+

Since the model is no JSON value, json.dumps(obj.attrs) or obj.to_parquet() fail for +objects with attached metadata. Use attrs_to_json(obj.attrs) to get attrs in their JSON form.

Where does the information come from?

@@ -120,54 +172,40 @@

Where does the information come from?include_inherited=True to use them as last resort anyway.

-

Entries which do not adhere to the conventions are skipped: metrics which are no objects or -whose field named by the metric key is no string, and value dimensions which are no objects or -whose column is no string. "value_dimensions" and "value_dimensions_shared" which are no -lists count as empty, a metric key which is no string as missing, i.e. "id" is used.

+

Metadata which does not follow the conventions provides no information at all: The functions +return None or empty results for it.

How to query nested information?

The information requested from get_metric_info, get_singlets_metric_info, -get_series_info and get_singlets_info is interpreted as a glom spec. A string is a -path in which dots separate the keys, so it also reaches into sub dictionaries. Let’s say the -metric of the series above additionally provides comments and information on its sensor (the -conventions allow additional fields at every level), and the label of one of its inherited -properties contains a dot:

+get_series_info and get_singlets_info (and from the corresponding methods of the models) +is a field or a path of fields separated by dots, which reaches into sub objects. Let’s say the +metric of the series above additionally provides information on its sensor (the conventions +allow additional fields at every level):

{
-    "external_id": "water.north.level",
-    "name": "Water level north",
-    "sensor": {
-        "manufacturer": "Acme",
-        "calibration": {"date": "2026-03-01", "offset": 0.02}
+    "id": "4c1f0e2a-7b3d-4e5f-9a8b-1c2d3e4f5a6b",
+    "external_id": "water.north.level",
+    "name": "Water level north",
+    "sensor": {
+        "manufacturer": "Acme",
+        "calibration": {"date": "2026-03-01", "offset": 0.02}
     },
-    "comments": [
-        {"content": "Gauge maintenance"},
-        {"content": "Sensor replaced"}
-    ],
-    "value_dimensions": [{"column": "value", "unit": "cm"}],
-    "inherited": {"latitude": 51.43, "operator.name": "Ruhrverband"}
+    "value_dimensions": [{"column": "value", "unit": "cm", "measurement": "level"}],
+    "inherited": {"latitude": 51.43}
 }

We can retrieve the nested information with the following code

-
from glom import Path
-from hdhelpers.metadata import get_series_info
+
from hdhelpers.metadata import get_series_info
 
 def main(*, series):
     # entrypoint function for this component
     # ***** DO NOT EDIT LINES ABOVE *****
 
-    manufacturer = get_series_info(series, "sensor.manufacturer")  # "Acme"
-    calibration_date = get_series_info(series, "sensor.calibration.date")  # "2026-03-01"
-    latitude = get_series_info(series, "inherited.latitude")  # 51.43, never the metric's own
-    firmware = get_series_info(series, "sensor.firmware")  # None, not provided
-
-    # any other glom spec, e.g. the content of each comment
-    comments = get_series_info(series, ("comments", ["content"]))
-    # ["Gauge maintenance", "Sensor replaced"]
-
-    # a key which itself contains dots requires a glom Path
-    operator = get_series_info(series, Path("operator.name"))  # "Ruhrverband"
+    manufacturer = get_series_info(series, "sensor.manufacturer")  # "Acme"
+    calibration_date = get_series_info(series, "sensor.calibration.date")  # "2026-03-01"
+    latitude = get_series_info(series, "inherited.latitude")  # 51.43, never the metric's own
+    firmware = get_series_info(series, "sensor.firmware")  # None, not provided
 
     ...
 

@@ -176,13 +214,9 @@ 
How to query nested information?"sensor.manufacturer" is looked
 up in the metric first, then in its value dimension "value" and so on. A path which is not
-present yields None instead of an error. Note that a plain string like "operator.name" is
-always split at its dots, so it does not find the inherited property "operator.name".

-
A spec may also transform the information, e.g. ("name", str.upper). Since information
-provided as null counts as missing, such a spec may receive null: on the metric itself and, if
-the metric does not provide the information, on the levels it falls back to. Errors raised by
-the spec are not caught. So either let it handle None, or rather transform the information
-after retrieving it.

+present yields None instead of an error. Paths cannot address list items or keys which contain
+dots themselves. Such information, e.g. the comments of a metric, is available via the models:
+read_metadata(series).metric().comments.

Which time zone to compute in?

@@ -212,10 +246,10 @@

Which time zone to compute in?# ***** DO NOT EDIT LINES ABOVE ***** # compute: each day starts at midnight of the default time zone of the data, - # e.g. at midnight in "Europe/Berlin", and has 23 or 25 hours when switching + # e.g. at midnight in "Europe/Berlin", and has 23 or 25 hours when switching # from or to daylight saving time - timezone = get_series_default_timezone(series) or "UTC" - daily_sums = modify_timezone(series, to_timezone=timezone).resample("1D").sum() + timezone = get_series_default_timezone(series) or "UTC" + daily_sums = modify_timezone(series, to_timezone=timezone).resample("1D").sum() # view: show the timestamps in the time zone of the plot target settings daily_sums = modify_timezone(daily_sums) @@ -235,11 +269,11 @@

Which time zone to compute in?timezones = get_default_timezones(multitsframe) daily_sums = { - metric: group.set_index("timestamp")["value"] - .tz_convert(timezones[metric] or "UTC") - .resample("1D") + metric: group.set_index("timestamp")["value"] + .tz_convert(timezones[metric] or "UTC") + .resample("1D") .sum() - for metric, group in multitsframe.groupby("metric") + for metric, group in multitsframe.groupby("metric") } ... @@ -266,18 +300,18 @@

How to use hdhelpers for plotting? (tbd)series = modify_timezone(series) - fig = go.Figure([go.Scatter(x=series.index, y=series.values, marker={"color": resolve_color("ki.vision")})]) + fig = go.Figure([go.Scatter(x=series.index, y=series.values, marker={"color": resolve_color("ki.vision")})]) start, end = get_queried_interval(series) fig.update_xaxes(range=(start, end)) - full_title = get_series_name(series) or "Level" + full_title = get_series_name(series) or "Level" fig.update_layout(yaxis_title=full_title) set_dt_ticks(fig) set_agnostic_theme(fig) - return {"plot": fig} + return {"plot": fig}

@@ -295,6 +329,7 @@

Table of Contents