From 887d90d1f7d49a25be86eeb66af6d3e33c2d5e41 Mon Sep 17 00:00:00 2001 From: stewit <> Date: Sat, 31 Oct 2026 14:12:48 +0100 Subject: [PATCH] describe metadata models in JSON terms in their JSON schemas The docstrings and field descriptions of the convention models become the descriptions of the JSON Schemas in the openapi.json of the backend. Replace Python terms like None, read_metadata, include_inherited and isoformat with JSON terms and move Python-specific notes to the Sphinx reference. Co-Authored-By: Claude Opus 5.5 --- .../hdhelpers/metadata/conventions.html | 3485 ++++++++--------- docs/_sources/metadata_models.rst.txt | 18 + docs/metadata_models.html | 59 +- docs/searchindex.js | 2 +- sphinx/source/metadata_models.rst | 18 + src/hdhelpers/metadata/conventions.py | 67 +- tests/data/convention_schemas.json | 32 +- 7 files changed, 1851 insertions(+), 1830 deletions(-) diff --git a/docs/_modules/hdhelpers/metadata/conventions.html b/docs/_modules/hdhelpers/metadata/conventions.html index 6749dba..6558a24 100644 --- a/docs/_modules/hdhelpers/metadata/conventions.html +++ b/docs/_modules/hdhelpers/metadata/conventions.html @@ -58,1887 +58,1886 @@

Source code for hdhelpers.metadata.conventions

18The models document the request body of the hetida designer backend endpoint for checking 19metadata. Hence they occur as JSON Schemas in the openapi.json of the backend, so that third party 20projects, e.g. adapters providing or receiving such metadata, can validate against them. - 21Additional fields are allowed at every level. The models are immutable, see ConventionModel. - 22Metadata timestamps are UTC ISO strings with an explicit Z or +00:00 offset. - 23 - 24check_metadata_conventions additionally checks the rules which JSON Schema cannot express, - 25optionally against the actual data. - 26""" - 27 - 28import datetime - 29import difflib - 30import json - 31import math - 32import re - 33from collections import Counter - 34from collections.abc import Mapping - 35from enum import StrEnum - 36from functools import cache, lru_cache - 37from importlib import resources - 38from typing import Annotated, Any, ClassVar, Literal, NoReturn, Self - 39 - 40import pandas as pd - 41from pydantic import ( - 42 AfterValidator, - 43 BaseModel, - 44 BeforeValidator, - 45 ConfigDict, - 46 Field, - 47 GetJsonSchemaHandler, - 48 ModelWrapValidatorHandler, - 49 StrictBool, - 50 StrictFloat, - 51 StrictInt, - 52 StrictStr, - 53 ValidationError, - 54 ValidationInfo, - 55 model_validator, - 56) - 57from pydantic.json_schema import JsonSchemaValue - 58from pydantic_core import CoreSchema, PydanticCustomError - 59 - 60from hdhelpers.metadata._reading import MetadataReading, id_of - 61 - 62TimeseriesDataType = Literal["SERIES", "SINGLETSFRAME", "MULTITSFRAME"] - 63 + 21Their docstrings and field descriptions become the descriptions of these schemas, so they are + 22written for JSON users: null, true and false instead of their Python counterparts and no Python + 23names, e.g. of functions or parameters. + 24Additional fields are allowed at every level. The models are immutable, see ConventionModel. + 25Metadata timestamps are UTC ISO strings with an explicit Z or +00:00 offset. + 26 + 27check_metadata_conventions additionally checks the rules which JSON Schema cannot express, + 28optionally against the actual data. + 29""" + 30 + 31import datetime + 32import difflib + 33import json + 34import math + 35import re + 36from collections import Counter + 37from collections.abc import Mapping + 38from enum import StrEnum + 39from functools import cache, lru_cache + 40from importlib import resources + 41from typing import Annotated, Any, ClassVar, Literal, NoReturn, Self + 42 + 43import pandas as pd + 44from pydantic import ( + 45 AfterValidator, + 46 BaseModel, + 47 BeforeValidator, + 48 ConfigDict, + 49 Field, + 50 GetJsonSchemaHandler, + 51 ModelWrapValidatorHandler, + 52 StrictBool, + 53 StrictFloat, + 54 StrictInt, + 55 StrictStr, + 56 ValidationError, + 57 ValidationInfo, + 58 model_validator, + 59) + 60from pydantic.json_schema import JsonSchemaValue + 61from pydantic_core import CoreSchema, PydanticCustomError + 62 + 63from hdhelpers.metadata._reading import MetadataReading, id_of 64 + 65TimeseriesDataType = Literal["SERIES", "SINGLETSFRAME", "MULTITSFRAME"] + 66 + 67

[docs] - 65class IntervalType(StrEnum): - 66 """Which boundaries belong to a time interval - 67 - 68 closed: both, open: none, left_closed / right_open: only the start, - 69 right_closed / left_open: only the end. - 70 """ - 71 - 72 CLOSED = "closed" - 73 LEFT_CLOSED = "left_closed" - 74 RIGHT_CLOSED = "right_closed" - 75 OPEN = "open" - 76 LEFT_OPEN = "left_open" - 77 RIGHT_OPEN = "right_open"
+ 68class IntervalType(StrEnum): + 69 """Which boundaries belong to a time interval + 70 + 71 closed: both, open: none, left_closed / right_open: only the start, + 72 right_closed / left_open: only the end. + 73 """ + 74 + 75 CLOSED = "closed" + 76 LEFT_CLOSED = "left_closed" + 77 RIGHT_CLOSED = "right_closed" + 78 OPEN = "open" + 79 LEFT_OPEN = "left_open" + 80 RIGHT_OPEN = "right_open"
- 78 - 79 - 80# [0-9] instead of \d, since \d also matches non-ASCII digits in some regex dialects - 81UTC_TIMESTAMP_PATTERN = ( - 82 r"^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])" - 83 r"T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\.[0-9]+)?(Z|\+00:00)$" - 84) - 85 - 86 - 87def _check_point_in_time(timestamp: str) -> str: - 88 """Reject timestamps of the pattern which are no actual points in time, e.g. February 30""" - 89 try: - 90 datetime.datetime.fromisoformat(timestamp) - 91 except ValueError as e: - 92 raise ValueError(f"{timestamp!r} is not an actual point in time: {e}.") from e - 93 return timestamp - 94 - 95 - 96def _utc_isoformat(value: Any) -> Any: - 97 """A datetime with UTC offset 0 as isoformat string, so that producers can pass datetimes""" - 98 if not isinstance(value, datetime.datetime): - 99 return value - 100 if value.utcoffset() is None: - 101 raise ValueError(f"{value!r} is naive, but needs to be in UTC.") - 102 if value.utcoffset() != datetime.timedelta(0): - 103 raise ValueError(f"{value!r} is not in UTC.") - 104 return value.isoformat() - 105 - 106 - 107# The pattern specifies the format in the JSON Schema, which cannot express that the date and - 108# time must exist. Pydantic's datetime type is not used, since it accepts further formats and - 109# offsets, so that the check would be more lenient than the published JSON Schema. - 110UtcTimestamp = Annotated[ - 111 str, - 112 Field(pattern=UTC_TIMESTAMP_PATTERN, json_schema_extra={"format": "date-time"}), - 113 AfterValidator(_check_point_in_time), - 114 # last, so that it runs first and the JSON Schema keeps the pattern - 115 BeforeValidator(_utc_isoformat), - 116] - 117 - 118 + 81 + 82 + 83# [0-9] instead of \d, since \d also matches non-ASCII digits in some regex dialects + 84UTC_TIMESTAMP_PATTERN = ( + 85 r"^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])" + 86 r"T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\.[0-9]+)?(Z|\+00:00)$" + 87) + 88 + 89 + 90def _check_point_in_time(timestamp: str) -> str: + 91 """Reject timestamps of the pattern which are no actual points in time, e.g. February 30""" + 92 try: + 93 datetime.datetime.fromisoformat(timestamp) + 94 except ValueError as e: + 95 raise ValueError(f"{timestamp!r} is not an actual point in time: {e}.") from e + 96 return timestamp + 97 + 98 + 99def _utc_isoformat(value: Any) -> Any: + 100 """A datetime with UTC offset 0 as isoformat string, so that producers can pass datetimes""" + 101 if not isinstance(value, datetime.datetime): + 102 return value + 103 if value.utcoffset() is None: + 104 raise ValueError(f"{value!r} is naive, but needs to be in UTC.") + 105 if value.utcoffset() != datetime.timedelta(0): + 106 raise ValueError(f"{value!r} is not in UTC.") + 107 return value.isoformat() + 108 + 109 + 110# The pattern specifies the format in the JSON Schema, which cannot express that the date and + 111# time must exist. Pydantic's datetime type is not used, since it accepts further formats and + 112# offsets, so that the check would be more lenient than the published JSON Schema. + 113UtcTimestamp = Annotated[ + 114 str, + 115 Field(pattern=UTC_TIMESTAMP_PATTERN, json_schema_extra={"format": "date-time"}), + 116 AfterValidator(_check_point_in_time), + 117 # last, so that it runs first and the JSON Schema keeps the pattern + 118 BeforeValidator(_utc_isoformat), + 119] + 120 + 121
[docs] - 119class AggregationMethod(StrEnum): - 120 """Method for aggregating timeseries values into coarser intervals. - 121 - 122 Metadata lookups normalize AVG to MEAN; they are distinct enum values. - 123 """ + 122class AggregationMethod(StrEnum): + 123 """Method for aggregating timeseries values into coarser intervals. 124 - 125 MEAN = "MEAN" - 126 AVG = "AVG" - 127 MEDIAN = "MEDIAN" - 128 SUM = "SUM" - 129 MIN = "MIN" - 130 MAX = "MAX" - 131 FIRST = "FIRST" - 132 LAST = "LAST" - 133 COUNT = "COUNT"
+ 125 AVG is an alias of MEAN. + 126 """ + 127 + 128 MEAN = "MEAN" + 129 AVG = "AVG" + 130 MEDIAN = "MEDIAN" + 131 SUM = "SUM" + 132 MIN = "MIN" + 133 MAX = "MAX" + 134 FIRST = "FIRST" + 135 LAST = "LAST" + 136 COUNT = "COUNT" - 134 - 135 + 137 + 138
[docs] - 136class ValueDataType(StrEnum): - 137 """Data type of the values of a value dimension""" - 138 - 139 FLOAT = "float" - 140 INTEGER = "integer" - 141 BOOLEAN = "boolean" - 142 STRING = "string"
+ 139class ValueDataType(StrEnum): + 140 """Data type of the values of a value dimension""" + 141 + 142 FLOAT = "float" + 143 INTEGER = "integer" + 144 BOOLEAN = "boolean" + 145 STRING = "string" - 143 - 144 + 146 + 147
[docs] - 145class RecordingMode(StrEnum): - 146 """When the values of a timeseries are recorded - 147 - 148 periodic: one value per interval of data_frequency, timestamped with the start of the - 149 interval. A missing value is a gap. - 150 on_change: a value is recorded when it changes (by more than the deadband of its value - 151 dimension) and holds until the next one. Missing timestamps are no gaps, as long as - 152 data_frequency (the heartbeat) is not exceeded. - 153 irregular: values at arbitrary points in time. - 154 """ - 155 - 156 PERIODIC = "periodic" - 157 ON_CHANGE = "on_change" - 158 IRREGULAR = "irregular"
+ 148class RecordingMode(StrEnum): + 149 """When the values of a timeseries are recorded + 150 + 151 periodic: one value per interval of data_frequency, timestamped with the start of the + 152 interval. A missing value is a gap. + 153 on_change: a value is recorded when it changes (by more than the deadband of its value + 154 dimension) and holds until the next one. Missing timestamps are no gaps, as long as + 155 data_frequency (the heartbeat) is not exceeded. + 156 irregular: values at arbitrary points in time. + 157 """ + 158 + 159 PERIODIC = "periodic" + 160 ON_CHANGE = "on_change" + 161 IRREGULAR = "irregular" - 159 - 160 - 161_FREQUENCY_FORMAT = 'as Pandas date offset alias or timedelta string, e.g. "15min" or "1D"' - 162_OFFSET_DESCRIPTION = ( - 163 "Offset of the timestamps against the starts of the intervals of {frequency} (see" - 164 ' {timezone}) as timedelta string, e.g. "4min": not negative and smaller than {frequency}. 0' - 165 " if missing or null. Requires {frequency}." - 166) - 167 - 168_TIMEZONE_FORMAT = ( - 169 'as IANA time zone name like "Europe/Berlin" or "UTC" (preferred, since it respects daylight' - 170 ' saving time) or as fixed UTC offset like "+02:00"' - 171) - 172 - 173_ALIGNMENT_DESCRIPTION = ( - 174 "Time zone the intervals of {frequency} are aligned to, " - 175 + _TIMEZONE_FORMAT - 176 + ". Without it, the intervals start at the multiples of {frequency} since" - 177 " 1970-01-01T00:00:00Z. With it, for a {frequency} shorter than a day, the intervals restart" - 178 " at each local midnight: They start at local midnight plus multiples of {frequency}, and the" - 179 " last interval of a day ends at the next local midnight, so it may be shorter, e.g. on the" - 180 " day of a daylight saving time change. For a {frequency} of a day or longer, the intervals" - 181 " follow the calendar of the time zone, e.g. days from local midnight to local midnight. The" - 182 " offset shifts the intervals in both cases. Requires {frequency}." - 183) - 184_TIMEZONE_PURPOSE = ( - 185 " Components evaluating the timeseries in calendar intervals use it, e.g. a report" - 186 " aggregating days, which start at midnight of this time zone, so the results differ between" - 187 " time zones. This is not the time zone timestamps are displayed in: That is the plot target" - 188 " timezone of the execution context, which depends on the viewer, while this time zone belongs" - 189 " to the data, e.g. the time zone of the plant." - 190) - 191 - 192 - 193def _immutable(self: Any, *_args: Any, **_kwargs: Any) -> NoReturn: - 194 raise TypeError(f"{type(self).__name__} is immutable") + 162 + 163 + 164_FREQUENCY_FORMAT = 'as Pandas date offset alias or timedelta string, e.g. "15min" or "1D"' + 165_OFFSET_DESCRIPTION = ( + 166 "Offset of the timestamps against the starts of the intervals of {frequency} (see" + 167 ' {timezone}) as timedelta string, e.g. "4min": not negative and smaller than {frequency}. 0' + 168 " if missing or null. Requires {frequency}." + 169) + 170 + 171_TIMEZONE_FORMAT = ( + 172 'as IANA time zone name like "Europe/Berlin" or "UTC" (preferred, since it respects daylight' + 173 ' saving time) or as fixed UTC offset like "+02:00"' + 174) + 175 + 176_ALIGNMENT_DESCRIPTION = ( + 177 "Time zone the intervals of {frequency} are aligned to, " + 178 + _TIMEZONE_FORMAT + 179 + ". Without it, the intervals start at the multiples of {frequency} since" + 180 " 1970-01-01T00:00:00Z. With it, for a {frequency} shorter than a day, the intervals restart" + 181 " at each local midnight: They start at local midnight plus multiples of {frequency}, and the" + 182 " last interval of a day ends at the next local midnight, so it may be shorter, e.g. on the" + 183 " day of a daylight saving time change. For a {frequency} of a day or longer, the intervals" + 184 " follow the calendar of the time zone, e.g. days from local midnight to local midnight. The" + 185 " offset shifts the intervals in both cases. Requires {frequency}." + 186) + 187_TIMEZONE_PURPOSE = ( + 188 " Components evaluating the timeseries in calendar intervals use it, e.g. a report" + 189 " aggregating days, which start at midnight of this time zone, so the results differ between" + 190 " time zones. This is not the time zone timestamps are displayed in: That is the plot target" + 191 " timezone of the execution context, which depends on the viewer, while this time zone belongs" + 192 " to the data, e.g. the time zone of the plant." + 193) + 194 195 - 196 - 197class FrozenDict(dict[str, Any]): - 198 """A dict which cannot be changed + 196def _immutable(self: Any, *_args: Any, **_kwargs: Any) -> NoReturn: + 197 raise TypeError(f"{type(self).__name__} is immutable") + 198 199 - 200 Holds dict values of extra fields of the convention models, since the models are shared - 201 between pandas objects instead of being copied. Use it as a read-only mapping, not as a dict: - 202 The builtin frozendict of Python 3.15, which is planned to replace it, is no dict subclass. - 203 """ - 204 - 205 __slots__ = () - 206 __setitem__ = __delitem__ = __ior__ = _immutable # type: ignore[assignment] - 207 clear = pop = popitem = setdefault = update = _immutable # type: ignore[assignment] - 208 - 209 def __copy__(self) -> Self: - 210 return self + 200class FrozenDict(dict[str, Any]): + 201 """A dict which cannot be changed + 202 + 203 Holds dict values of extra fields of the convention models, since the models are shared + 204 between pandas objects instead of being copied. Use it as a read-only mapping, not as a dict: + 205 The builtin frozendict of Python 3.15, which is planned to replace it, is no dict subclass. + 206 """ + 207 + 208 __slots__ = () + 209 __setitem__ = __delitem__ = __ior__ = _immutable # type: ignore[assignment] + 210 clear = pop = popitem = setdefault = update = _immutable # type: ignore[assignment] 211 - 212 def __deepcopy__(self, memo: dict[int, Any]) -> Self: + 212 def __copy__(self) -> Self: 213 return self 214 - 215 def __reduce__(self) -> tuple[Any, ...]: - 216 return (FrozenDict, (dict(self),)) + 215 def __deepcopy__(self, memo: dict[int, Any]) -> Self: + 216 return self 217 - 218 - 219def _freeze(value: Any) -> Any: - 220 """A JSON value with dicts and lists as FrozenDicts and tuples, recursively""" - 221 if isinstance(value, dict): - 222 if not all(isinstance(key, str) for key in value): - 223 raise ValueError("Keys of JSON objects must be strings.") - 224 return FrozenDict({key: _freeze(item) for key, item in value.items()}) - 225 if isinstance(value, list | tuple): - 226 return tuple(_freeze(item) for item in value) - 227 if ( - 228 value is None - 229 or isinstance(value, str | int) - 230 or (isinstance(value, float) and math.isfinite(value)) - 231 ): - 232 return value - 233 raise ValueError(f"{value!r} is not a JSON value.") - 234 - 235 - 236_JsonFloat = Annotated[StrictFloat, Field(allow_inf_nan=False)] + 218 def __reduce__(self) -> tuple[Any, ...]: + 219 return (FrozenDict, (dict(self),)) + 220 + 221 + 222def _freeze(value: Any) -> Any: + 223 """A JSON value with dicts and lists as FrozenDicts and tuples, recursively""" + 224 if isinstance(value, dict): + 225 if not all(isinstance(key, str) for key in value): + 226 raise ValueError("Keys of JSON objects must be strings.") + 227 return FrozenDict({key: _freeze(item) for key, item in value.items()}) + 228 if isinstance(value, list | tuple): + 229 return tuple(_freeze(item) for item in value) + 230 if ( + 231 value is None + 232 or isinstance(value, str | int) + 233 or (isinstance(value, float) and math.isfinite(value)) + 234 ): + 235 return value + 236 raise ValueError(f"{value!r} is not a JSON value.") 237 - 238# Scalars are validated within pydantic-core. Only dicts and lists need Python for freezing them. - 239_FrozenValue = Annotated[ - 240 StrictStr - 241 | StrictInt - 242 | _JsonFloat - 243 | StrictBool - 244 | None - 245 | Annotated[Any, AfterValidator(_freeze)], - 246 Field(union_mode="left_to_right"), - 247] - 248 - 249 - 250def optional(default: Any = None, **kwargs: Any) -> Any: - 251 """Field for an optional value, left out of dumps while it is None or the default""" - 252 return Field(default, exclude_if=lambda value: value is None or value == default, **kwargs) - 253 - 254 - 255def _item_count(min_length: int, max_length: int | None = None) -> BeforeValidator: - 256 """Check the number of items before validating them + 238 + 239_JsonFloat = Annotated[StrictFloat, Field(allow_inf_nan=False)] + 240 + 241# Scalars are validated within pydantic-core. Only dicts and lists need Python for freezing them. + 242_FrozenValue = Annotated[ + 243 StrictStr + 244 | StrictInt + 245 | _JsonFloat + 246 | StrictBool + 247 | None + 248 | Annotated[Any, AfterValidator(_freeze)], + 249 Field(union_mode="left_to_right"), + 250] + 251 + 252 + 253def optional(default: Any = None, **kwargs: Any) -> Any: + 254 """Field for an optional value, left out of dumps while it is None or the default""" + 255 return Field(default, exclude_if=lambda value: value is None or value == default, **kwargs) + 256 257 - 258 Pydantic counts the items of a tuple after validating them, so that an invalid item would - 259 also be reported as missing. The JSON Schema states the counts via json_schema_extra. - 260 """ - 261 - 262 def items(count: int) -> str: - 263 return f"{count} item" if count == 1 else f"{count} items" + 258def _item_count(min_length: int, max_length: int | None = None) -> BeforeValidator: + 259 """Check the number of items before validating them + 260 + 261 Pydantic counts the items of a tuple after validating them, so that an invalid item would + 262 also be reported as missing. The JSON Schema states the counts via json_schema_extra. + 263 """ 264 - 265 def check(value: Any) -> Any: - 266 if isinstance(value, list | tuple): - 267 if len(value) < min_length: - 268 raise PydanticCustomError( - 269 "too_short", f"Should have at least {items(min_length)}, not {len(value)}" - 270 ) - 271 if max_length is not None and len(value) > max_length: - 272 raise PydanticCustomError( - 273 "too_long", f"Should have at most {items(max_length)}, not {len(value)}" - 274 ) - 275 # a tuple, since strict validation after a before validator rejects lists - 276 return tuple(value) - 277 return value - 278 - 279 return BeforeValidator(check) - 280 + 265 def items(count: int) -> str: + 266 return f"{count} item" if count == 1 else f"{count} items" + 267 + 268 def check(value: Any) -> Any: + 269 if isinstance(value, list | tuple): + 270 if len(value) < min_length: + 271 raise PydanticCustomError( + 272 "too_short", f"Should have at least {items(min_length)}, not {len(value)}" + 273 ) + 274 if max_length is not None and len(value) > max_length: + 275 raise PydanticCustomError( + 276 "too_long", f"Should have at most {items(max_length)}, not {len(value)}" + 277 ) + 278 # a tuple, since strict validation after a before validator rejects lists + 279 return tuple(value) + 280 return value 281 - 282class ConventionModel(BaseModel): - 283 """Base of the models of the metadata conventions + 282 return BeforeValidator(check) + 283 284 - 285 The models are immutable, including the values of extra fields, so that pandas objects can - 286 share them instead of copying them: pandas deep-copies the attrs of an object for (almost) - 287 every object derived from it. A changed version of a model is a new model, which shares all - 288 unchanged parts with the original. - 289 """ - 290 - 291 # Validation is lax, so that Python input may contain lists for the tuples. Fields whose type - 292 # lax validation would coerce (bool, int, float) use the strict types instead, so that the - 293 # results equal strict validation of JSON input. - 294 model_config = ConfigDict(extra="allow", frozen=True) - 295 __pydantic_extra__: dict[str, _FrozenValue] - 296 - 297 def __deepcopy__(self, memo: dict[int, Any] | None = None) -> Self: - 298 return self + 285class ConventionModel(BaseModel): + 286 """Base of the models of the metadata conventions + 287 + 288 The models are immutable, including the values of extra fields, so that pandas objects can + 289 share them instead of copying them: pandas deep-copies the attrs of an object for (almost) + 290 every object derived from it. A changed version of a model is a new model, which shares all + 291 unchanged parts with the original. + 292 """ + 293 + 294 # Validation is lax, so that Python input may contain lists for the tuples. Fields whose type + 295 # lax validation would coerce (bool, int, float) use the strict types instead, so that the + 296 # results equal strict validation of JSON input. + 297 model_config = ConfigDict(extra="allow", frozen=True) + 298 __pydantic_extra__: dict[str, _FrozenValue] 299 - 300 def model_copy( - 301 self, - 302 *, - 303 update: Mapping[str, Any] | None = None, - 304 deep: bool = False, # noqa: ARG002 - 305 ) -> Self: - 306 """Copy with the given fields replaced, without validating them - 307 - 308 Always shallow, since the model is immutable. A deep copy would apply the update to the - 309 original, since __deepcopy__ returns the original. - 310 """ - 311 return super().model_copy(update=update, deep=False) - 312 - 313 @classmethod - 314 def __get_pydantic_json_schema__( - 315 cls, core_schema: CoreSchema, handler: GetJsonSchemaHandler - 316 ) -> JsonSchemaValue: - 317 json_schema = handler(core_schema) - 318 # Any JSON value is allowed in extra fields, as without the typed __pydantic_extra__ - 319 handler.resolve_ref_schema(json_schema)["additionalProperties"] = True - 320 return json_schema - 321 - 322 + 300 def __deepcopy__(self, memo: dict[int, Any] | None = None) -> Self: + 301 return self + 302 + 303 def model_copy( + 304 self, + 305 *, + 306 update: Mapping[str, Any] | None = None, + 307 deep: bool = False, # noqa: ARG002 + 308 ) -> Self: + 309 """Copy with the given fields replaced, without validating them + 310 + 311 Always shallow, since the model is immutable. A deep copy would apply the update to the + 312 original, since __deepcopy__ returns the original. + 313 """ + 314 return super().model_copy(update=update, deep=False) + 315 + 316 @classmethod + 317 def __get_pydantic_json_schema__( + 318 cls, core_schema: CoreSchema, handler: GetJsonSchemaHandler + 319 ) -> JsonSchemaValue: + 320 json_schema = handler(core_schema) + 321 # Any JSON value is allowed in extra fields, as without the typed __pydantic_extra__ + 322 handler.resolve_ref_schema(json_schema)["additionalProperties"] = True + 323 return json_schema + 324 + 325
[docs] - 323class SharedValueDimensionMetadata(ConventionModel): - 324 """Default metadata for a value column shared across metrics. - 325 - 326 Matched by column name ("value" for a SERIES) and used when the corresponding field - 327 is missing from metric-specific metadata. - 328 """ - 329 - 330 column: str = Field( - 331 ..., - 332 description="The actual column name in the MULTITSFRAME or SINGLETSFRAME, connecting the" - 333 ' value dimension to its column. "value" for a SERIES.', - 334 ) - 335 name: str | None = optional( - 336 description='Name of the value dimension, e.g. "temperature". May equal column.' + 326class SharedValueDimensionMetadata(ConventionModel): + 327 """Default metadata for a value column shared across metrics. + 328 + 329 Matched by column name ("value" for a SERIES) and used when the corresponding field + 330 is missing from metric-specific metadata. + 331 """ + 332 + 333 column: str = Field( + 334 ..., + 335 description="The actual column name in the MULTITSFRAME or SINGLETSFRAME, connecting the" + 336 ' value dimension to its column. "value" for a SERIES.', 337 ) - 338 display_name: str | None = optional( - 339 description="A shorter name for labels, captions or plot titles. Falls back to name.", + 338 name: str | None = optional( + 339 description='Name of the value dimension, e.g. "temperature". May equal column.' 340 ) - 341 short_display_name: str | None = optional( - 342 description="A short name for legends in plots. Falls back to display_name." + 341 display_name: str | None = optional( + 342 description="A shorter name for labels, captions or plot titles. Falls back to name.", 343 ) - 344 description: str | None = optional( - 345 description="A description. Should be considered empty if missing." + 344 short_display_name: str | None = optional( + 345 description="A short name for legends in plots. Falls back to display_name." 346 ) - 347 unit: str | None = optional( - 348 description='Unit of the values, e.g. "°C" or "m/s", "" for dimensionless quantities', + 347 description: str | None = optional( + 348 description="A description. Should be considered empty if missing." 349 ) - 350 measurement: str | None = optional(description='The measured quantity, e.g. "temperature"') - 351 value_data_type: ValueDataType | None = optional(description="Data type of the values") - 352 deadband: StrictFloat | None = optional( - 353 ge=0, - 354 allow_inf_nan=False, - 355 description="For a metric with recording_mode on_change: the change of the values, in" - 356 " their unit, which has to be exceeded for a value to be recorded. 0 records every" - 357 " change.", - 358 ) - 359 decimals: StrictInt | None = optional( - 360 ge=0, - 361 description="Number of decimal places numeric values should be displayed with, in their" - 362 " unit, e.g. 2 for 12.35. Only affects how values are displayed, e.g. in tables, tooltips" - 363 " or reports; the values themselves are not rounded. Up to the consumer if missing or" - 364 " null.", - 365 )
+ 350 unit: str | None = optional( + 351 description='Unit of the values, e.g. "°C" or "m/s", "" for dimensionless quantities', + 352 ) + 353 measurement: str | None = optional(description='The measured quantity, e.g. "temperature"') + 354 value_data_type: ValueDataType | None = optional(description="Data type of the values") + 355 deadband: StrictFloat | None = optional( + 356 ge=0, + 357 allow_inf_nan=False, + 358 description="For a metric with recording_mode on_change: the change of the values, in" + 359 " their unit, which has to be exceeded for a value to be recorded. 0 records every" + 360 " change.", + 361 ) + 362 decimals: StrictInt | None = optional( + 363 ge=0, + 364 description="Number of decimal places numeric values should be displayed with, in their" + 365 " unit, e.g. 2 for 12.35. Only affects how values are displayed, e.g. in tables, tooltips" + 366 " or reports; the values themselves are not rounded. Up to the consumer if missing or" + 367 " null.", + 368 ) - 366 - 367 + 369 + 370
[docs] - 368class ValueDimensionMetadata(SharedValueDimensionMetadata): - 369 """Metadata for one metric's value column, such as its unit and measured quantity. - 370 - 371 Each metric describes its value columns; a SERIES uses "value". Non-null fields take - 372 precedence over shared metadata for the same column. Unit and measurement are required - 373 fields, but may be None if unknown. - 374 """ - 375 - 376 unit: str | None = Field( - 377 ..., - 378 description='Unit of the values, e.g. "°C" or "m/s". "" for dimensionless quantities, null' - 379 " if unknown.", - 380 ) - 381 measurement: str | None = Field( - 382 ..., description='The measured quantity, e.g. "temperature". Null if unknown.' - 383 )
+ 371class ValueDimensionMetadata(SharedValueDimensionMetadata): + 372 """Metadata for one metric's value column, such as its unit and measured quantity. + 373 + 374 Each metric describes its value columns; a SERIES uses "value". Non-null fields take + 375 precedence over shared metadata for the same column. Unit and measurement are required + 376 fields, but may be null if unknown. + 377 """ + 378 + 379 unit: str | None = Field( + 380 ..., + 381 description='Unit of the values, e.g. "°C" or "m/s". "" for dimensionless quantities, null' + 382 " if unknown.", + 383 ) + 384 measurement: str | None = Field( + 385 ..., description='The measured quantity, e.g. "temperature". Null if unknown.' + 386 ) - 384 - 385 + 387 + 388
[docs] - 386class InheritedMetricMetadata(ConventionModel): - 387 """Properties supplied by a source system for a metric and its parent assets. - 388 - 389 Stores already-resolved values, e.g. a plant's latitude; the source system resolves the - 390 hierarchy. Metadata lookups use these properties as fallbacks only with - 391 ``include_inherited=True``. Convention-defined fields such as name, id and unit belong in - 392 the metric or value-dimension metadata. - 393 - 394 Use JSON types matching each value's meaning: numbers, booleans and UTC ISO timestamp - 395 strings for points in time. For example, hetida platform supplies properties by label, - 396 using a signal's own value or that of the nearest parent asset providing one. - 397 """
+ 389class InheritedMetricMetadata(ConventionModel): + 390 """Properties supplied by a source system for a metric and its parent assets. + 391 + 392 Stores already-resolved values, e.g. a plant's latitude; the source system resolves the + 393 hierarchy. Consumers use these properties only as last resort and not for information + 394 describing the metric itself, like its unit or names. Convention-defined fields such as name, + 395 id and unit belong in the metric or value-dimension metadata. + 396 + 397 Use JSON types matching each value's meaning: numbers, booleans and UTC ISO timestamp + 398 strings for points in time. For example, hetida platform supplies properties by label, + 399 using a signal's own value or that of the nearest parent asset providing one. + 400 """ - 398 - 399 + 401 + 402
[docs] - 400class MetricCommentReply(ConventionModel): - 401 """A reply to a comment attached to a metric""" - 402 - 403 id: str | None = optional(description="Id of the reply, e.g. a UUID") - 404 created_at: UtcTimestamp | None = optional(description="When the reply was created") - 405 edited_at: UtcTimestamp | None = optional( - 406 description="When the reply was last edited. Null or missing if never edited." - 407 ) - 408 content: str | None = optional(description="Free text of the reply") - 409 username: str | None = optional(description="Username of the author of the reply")
+ 403class MetricCommentReply(ConventionModel): + 404 """A reply to a comment attached to a metric""" + 405 + 406 id: str | None = optional(description="Id of the reply, e.g. a UUID") + 407 created_at: UtcTimestamp | None = optional(description="When the reply was created") + 408 edited_at: UtcTimestamp | None = optional( + 409 description="When the reply was last edited. Null or missing if never edited." + 410 ) + 411 content: str | None = optional(description="Free text of the reply") + 412 username: str | None = optional(description="Username of the author of the reply") - 410 - 411 + 413 + 414
[docs] - 412class MetricComment(ConventionModel): - 413 """A comment attached to a metric, describing a time range of its timeseries data - 414 - 415 E.g. the hetida platform provides the comments on a signal whose time range overlaps the - 416 queried interval and which existed at query time, newest first. + 415class MetricComment(ConventionModel): + 416 """A comment attached to a metric, describing a time range of its timeseries data 417 - 418 Start and end of the time range must be provided together. They are equal for a comment on a - 419 single point in time. - 420 """ - 421 - 422 id: str | None = optional(description="Id of the comment, e.g. a UUID") - 423 start_date: UtcTimestamp | None = optional( - 424 description="Start of the time range the comment refers to" - 425 ) - 426 end_date: UtcTimestamp | None = optional( - 427 description="End of the time range the comment refers to" + 418 E.g. the hetida platform provides the comments on a signal whose time range overlaps the + 419 queried interval and which existed at query time, newest first. + 420 + 421 Start and end of the time range must be provided together. They are equal for a comment on a + 422 single point in time. + 423 """ + 424 + 425 id: str | None = optional(description="Id of the comment, e.g. a UUID") + 426 start_date: UtcTimestamp | None = optional( + 427 description="Start of the time range the comment refers to" 428 ) - 429 created_at: UtcTimestamp | None = optional(description="When the comment was created") - 430 edited_at: UtcTimestamp | None = optional( - 431 description="When the comment was last edited. Null or missing if never edited." - 432 ) - 433 content: str | None = optional(description="Free text of the comment") - 434 username: str | None = optional(description="Username of the author of the comment") - 435 tags: tuple[str, ...] | None = optional(description="Free text tags classifying the comment") - 436 replies: tuple[MetricCommentReply, ...] | None = optional( - 437 description="Replies to the comment, oldest first" - 438 )
+ 429 end_date: UtcTimestamp | None = optional( + 430 description="End of the time range the comment refers to" + 431 ) + 432 created_at: UtcTimestamp | None = optional(description="When the comment was created") + 433 edited_at: UtcTimestamp | None = optional( + 434 description="When the comment was last edited. Null or missing if never edited." + 435 ) + 436 content: str | None = optional(description="Free text of the comment") + 437 username: str | None = optional(description="Username of the author of the comment") + 438 tags: tuple[str, ...] | None = optional(description="Free text tags classifying the comment") + 439 replies: tuple[MetricCommentReply, ...] | None = optional( + 440 description="Replies to the comment, oldest first" + 441 ) - 439 - 440 + 442 + 443
[docs] - 441class MetricMetadata(ConventionModel): - 442 """Metadata of a metric, i.e. of a single, possibly multidimensional timeseries - 443 - 444 A metric is identified by its field named by dataset_metadata.metric_key, whose values occur - 445 in the metric column of a MULTITSFRAME. Every metric must have this field as a string, - 446 unique among the metrics. - 447 """ - 448 - 449 id: str = Field( - 450 ..., - 451 description="Id of the metric, also if another field identifies the metric in the data" - 452 " via dataset_metadata.metric_key. E.g. the hetida platform provides the signal id here.", - 453 ) - 454 name: str = Field(..., description="Long name of the metric") - 455 display_name: str | None = optional( - 456 description="A shorter name for labels, captions or plot titles" - 457 ) - 458 short_display_name: str | None = optional(description="A short name for legends in plots") - 459 description: str | None = optional(description="A one sentence description") - 460 external_id: str | None = optional( - 461 description="Unique id / key in external source systems. Often used to identify the" - 462 " metric in the metric column, i.e. as metric_key.", - 463 ) - 464 signal_id: str | None = optional(description="Actual platform signal id") - 465 channel_id: str | None = optional( - 466 description="Actual platform channel id. Deprecated, since the platform calls channels" - 467 " signals now.", - 468 deprecated='Use "signal_id" instead.', - 469 ) - 470 value_dimensions: Annotated[tuple[ValueDimensionMetadata, ...], _item_count(1)] = Field( - 471 ..., - 472 json_schema_extra={"minItems": 1}, - 473 description="Metadata of the value dimensions of this metric: an entry for each value" - 474 ' column of the data, i.e. exactly the value dimension "value" for a SERIES. Each column' - 475 " may occur at most once.", - 476 ) - 477 default_aggregation_method: AggregationMethod | None = optional( - 478 description="How the timeseries should be aggregated by default, e.g. when components" - 479 " plot or report it in coarser intervals. If missing or null: LAST if the value_data_type" - 480 ' of the value dimension "value" is boolean or string, otherwise MEAN. AVG is an alias' - 481 " of MEAN. E.g. the hetida platform provides the method it aggregates the signal with when" - 482 " bucketing it, for a signal group member the one configured for the member.", - 483 ) - 484 default_timezone: str | None = optional( - 485 description="Time zone the timeseries should be evaluated in by default, " - 486 + _TIMEZONE_FORMAT - 487 + "." - 488 + _TIMEZONE_PURPOSE - 489 + " Falls back to dataset_metadata.default_timezone.", - 490 ) - 491 path: tuple[str, ...] | None = optional( - 492 description="Location of the metric in the hierarchy of its source system: the names of" - 493 " the levels from the top down to the object the metric belongs to, without the metric" - 494 ' itself, e.g. ["Plant A", "Pickling Unit", "Influx"]. Corresponds to the breadcrumb' - 495 " path of the sources of generic REST adapters.", - 496 ) - 497 origin: str | None = optional( - 498 description="Where the values come from, as free text of the source system, e.g." - 499 ' "OPC_UA", "MQTT" or "FORM" for values entered manually.', - 500 ) - 501 recording_mode: RecordingMode | None = optional( - 502 description="When the values are recorded. Unknown if missing or null. periodic requires" - 503 " data_frequency (or dataset_metadata.ref_data_frequency).", - 504 ) - 505 data_frequency: str | None = optional( - 506 description="Interval in which values of the metric are recorded " - 507 + _FREQUENCY_FORMAT - 508 + ": For recording_mode periodic the interval between subsequent values, for on_change" - 509 " the interval after which the value is recorded again at the latest, also without a" - 510 " change (heartbeat). Together with data_frequency_offset and data_frequency_timezone it" - 511 " takes precedence over ref_data_frequency, ref_data_frequency_offset and" - 512 " ref_data_frequency_timezone of the dataset metadata for this metric.", - 513 ) - 514 data_frequency_offset: str | None = optional( - 515 description=_OFFSET_DESCRIPTION.format( - 516 frequency="data_frequency", timezone="data_frequency_timezone" - 517 ), - 518 ) - 519 data_frequency_timezone: str | None = optional( - 520 description=_ALIGNMENT_DESCRIPTION.format(frequency="data_frequency") + 444class MetricMetadata(ConventionModel): + 445 """Metadata of a metric, i.e. of a single, possibly multidimensional timeseries + 446 + 447 A metric is identified by its field named by dataset_metadata.metric_key, whose values occur + 448 in the metric column of a MULTITSFRAME. Every metric must have this field as a string, + 449 unique among the metrics. + 450 """ + 451 + 452 id: str = Field( + 453 ..., + 454 description="Id of the metric, also if another field identifies the metric in the data" + 455 " via dataset_metadata.metric_key. E.g. the hetida platform provides the signal id here.", + 456 ) + 457 name: str = Field(..., description="Long name of the metric") + 458 display_name: str | None = optional( + 459 description="A shorter name for labels, captions or plot titles" + 460 ) + 461 short_display_name: str | None = optional(description="A short name for legends in plots") + 462 description: str | None = optional(description="A one sentence description") + 463 external_id: str | None = optional( + 464 description="Unique id / key in external source systems. Often used to identify the" + 465 " metric in the metric column, i.e. as metric_key.", + 466 ) + 467 signal_id: str | None = optional(description="Actual platform signal id") + 468 channel_id: str | None = optional( + 469 description="Actual platform channel id. Deprecated, since the platform calls channels" + 470 " signals now.", + 471 deprecated='Use "signal_id" instead.', + 472 ) + 473 value_dimensions: Annotated[tuple[ValueDimensionMetadata, ...], _item_count(1)] = Field( + 474 ..., + 475 json_schema_extra={"minItems": 1}, + 476 description="Metadata of the value dimensions of this metric: an entry for each value" + 477 ' column of the data, i.e. exactly the value dimension "value" for a SERIES. Each column' + 478 " may occur at most once.", + 479 ) + 480 default_aggregation_method: AggregationMethod | None = optional( + 481 description="How the timeseries should be aggregated by default, e.g. when components" + 482 " plot or report it in coarser intervals. If missing or null: LAST if the value_data_type" + 483 ' of the value dimension "value" is boolean or string, otherwise MEAN. AVG is an alias' + 484 " of MEAN. E.g. the hetida platform provides the method it aggregates the signal with when" + 485 " bucketing it, for a signal group member the one configured for the member.", + 486 ) + 487 default_timezone: str | None = optional( + 488 description="Time zone the timeseries should be evaluated in by default, " + 489 + _TIMEZONE_FORMAT + 490 + "." + 491 + _TIMEZONE_PURPOSE + 492 + " Falls back to dataset_metadata.default_timezone.", + 493 ) + 494 path: tuple[str, ...] | None = optional( + 495 description="Location of the metric in the hierarchy of its source system: the names of" + 496 " the levels from the top down to the object the metric belongs to, without the metric" + 497 ' itself, e.g. ["Plant A", "Pickling Unit", "Influx"]. Corresponds to the breadcrumb' + 498 " path of the sources of generic REST adapters.", + 499 ) + 500 origin: str | None = optional( + 501 description="Where the values come from, as free text of the source system, e.g." + 502 ' "OPC_UA", "MQTT" or "FORM" for values entered manually.', + 503 ) + 504 recording_mode: RecordingMode | None = optional( + 505 description="When the values are recorded. Unknown if missing or null. periodic requires" + 506 " data_frequency (or dataset_metadata.ref_data_frequency).", + 507 ) + 508 data_frequency: str | None = optional( + 509 description="Interval in which values of the metric are recorded " + 510 + _FREQUENCY_FORMAT + 511 + ": For recording_mode periodic the interval between subsequent values, for on_change" + 512 " the interval after which the value is recorded again at the latest, also without a" + 513 " change (heartbeat). Together with data_frequency_offset and data_frequency_timezone it" + 514 " takes precedence over ref_data_frequency, ref_data_frequency_offset and" + 515 " ref_data_frequency_timezone of the dataset metadata for this metric.", + 516 ) + 517 data_frequency_offset: str | None = optional( + 518 description=_OFFSET_DESCRIPTION.format( + 519 frequency="data_frequency", timezone="data_frequency_timezone" + 520 ), 521 ) - 522 recording_aggregation_method: AggregationMethod | None = optional( - 523 description="Only for recording_mode periodic: how each value aggregates the values of" - 524 " the interval [timestamp, timestamp + data_frequency), e.g. MEAN for an average or MAX" - 525 " for a maximum. Null or missing if the values are samples, e.g. the value holding at" - 526 " the timestamp.", - 527 ) - 528 inherited: InheritedMetricMetadata | None = optional() - 529 comments: tuple[MetricComment, ...] | None = optional( - 530 description="Comments attached to the metric, describing situations in its timeseries data", - 531 )
+ 522 data_frequency_timezone: str | None = optional( + 523 description=_ALIGNMENT_DESCRIPTION.format(frequency="data_frequency") + 524 ) + 525 recording_aggregation_method: AggregationMethod | None = optional( + 526 description="Only for recording_mode periodic: how each value aggregates the values of" + 527 " the interval [timestamp, timestamp + data_frequency), e.g. MEAN for an average or MAX" + 528 " for a maximum. Null or missing if the values are samples, e.g. the value holding at" + 529 " the timestamp.", + 530 ) + 531 inherited: InheritedMetricMetadata | None = optional() + 532 comments: tuple[MetricComment, ...] | None = optional( + 533 description="Comments attached to the metric, describing situations in its timeseries data", + 534 ) - 532 - 533 + 535 + 536
[docs] - 534class SignalGroupMetadata(ConventionModel): - 535 """A group of signals a dataset was read from - 536 - 537 E.g. the hetida platform provides the signal group a MULTITSFRAME was read from. - 538 """ + 537class SignalGroupMetadata(ConventionModel): + 538 """A group of signals a dataset was read from 539 - 540 id: str | None = optional(description="Id of the signal group") - 541 name: str | None = optional(description="Name of the signal group") - 542 description: str | None = optional(description="Description of the signal group")
+ 540 E.g. the hetida platform provides the signal group a MULTITSFRAME was read from. + 541 """ + 542 + 543 id: str | None = optional(description="Id of the signal group") + 544 name: str | None = optional(description="Name of the signal group") + 545 description: str | None = optional(description="Description of the signal group") - 543 - 544 - 545REF_INTERVAL_FIELDS = ( - 546 "ref_interval_start_timestamp", - 547 "ref_interval_end_timestamp", - 548 "ref_interval_type", - 549) - 550 - 551# JSON Schema of the rule that the ref interval is required unless ref_dataset_discrete is true - 552_REF_INTERVAL_REQUIRED_SCHEMA: dict[str, Any] = { - 553 "if": { - 554 "properties": {"ref_dataset_discrete": {"const": True}}, - 555 "required": ["ref_dataset_discrete"], - 556 }, - 557 "else": { - 558 "required": list(REF_INTERVAL_FIELDS), - 559 "properties": {field: {"type": "string"} for field in REF_INTERVAL_FIELDS}, - 560 }, - 561} - 562 - 563 - 564class _DatasetMetadataBase(ConventionModel): - 565 model_config = ConfigDict(json_schema_extra=_REF_INTERVAL_REQUIRED_SCHEMA) + 546 + 547 + 548REF_INTERVAL_FIELDS = ( + 549 "ref_interval_start_timestamp", + 550 "ref_interval_end_timestamp", + 551 "ref_interval_type", + 552) + 553 + 554# JSON Schema of the rule that the ref interval is required unless ref_dataset_discrete is true + 555_REF_INTERVAL_REQUIRED_SCHEMA: dict[str, Any] = { + 556 "if": { + 557 "properties": {"ref_dataset_discrete": {"const": True}}, + 558 "required": ["ref_dataset_discrete"], + 559 }, + 560 "else": { + 561 "required": list(REF_INTERVAL_FIELDS), + 562 "properties": {field: {"type": "string"} for field in REF_INTERVAL_FIELDS}, + 563 }, + 564} + 565 566 - 567 metric_key: str = Field( - 568 ..., - 569 description="Which field of the metric objects under metrics identifies a metric, i.e." - 570 ' occurs in the metric column of a MULTITSFRAME, e.g. "id" or "external_id".', - 571 ) - 572 queried_metrics: tuple[str, ...] = Field( - 573 ..., - 574 description="The requested metrics. Since not all queried metrics may have data in the" - 575 " queried interval, a metric may be missing in the data.", - 576 ) - 577 signal_group: SignalGroupMetadata | None = optional() - 578 ref_dataset_discrete: StrictBool | None = optional( - 579 False, - 580 description="Whether the ref dataset should be considered as discrete, i.e. isolated" - 581 " datapoints instead of an interval. If true, the ref interval is not required.", - 582 ) - 583 ref_interval_start_timestamp: UtcTimestamp | None = optional( - 584 description="Start of the queried time interval (ref interval) in UTC isoformat." - 585 " Required unless ref_dataset_discrete is true.", - 586 ) - 587 ref_interval_end_timestamp: UtcTimestamp | None = optional( - 588 description="End of the queried time interval (ref interval) in UTC isoformat." - 589 " Required unless ref_dataset_discrete is true.", - 590 ) - 591 ref_interval_type: IntervalType | None = optional( - 592 description="Type of the ref interval. Required unless ref_dataset_discrete is true.", + 567class _DatasetMetadataBase(ConventionModel): + 568 model_config = ConfigDict(json_schema_extra=_REF_INTERVAL_REQUIRED_SCHEMA) + 569 + 570 metric_key: str = Field( + 571 ..., + 572 description="Which field of the metric objects under metrics identifies a metric, i.e." + 573 ' occurs in the metric column of a MULTITSFRAME, e.g. "id" or "external_id".', + 574 ) + 575 queried_metrics: tuple[str, ...] = Field( + 576 ..., + 577 description="The requested metrics. Since not all queried metrics may have data in the" + 578 " queried interval, a metric may be missing in the data.", + 579 ) + 580 signal_group: SignalGroupMetadata | None = optional() + 581 ref_dataset_discrete: StrictBool | None = optional( + 582 False, + 583 description="Whether the ref dataset should be considered as discrete, i.e. isolated" + 584 " datapoints instead of an interval. If true, the ref interval is not required.", + 585 ) + 586 ref_interval_start_timestamp: UtcTimestamp | None = optional( + 587 description="Start of the queried time interval (ref interval) as ISO 8601 timestamp" + 588 " in UTC. Required unless ref_dataset_discrete is true.", + 589 ) + 590 ref_interval_end_timestamp: UtcTimestamp | None = optional( + 591 description="End of the queried time interval (ref interval) as ISO 8601 timestamp in" + 592 " UTC. Required unless ref_dataset_discrete is true.", 593 ) - 594 ref_data_frequency: str | None = optional( - 595 description="Frequency of the timeseries " - 596 + _FREQUENCY_FORMAT - 597 + ": the data_frequency of all metrics without their own, with the same meaning as" - 598 " there. Together with ref_data_frequency_offset and ref_data_frequency_timezone.", - 599 ) - 600 ref_data_frequency_offset: str | None = optional( - 601 description=_OFFSET_DESCRIPTION.format( - 602 frequency="ref_data_frequency", timezone="ref_data_frequency_timezone" - 603 ) - 604 + " Applies to all metrics without their own data_frequency.", - 605 ) - 606 ref_data_frequency_timezone: str | None = optional( - 607 description=_ALIGNMENT_DESCRIPTION.format(frequency="ref_data_frequency") - 608 + " Applies to all metrics without their own data_frequency.", - 609 ) - 610 query_timestamp: UtcTimestamp | None = optional( - 611 description="The point in time whose state of the data was delivered in UTC isoformat:" - 612 " Data stored, changed or invalidated later is not included. A source supporting it" - 613 " delivers the same data again when queried for this point in time, e.g. for" - 614 " reproducing an execution.", - 615 ) - 616 default_aggregation_interval: str | None = optional( - 617 description="Interval to aggregate the timeseries to by default, e.g. when components" - 618 " plot or report them, " - 619 + _FREQUENCY_FORMAT - 620 + ". The metrics are aggregated with their default_aggregation_method. E.g. a hetida" - 621 " platform signal group is configured with such an interval.", - 622 ) - 623 default_timezone: str | None = optional( - 624 description="Time zone the timeseries should be evaluated in by default, " - 625 + _TIMEZONE_FORMAT - 626 + "." - 627 + _TIMEZONE_PURPOSE - 628 + " Applies to all metrics without their own default_timezone.", - 629 ) - 630 invalidation_interval_start: UtcTimestamp | None = optional( - 631 description="Start of an invalidation interval separate from the ref interval in UTC" - 632 " isoformat", - 633 ) - 634 invalidation_interval_end: UtcTimestamp | None = optional( - 635 description="End of an invalidation interval separate from the ref interval in UTC" - 636 " isoformat", - 637 ) - 638 invalidation_interval_type: IntervalType | None = optional( - 639 IntervalType.CLOSED, description="Type of the invalidation interval" + 594 ref_interval_type: IntervalType | None = optional( + 595 description="Type of the ref interval. Required unless ref_dataset_discrete is true.", + 596 ) + 597 ref_data_frequency: str | None = optional( + 598 description="Frequency of the timeseries " + 599 + _FREQUENCY_FORMAT + 600 + ": the data_frequency of all metrics without their own, with the same meaning as" + 601 " there. Together with ref_data_frequency_offset and ref_data_frequency_timezone.", + 602 ) + 603 ref_data_frequency_offset: str | None = optional( + 604 description=_OFFSET_DESCRIPTION.format( + 605 frequency="ref_data_frequency", timezone="ref_data_frequency_timezone" + 606 ) + 607 + " Applies to all metrics without their own data_frequency.", + 608 ) + 609 ref_data_frequency_timezone: str | None = optional( + 610 description=_ALIGNMENT_DESCRIPTION.format(frequency="ref_data_frequency") + 611 + " Applies to all metrics without their own data_frequency.", + 612 ) + 613 query_timestamp: UtcTimestamp | None = optional( + 614 description="The point in time whose state of the data was delivered, as ISO 8601" + 615 " timestamp in UTC: Data stored, changed or invalidated later is not included. A source" + 616 " supporting it delivers the same data again when queried for this point in time, e.g." + 617 " for reproducing an execution.", + 618 ) + 619 default_aggregation_interval: str | None = optional( + 620 description="Interval to aggregate the timeseries to by default, e.g. when components" + 621 " plot or report them, " + 622 + _FREQUENCY_FORMAT + 623 + ". The metrics are aggregated with their default_aggregation_method. E.g. a hetida" + 624 " platform signal group is configured with such an interval.", + 625 ) + 626 default_timezone: str | None = optional( + 627 description="Time zone the timeseries should be evaluated in by default, " + 628 + _TIMEZONE_FORMAT + 629 + "." + 630 + _TIMEZONE_PURPOSE + 631 + " Applies to all metrics without their own default_timezone.", + 632 ) + 633 invalidation_interval_start: UtcTimestamp | None = optional( + 634 description="Start of an invalidation interval separate from the ref interval, as ISO" + 635 " 8601 timestamp in UTC", + 636 ) + 637 invalidation_interval_end: UtcTimestamp | None = optional( + 638 description="End of an invalidation interval separate from the ref interval, as ISO" + 639 " 8601 timestamp in UTC", 640 ) - 641 invalidate_dataset: StrictBool | None = optional( - 642 description="Whether existing data should be invalidated: in the invalidation interval if" - 643 " specified, otherwise in the ref interval. If neither is specified, a closed ref interval" - 644 " from the first to the last datapoint is inferred. Without datapoints no invalidation can" - 645 " occur then. What null means is up to the adapter.", - 646 ) - 647 delete_invalidated: StrictBool | None = optional( - 648 description="Whether data to be invalidated should be deleted instead. What null means is" - 649 " up to the adapter. Adapters do not need to support deletion.", - 650 ) - 651 invalidation_timestamp: UtcTimestamp | None = optional( - 652 description="Invalidation timestamp for all invalidated datapoints. If null, the" - 653 " execution_timestamp query parameter sent to generic REST adapters or, if that is not" - 654 " present either, the current time should be used.", - 655 ) - 656 only_invalidate: StrictBool | None = optional( - 657 False, - 658 description="If true, existing data should only be invalidated and the new data should" - 659 " not be stored. Cannot be true if invalidate_dataset is false.", - 660 ) - 661 new_data_invalidation_date: UtcTimestamp | None = optional( - 662 description="An invalidation date to be stored together with all new datapoints, i.e." - 663 " invalidating them in advance. Ignored if null.", - 664 ) - 665 - 666 + 641 invalidation_interval_type: IntervalType | None = optional( + 642 IntervalType.CLOSED, description="Type of the invalidation interval" + 643 ) + 644 invalidate_dataset: StrictBool | None = optional( + 645 description="Whether existing data should be invalidated: in the invalidation interval if" + 646 " specified, otherwise in the ref interval. If neither is specified, a closed ref interval" + 647 " from the first to the last datapoint is inferred. Without datapoints no invalidation can" + 648 " occur then. What null means is up to the adapter.", + 649 ) + 650 delete_invalidated: StrictBool | None = optional( + 651 description="Whether data to be invalidated should be deleted instead. What null means is" + 652 " up to the adapter. Adapters do not need to support deletion.", + 653 ) + 654 invalidation_timestamp: UtcTimestamp | None = optional( + 655 description="Invalidation timestamp for all invalidated datapoints. If null, the" + 656 " execution_timestamp query parameter sent to generic REST adapters or, if that is not" + 657 " present either, the current time should be used.", + 658 ) + 659 only_invalidate: StrictBool | None = optional( + 660 False, + 661 description="If true, existing data should only be invalidated and the new data should" + 662 " not be stored. Cannot be true if invalidate_dataset is false.", + 663 ) + 664 new_data_invalidation_date: UtcTimestamp | None = optional( + 665 description="An invalidation date to be stored together with all new datapoints, i.e." + 666 " invalidating them in advance. Ignored if null.", + 667 ) + 668 + 669
[docs] - 667class SingleMetricDatasetMetadata(_DatasetMetadataBase): - 668 """Metadata associated with a SERIES or SINGLETSFRAME as a whole - 669 - 670 E.g. the queried time interval or how existing data should be handled when the dataset is - 671 sent to an adapter. The fields concerning invalidation and deletion are only relevant for - 672 adapters actually supporting these operations, like the built-in SQL adapter. Other adapters - 673 may ignore them. - 674 - 675 Start and end of an interval must be provided together. - 676 """ + 670class SingleMetricDatasetMetadata(_DatasetMetadataBase): + 671 """Metadata associated with a SERIES or SINGLETSFRAME as a whole + 672 + 673 E.g. the queried time interval or how existing data should be handled when the dataset is + 674 sent to an adapter. The fields concerning invalidation and deletion are only relevant for + 675 adapters actually supporting these operations, like the built-in SQL adapter. Other adapters + 676 may ignore them. 677 - 678 single_metric: str = Field( - 679 ..., description="The metric_key field value of the single metric under metrics." - 680 )
+ 678 Start and end of an interval must be provided together. + 679 """ + 680 + 681 single_metric: str = Field( + 682 ..., description="The metric_key field value of the single metric under metrics." + 683 ) - 681 - 682 + 684 + 685
[docs] - 683class MultiTSFrameDatasetMetadata(_DatasetMetadataBase): - 684 """Metadata associated with a MULTITSFRAME as a whole - 685 - 686 E.g. the queried time interval or how existing data should be handled when the dataset is - 687 sent to an adapter. The fields concerning invalidation and deletion are only relevant for - 688 adapters actually supporting these operations, like the built-in SQL adapter. Other adapters - 689 may ignore them. - 690 - 691 Start and end of an interval must be provided together. - 692 """ + 686class MultiTSFrameDatasetMetadata(_DatasetMetadataBase): + 687 """Metadata associated with a MULTITSFRAME as a whole + 688 + 689 E.g. the queried time interval or how existing data should be handled when the dataset is + 690 sent to an adapter. The fields concerning invalidation and deletion are only relevant for + 691 adapters actually supporting these operations, like the built-in SQL adapter. Other adapters + 692 may ignore them. 693 - 694 single_metric: str | None = optional( - 695 description="If the MULTITSFRAME holds a single metric: its metric_key field value. Must" - 696 " not be set if more than one metric is present.", - 697 )
+ 694 Start and end of an interval must be provided together. + 695 """ + 696 + 697 single_metric: str | None = optional( + 698 description="If the MULTITSFRAME holds a single metric: its metric_key field value. Must" + 699 " not be set if more than one metric is present.", + 700 ) - 698 - 699 - 700class _TimeseriesMetadataBase(MetadataReading, ConventionModel): - 701 """Base of the root models, the metadata of a SERIES, SINGLETSFRAME or MULTITSFRAME + 701 702 - 703 The methods of MetadataReading look up information with the fallbacks of the conventions. - 704 The with_* methods return changed versions, which share all unchanged parts with this one. - 705 """ - 706 - 707 _data_type: ClassVar[TimeseriesDataType] - 708 - 709 value_dimensions_shared: tuple[SharedValueDimensionMetadata, ...] | None = optional( - 710 description="Metadata of value dimensions that have the same meaning for all metrics." - 711 " Consumers use it as fallback, but it does not replace the value dimensions of the" - 712 " metrics. Each column may occur at most once.", - 713 ) - 714 - 715 @model_validator(mode="wrap") - 716 @classmethod - 717 def _conforms( - 718 cls, data: Any, handler: ModelWrapValidatorHandler[Self], info: ValidationInfo - 719 ) -> Self: - 720 """Reject metadata violating the rules of the conventions which JSON Schema cannot express - 721 - 722 check_metadata_conventions reports these errors itself, at their exact locations. - 723 """ - 724 model = handler(data) - 725 if (info.context or {}).get(_SKIP_RULES) is True: - 726 return model - 727 errors = _rule_errors(data, model, cls._data_type) - 728 if len(errors) > 0: - 729 raise MetadataConventionError(errors) - 730 return model - 731 - 732 def model_copy(self, *, update: Mapping[str, Any] | None = None, deep: bool = False) -> Self: - 733 copied = super().model_copy(update=update, deep=deep) - 734 copied.__dict__.pop("_metrics_by_id", None) # the metrics may differ - 735 return copied - 736 - 737 def _converted[T: "_TimeseriesMetadataBase"](self, model: type[T], **fields: Any) -> T: - 738 """The metadata as model with the given fields, keeping the extra fields""" - 739 return model.model_validate(self.__pydantic_extra__ | fields) - 740 - 741 def with_metric(self, metric_id: str | None = None, /, **changes: Any) -> Self: - 742 """A version with fields of a metric changed, the other metrics are shared + 703class _TimeseriesMetadataBase(MetadataReading, ConventionModel): + 704 """Base of the root models, the metadata of a SERIES, SINGLETSFRAME or MULTITSFRAME + 705 + 706 The methods of MetadataReading look up information with the fallbacks of the conventions. + 707 The with_* methods return changed versions, which share all unchanged parts with this one. + 708 """ + 709 + 710 _data_type: ClassVar[TimeseriesDataType] + 711 + 712 value_dimensions_shared: tuple[SharedValueDimensionMetadata, ...] | None = optional( + 713 description="Metadata of value dimensions that have the same meaning for all metrics." + 714 " Consumers use it as fallback, but it does not replace the value dimensions of the" + 715 " metrics. Each column may occur at most once.", + 716 ) + 717 + 718 @model_validator(mode="wrap") + 719 @classmethod + 720 def _conforms( + 721 cls, data: Any, handler: ModelWrapValidatorHandler[Self], info: ValidationInfo + 722 ) -> Self: + 723 """Reject metadata violating the rules of the conventions which JSON Schema cannot express + 724 + 725 check_metadata_conventions reports these errors itself, at their exact locations. + 726 """ + 727 model = handler(data) + 728 if (info.context or {}).get(_SKIP_RULES) is True: + 729 return model + 730 errors = _rule_errors(data, model, cls._data_type) + 731 if len(errors) > 0: + 732 raise MetadataConventionError(errors) + 733 return model + 734 + 735 def model_copy(self, *, update: Mapping[str, Any] | None = None, deep: bool = False) -> Self: + 736 copied = super().model_copy(update=update, deep=deep) + 737 copied.__dict__.pop("_metrics_by_id", None) # the metrics may differ + 738 return copied + 739 + 740 def _converted[T: "_TimeseriesMetadataBase"](self, model: type[T], **fields: Any) -> T: + 741 """The metadata as model with the given fields, keeping the extra fields""" + 742 return model.model_validate(self.__pydantic_extra__ | fields) 743 - 744 The changed metric is validated, including the rules of the conventions which concern it. - 745 """ - 746 old = self.metric(metric_id) - 747 new = type(old).model_validate(old.model_dump() | changes) - 748 errors = _metric_rule_errors(self, new) - 749 new_id = id_of(new, self.dataset_metadata.metric_key) - 750 if new_id != id_of(old, self.dataset_metadata.metric_key) and new_id in self.metric_ids(): - 751 errors.append(_error("metrics", f'Metric "{new_id}" occurs more than once.')) - 752 if len(errors) > 0: - 753 raise MetadataConventionError(errors) - 754 return self.model_copy( - 755 update={"metrics": tuple(new if metric is old else metric for metric in self.metrics)} - 756 ) - 757 - 758 def with_value_dimension( - 759 self, metric_id: str | None = None, column: str = "value", /, **changes: Any - 760 ) -> Self: - 761 """A version with fields of a value dimension of a metric changed, see with_metric""" - 762 metric = self.metric(metric_id) - 763 if column not in {value_dimension.column for value_dimension in metric.value_dimensions}: - 764 raise KeyError(column) - 765 return self.with_metric( - 766 metric_id, - 767 value_dimensions=tuple( - 768 value_dimension.model_dump() | changes - 769 if value_dimension.column == column - 770 else value_dimension - 771 for value_dimension in metric.value_dimensions - 772 ), - 773 ) - 774 - 775 def with_dataset_metadata(self, **changes: Any) -> Self: - 776 """A version with fields of the dataset metadata changed, the metrics are shared + 744 def with_metric(self, metric_id: str | None = None, /, **changes: Any) -> Self: + 745 """A version with fields of a metric changed, the other metrics are shared + 746 + 747 The changed metric is validated, including the rules of the conventions which concern it. + 748 """ + 749 old = self.metric(metric_id) + 750 new = type(old).model_validate(old.model_dump() | changes) + 751 errors = _metric_rule_errors(self, new) + 752 new_id = id_of(new, self.dataset_metadata.metric_key) + 753 if new_id != id_of(old, self.dataset_metadata.metric_key) and new_id in self.metric_ids(): + 754 errors.append(_error("metrics", f'Metric "{new_id}" occurs more than once.')) + 755 if len(errors) > 0: + 756 raise MetadataConventionError(errors) + 757 return self.model_copy( + 758 update={"metrics": tuple(new if metric is old else metric for metric in self.metrics)} + 759 ) + 760 + 761 def with_value_dimension( + 762 self, metric_id: str | None = None, column: str = "value", /, **changes: Any + 763 ) -> Self: + 764 """A version with fields of a value dimension of a metric changed, see with_metric""" + 765 metric = self.metric(metric_id) + 766 if column not in {value_dimension.column for value_dimension in metric.value_dimensions}: + 767 raise KeyError(column) + 768 return self.with_metric( + 769 metric_id, + 770 value_dimensions=tuple( + 771 value_dimension.model_dump() | changes + 772 if value_dimension.column == column + 773 else value_dimension + 774 for value_dimension in metric.value_dimensions + 775 ), + 776 ) 777 - 778 Validated including all rules of the conventions, since the dataset metadata concerns - 779 all metrics. - 780 """ - 781 old = self.dataset_metadata - 782 changed = self.model_copy( - 783 update={"dataset_metadata": type(old).model_validate(old.model_dump() | changes)} - 784 ) - 785 errors = _rule_errors(None, changed, self._data_type) - 786 if len(errors) > 0: - 787 raise MetadataConventionError(errors) - 788 return changed - 789 - 790 - 791def _value_dimension(column: str, unit: str | None, measurement: str | None) -> dict[str, Any]: - 792 return {"column": column, "unit": unit, "measurement": measurement} + 778 def with_dataset_metadata(self, **changes: Any) -> Self: + 779 """A version with fields of the dataset metadata changed, the metrics are shared + 780 + 781 Validated including all rules of the conventions, since the dataset metadata concerns + 782 all metrics. + 783 """ + 784 old = self.dataset_metadata + 785 changed = self.model_copy( + 786 update={"dataset_metadata": type(old).model_validate(old.model_dump() | changes)} + 787 ) + 788 errors = _rule_errors(None, changed, self._data_type) + 789 if len(errors) > 0: + 790 raise MetadataConventionError(errors) + 791 return changed + 792 793 - 794 - 795SERIES_METADATA_EXAMPLE: dict[str, Any] = { - 796 "dataset_metadata": { - 797 "metric_key": "id", - 798 "single_metric": "7ab8c419-8745-4714-b588-57c0c772cc95", - 799 "queried_metrics": ["7ab8c419-8745-4714-b588-57c0c772cc95"], - 800 "ref_interval_start_timestamp": "2026-01-01T00:00:00+00:00", - 801 "ref_interval_end_timestamp": "2026-01-02T00:00:00+00:00", - 802 "ref_interval_type": "right_open", - 803 "query_timestamp": "2026-01-02T00:05:00+00:00", - 804 }, - 805 "metrics": [ - 806 { - 807 "id": "7ab8c419-8745-4714-b588-57c0c772cc95", - 808 "name": "Water level north", - 809 "value_dimensions": [_value_dimension("value", "m", "level") | {"decimals": 2}], - 810 "default_aggregation_method": "MEAN", - 811 "default_timezone": "Europe/Berlin", - 812 "path": ["Waterworks North", "Gauge North"], - 813 "origin": "OPC_UA", - 814 "recording_mode": "periodic", - 815 "data_frequency": "15min", - 816 "recording_aggregation_method": "MEAN", - 817 } - 818 ], - 819} - 820 - 821SINGLETSFRAME_METADATA_EXAMPLE: dict[str, Any] = { - 822 "dataset_metadata": { - 823 "metric_key": "external_id", - 824 "single_metric": "abc.temp", - 825 "queried_metrics": ["abc.temp"], - 826 "ref_interval_start_timestamp": "2026-01-01T00:00:00+00:00", - 827 "ref_interval_end_timestamp": "2026-01-02T00:00:00+00:00", - 828 "ref_interval_type": "closed", - 829 }, - 830 "metrics": [ - 831 { - 832 "id": "3f2b1c4d-5e6f-4a7b-8c9d-0e1f2a3b4c5d", - 833 "external_id": "abc.temp", - 834 "name": "ABC temperature", - 835 "value_dimensions": [ - 836 _value_dimension("value", "°C", "temperature"), - 837 _value_dimension("state", None, None) | {"name": "measurement state"}, - 838 ], - 839 } - 840 ], - 841} - 842 - 843MULTITSFRAME_METADATA_EXAMPLE: dict[str, Any] = { - 844 "dataset_metadata": { - 845 "metric_key": "external_id", - 846 "queried_metrics": ["plant_a.pressure", "plant_a.temperature"], - 847 "ref_interval_start_timestamp": "2026-01-01T00:00:00+00:00", - 848 "ref_interval_end_timestamp": "2026-01-02T00:00:00+00:00", - 849 "ref_interval_type": "closed", - 850 "default_aggregation_interval": "1h", - 851 "default_timezone": "Europe/Berlin", - 852 }, - 853 "metrics": [ - 854 { - 855 "id": "c5a8e2f1-0b3d-4e6a-9f7c-1d2e3f4a5b6c", - 856 "external_id": "plant_a.pressure", - 857 "name": "Plant A pressure", - 858 "value_dimensions": [_value_dimension("value", "bar", "pressure")], - 859 }, - 860 { - 861 "id": "8d7c6b5a-4f3e-4d2c-8b1a-0f9e8d7c6b5a", - 862 "external_id": "plant_a.temperature", - 863 "name": "Plant A temperature", - 864 "value_dimensions": [ - 865 _value_dimension("value", "°C", "temperature") | {"deadband": 0.5, "decimals": 1} - 866 ], - 867 "default_aggregation_method": "MAX", - 868 "recording_mode": "on_change", - 869 "data_frequency": "1h", - 870 }, - 871 ], - 872} - 873 - 874_SINGLE_METRIC_DESCRIPTION = ( - 875 "Metadata of the single metric: exactly one metric, which has the field named by" - 876 " dataset_metadata.metric_key with the value of dataset_metadata.single_metric." - 877) - 878 - 879 - 880# the metric id field set when a metric gets another id, if the metric key field is taken - 881_METRIC_NAME_FIELD = "metric" + 794def _value_dimension(column: str, unit: str | None, measurement: str | None) -> dict[str, Any]: + 795 return {"column": column, "unit": unit, "measurement": measurement} + 796 + 797 + 798SERIES_METADATA_EXAMPLE: dict[str, Any] = { + 799 "dataset_metadata": { + 800 "metric_key": "id", + 801 "single_metric": "7ab8c419-8745-4714-b588-57c0c772cc95", + 802 "queried_metrics": ["7ab8c419-8745-4714-b588-57c0c772cc95"], + 803 "ref_interval_start_timestamp": "2026-01-01T00:00:00+00:00", + 804 "ref_interval_end_timestamp": "2026-01-02T00:00:00+00:00", + 805 "ref_interval_type": "right_open", + 806 "query_timestamp": "2026-01-02T00:05:00+00:00", + 807 }, + 808 "metrics": [ + 809 { + 810 "id": "7ab8c419-8745-4714-b588-57c0c772cc95", + 811 "name": "Water level north", + 812 "value_dimensions": [_value_dimension("value", "m", "level") | {"decimals": 2}], + 813 "default_aggregation_method": "MEAN", + 814 "default_timezone": "Europe/Berlin", + 815 "path": ["Waterworks North", "Gauge North"], + 816 "origin": "OPC_UA", + 817 "recording_mode": "periodic", + 818 "data_frequency": "15min", + 819 "recording_aggregation_method": "MEAN", + 820 } + 821 ], + 822} + 823 + 824SINGLETSFRAME_METADATA_EXAMPLE: dict[str, Any] = { + 825 "dataset_metadata": { + 826 "metric_key": "external_id", + 827 "single_metric": "abc.temp", + 828 "queried_metrics": ["abc.temp"], + 829 "ref_interval_start_timestamp": "2026-01-01T00:00:00+00:00", + 830 "ref_interval_end_timestamp": "2026-01-02T00:00:00+00:00", + 831 "ref_interval_type": "closed", + 832 }, + 833 "metrics": [ + 834 { + 835 "id": "3f2b1c4d-5e6f-4a7b-8c9d-0e1f2a3b4c5d", + 836 "external_id": "abc.temp", + 837 "name": "ABC temperature", + 838 "value_dimensions": [ + 839 _value_dimension("value", "°C", "temperature"), + 840 _value_dimension("state", None, None) | {"name": "measurement state"}, + 841 ], + 842 } + 843 ], + 844} + 845 + 846MULTITSFRAME_METADATA_EXAMPLE: dict[str, Any] = { + 847 "dataset_metadata": { + 848 "metric_key": "external_id", + 849 "queried_metrics": ["plant_a.pressure", "plant_a.temperature"], + 850 "ref_interval_start_timestamp": "2026-01-01T00:00:00+00:00", + 851 "ref_interval_end_timestamp": "2026-01-02T00:00:00+00:00", + 852 "ref_interval_type": "closed", + 853 "default_aggregation_interval": "1h", + 854 "default_timezone": "Europe/Berlin", + 855 }, + 856 "metrics": [ + 857 { + 858 "id": "c5a8e2f1-0b3d-4e6a-9f7c-1d2e3f4a5b6c", + 859 "external_id": "plant_a.pressure", + 860 "name": "Plant A pressure", + 861 "value_dimensions": [_value_dimension("value", "bar", "pressure")], + 862 }, + 863 { + 864 "id": "8d7c6b5a-4f3e-4d2c-8b1a-0f9e8d7c6b5a", + 865 "external_id": "plant_a.temperature", + 866 "name": "Plant A temperature", + 867 "value_dimensions": [ + 868 _value_dimension("value", "°C", "temperature") | {"deadband": 0.5, "decimals": 1} + 869 ], + 870 "default_aggregation_method": "MAX", + 871 "recording_mode": "on_change", + 872 "data_frequency": "1h", + 873 }, + 874 ], + 875} + 876 + 877_SINGLE_METRIC_DESCRIPTION = ( + 878 "Metadata of the single metric: exactly one metric, which has the field named by" + 879 " dataset_metadata.metric_key with the value of dataset_metadata.single_metric." + 880) + 881 882 - 883 - 884def _renamed[T: SharedValueDimensionMetadata]( - 885 value_dimensions: tuple[T, ...] | None, column: str, new_column: str - 886) -> tuple[T, ...] | None: - 887 """Only the value dimensions of column, renamed to new_column, None if there are none""" - 888 renamed = tuple( - 889 value_dimension.model_copy(update={"column": new_column}) - 890 for value_dimension in value_dimensions or () - 891 if value_dimension.column == column - 892 ) - 893 return renamed if len(renamed) > 0 else None - 894 - 895 - 896def _with_value_column(metric: MetricMetadata, column: str, new_column: str) -> MetricMetadata: - 897 """The metric with only the value dimension of column, renamed to new_column""" - 898 return metric.model_copy( - 899 update={"value_dimensions": _renamed(metric.value_dimensions, column, new_column) or ()} - 900 ) - 901 - 902 + 883# the metric id field set when a metric gets another id, if the metric key field is taken + 884_METRIC_NAME_FIELD = "metric" + 885 + 886 + 887def _renamed[T: SharedValueDimensionMetadata]( + 888 value_dimensions: tuple[T, ...] | None, column: str, new_column: str + 889) -> tuple[T, ...] | None: + 890 """Only the value dimensions of column, renamed to new_column, None if there are none""" + 891 renamed = tuple( + 892 value_dimension.model_copy(update={"column": new_column}) + 893 for value_dimension in value_dimensions or () + 894 if value_dimension.column == column + 895 ) + 896 return renamed if len(renamed) > 0 else None + 897 + 898 + 899def _with_value_column(metric: MetricMetadata, column: str, new_column: str) -> MetricMetadata: + 900 """The metric with only the value dimension of column, renamed to new_column""" + 901 return metric.model_copy( + 902 update={"value_dimensions": _renamed(metric.value_dimensions, column, new_column) or ()} + 903 ) + 904 + 905
[docs] - 903class SeriesMetadata(_TimeseriesMetadataBase): - 904 """Immutable metadata for a SERIES: one metric with a single "value" dimension. - 905 - 906 Read from a pandas Series with read_metadata. See the module documentation for storage, - 907 adapter exchange and common field rules. - 908 """ - 909 - 910 model_config = ConfigDict(json_schema_extra={"examples": [SERIES_METADATA_EXAMPLE]}) - 911 - 912 _data_type: ClassVar[TimeseriesDataType] = "SERIES" - 913 - 914 dataset_metadata: SingleMetricDatasetMetadata - 915 metrics: Annotated[tuple[MetricMetadata, ...], _item_count(1, 1)] = Field( - 916 ..., - 917 json_schema_extra={"minItems": 1, "maxItems": 1}, - 918 description=_SINGLE_METRIC_DESCRIPTION, - 919 ) - 920 + 906class SeriesMetadata(_TimeseriesMetadataBase): + 907 """Metadata of a SERIES: one metric with a single "value" dimension. + 908 + 909 Exchanged with adapters, e.g. in the Data-Attributes header of generic REST adapter endpoints + 910 or the __metadata__ field of the direct provisioning adapter's wrapped format. + 911 """ + 912 + 913 model_config = ConfigDict(json_schema_extra={"examples": [SERIES_METADATA_EXAMPLE]}) + 914 + 915 _data_type: ClassVar[TimeseriesDataType] = "SERIES" + 916 + 917 dataset_metadata: SingleMetricDatasetMetadata + 918 metrics: Annotated[tuple[MetricMetadata, ...], _item_count(1, 1)] = Field( + 919 ..., + 920 json_schema_extra={"minItems": 1, "maxItems": 1}, + 921 description=_SINGLE_METRIC_DESCRIPTION, + 922 ) + 923
[docs] - 921 def as_singletsframe(self, column: str = "value") -> "SingleTSFrameMetadata": - 922 """The metadata of the SINGLETSFRAME with the values of the SERIES in column - 923 - 924 Renames the value dimension "value" of the metric and the shared one to column. - 925 """ - 926 return self._converted( - 927 SingleTSFrameMetadata, - 928 dataset_metadata=self.dataset_metadata, - 929 metrics=(_with_value_column(self.metric(), "value", column),), - 930 value_dimensions_shared=_renamed(self.value_dimensions_shared, "value", column), - 931 )
+ 924 def as_singletsframe(self, column: str = "value") -> "SingleTSFrameMetadata": + 925 """The metadata of the SINGLETSFRAME with the values of the SERIES in column + 926 + 927 Renames the value dimension "value" of the metric and the shared one to column. + 928 """ + 929 return self._converted( + 930 SingleTSFrameMetadata, + 931 dataset_metadata=self.dataset_metadata, + 932 metrics=(_with_value_column(self.metric(), "value", column),), + 933 value_dimensions_shared=_renamed(self.value_dimensions_shared, "value", column), + 934 )
- 932 - 933 + 935 + 936
[docs] - 934class SingleTSFrameMetadata(_TimeseriesMetadataBase): - 935 """Immutable metadata for a SINGLETSFRAME: one metric with one or more value columns. - 936 - 937 Read from a pandas DataFrame with read_metadata. See the module documentation for storage, - 938 adapter exchange and common field rules. - 939 """ - 940 - 941 model_config = ConfigDict(json_schema_extra={"examples": [SINGLETSFRAME_METADATA_EXAMPLE]}) - 942 - 943 _data_type: ClassVar[TimeseriesDataType] = "SINGLETSFRAME" - 944 - 945 dataset_metadata: SingleMetricDatasetMetadata - 946 metrics: Annotated[tuple[MetricMetadata, ...], _item_count(1, 1)] = Field( - 947 ..., - 948 json_schema_extra={"minItems": 1, "maxItems": 1}, - 949 description=_SINGLE_METRIC_DESCRIPTION, - 950 ) - 951 + 937class SingleTSFrameMetadata(_TimeseriesMetadataBase): + 938 """Metadata of a SINGLETSFRAME: one metric with one or more value columns. + 939 + 940 Exchanged with adapters, e.g. in the Data-Attributes header of generic REST adapter endpoints + 941 or the __metadata__ field of the direct provisioning adapter's wrapped format. + 942 """ + 943 + 944 model_config = ConfigDict(json_schema_extra={"examples": [SINGLETSFRAME_METADATA_EXAMPLE]}) + 945 + 946 _data_type: ClassVar[TimeseriesDataType] = "SINGLETSFRAME" + 947 + 948 dataset_metadata: SingleMetricDatasetMetadata + 949 metrics: Annotated[tuple[MetricMetadata, ...], _item_count(1, 1)] = Field( + 950 ..., + 951 json_schema_extra={"minItems": 1, "maxItems": 1}, + 952 description=_SINGLE_METRIC_DESCRIPTION, + 953 ) + 954
[docs] - 952 def as_series(self, column: str = "value") -> SeriesMetadata: - 953 """The metadata of the SERIES with the values of column - 954 - 955 Keeps only the value dimensions of column, renamed to "value". KeyError if the metric - 956 has no value dimension of column. - 957 """ - 958 if column not in {vd.column for vd in self.metric().value_dimensions}: - 959 raise KeyError(column) - 960 return self._converted( - 961 SeriesMetadata, - 962 dataset_metadata=self.dataset_metadata, - 963 metrics=(_with_value_column(self.metric(), column, "value"),), - 964 value_dimensions_shared=_renamed(self.value_dimensions_shared, column, "value"), - 965 )
+ 955 def as_series(self, column: str = "value") -> SeriesMetadata: + 956 """The metadata of the SERIES with the values of column + 957 + 958 Keeps only the value dimensions of column, renamed to "value". KeyError if the metric + 959 has no value dimension of column. + 960 """ + 961 if column not in {vd.column for vd in self.metric().value_dimensions}: + 962 raise KeyError(column) + 963 return self._converted( + 964 SeriesMetadata, + 965 dataset_metadata=self.dataset_metadata, + 966 metrics=(_with_value_column(self.metric(), column, "value"),), + 967 value_dimensions_shared=_renamed(self.value_dimensions_shared, column, "value"), + 968 )
- 966 + 969
[docs] - 967 def as_multitsframe(self, metric_id: str | None = None) -> "MultiTSFrameMetadata": - 968 """The metadata of the MULTITSFRAME of the single metric, named metric_id if given - 969 - 970 If metric_id differs from the id of the metric, it becomes its id: in the field named by - 971 the metric key if the metric does not have it, otherwise in the field "metric", which - 972 then becomes the metric key. - 973 """ - 974 metric = self.metric() - 975 dataset_metadata = self.dataset_metadata.model_dump() - 976 if metric_id is not None and metric_id != self.metric_ids()[0]: - 977 metric_key = dataset_metadata["metric_key"] - 978 if id_of(metric, metric_key) is not None: - 979 metric_key = dataset_metadata["metric_key"] = _METRIC_NAME_FIELD - 980 metric = type(metric).model_validate(metric.model_dump() | {metric_key: metric_id}) - 981 metric_id = id_of(metric, dataset_metadata["metric_key"]) - 982 return self._converted( - 983 MultiTSFrameMetadata, - 984 dataset_metadata=dataset_metadata - 985 | {"single_metric": metric_id, "queried_metrics": [metric_id]}, - 986 metrics=(metric,), - 987 value_dimensions_shared=self.value_dimensions_shared, - 988 )
+ 970 def as_multitsframe(self, metric_id: str | None = None) -> "MultiTSFrameMetadata": + 971 """The metadata of the MULTITSFRAME of the single metric, named metric_id if given + 972 + 973 If metric_id differs from the id of the metric, it becomes its id: in the field named by + 974 the metric key if the metric does not have it, otherwise in the field "metric", which + 975 then becomes the metric key. + 976 """ + 977 metric = self.metric() + 978 dataset_metadata = self.dataset_metadata.model_dump() + 979 if metric_id is not None and metric_id != self.metric_ids()[0]: + 980 metric_key = dataset_metadata["metric_key"] + 981 if id_of(metric, metric_key) is not None: + 982 metric_key = dataset_metadata["metric_key"] = _METRIC_NAME_FIELD + 983 metric = type(metric).model_validate(metric.model_dump() | {metric_key: metric_id}) + 984 metric_id = id_of(metric, dataset_metadata["metric_key"]) + 985 return self._converted( + 986 MultiTSFrameMetadata, + 987 dataset_metadata=dataset_metadata + 988 | {"single_metric": metric_id, "queried_metrics": [metric_id]}, + 989 metrics=(metric,), + 990 value_dimensions_shared=self.value_dimensions_shared, + 991 ) - 989 - 990 + 992 + 993
[docs] - 991class MultiTSFrameMetadata(_TimeseriesMetadataBase): - 992 """Immutable metadata for a MULTITSFRAME: multiple metrics identified by a metric column. - 993 - 994 Each metric can have several value dimensions; a MULTITSFRAME may also hold one metric. - 995 Read from a pandas DataFrame with read_metadata. See the module documentation for storage, - 996 adapter exchange and common field rules. - 997 """ - 998 - 999 model_config = ConfigDict(json_schema_extra={"examples": [MULTITSFRAME_METADATA_EXAMPLE]}) -1000 -1001 _data_type: ClassVar[TimeseriesDataType] = "MULTITSFRAME" -1002 -1003 dataset_metadata: MultiTSFrameDatasetMetadata -1004 metrics: tuple[MetricMetadata, ...] = Field( -1005 ..., -1006 description="Metadata of the metrics: one for each metric in the data. Each metric must" -1007 " have the field named by dataset_metadata.metric_key as a string, unique among the" -1008 " metrics.", -1009 ) -1010 + 994class MultiTSFrameMetadata(_TimeseriesMetadataBase): + 995 """Metadata of a MULTITSFRAME: multiple metrics identified by a metric column. + 996 + 997 Each metric can have several value dimensions; a MULTITSFRAME may also hold one metric. + 998 Exchanged with adapters, e.g. in the Data-Attributes header of generic REST adapter endpoints + 999 or the __metadata__ field of the direct provisioning adapter's wrapped format. +1000 """ +1001 +1002 model_config = ConfigDict(json_schema_extra={"examples": [MULTITSFRAME_METADATA_EXAMPLE]}) +1003 +1004 _data_type: ClassVar[TimeseriesDataType] = "MULTITSFRAME" +1005 +1006 dataset_metadata: MultiTSFrameDatasetMetadata +1007 metrics: tuple[MetricMetadata, ...] = Field( +1008 ..., +1009 description="Metadata of the metrics: one for each metric in the data. Each metric must" +1010 " have the field named by dataset_metadata.metric_key as a string, unique among the" +1011 " metrics.", +1012 ) +1013
[docs] -1011 def narrowed_to(self, metric_id: str) -> SingleTSFrameMetadata: -1012 """The metadata of the SINGLETSFRAME of one metric, KeyError if it has no metadata -1013 -1014 The metric is shared, not copied. It becomes the single and only queried metric. -1015 """ -1016 return self._converted( -1017 SingleTSFrameMetadata, -1018 dataset_metadata=self.dataset_metadata.model_dump() -1019 | {"single_metric": metric_id, "queried_metrics": [metric_id]}, -1020 metrics=(self.metric(metric_id),), -1021 value_dimensions_shared=self.value_dimensions_shared, -1022 )
+1014 def narrowed_to(self, metric_id: str) -> SingleTSFrameMetadata: +1015 """The metadata of the SINGLETSFRAME of one metric, KeyError if it has no metadata +1016 +1017 The metric is shared, not copied. It becomes the single and only queried metric. +1018 """ +1019 return self._converted( +1020 SingleTSFrameMetadata, +1021 dataset_metadata=self.dataset_metadata.model_dump() +1022 | {"single_metric": metric_id, "queried_metrics": [metric_id]}, +1023 metrics=(self.metric(metric_id),), +1024 value_dimensions_shared=self.value_dimensions_shared, +1025 )
-1023 -1024 -1025METADATA_MODELS: dict[str, type[_TimeseriesMetadataBase]] = { -1026 "SERIES": SeriesMetadata, -1027 "SINGLETSFRAME": SingleTSFrameMetadata, -1028 "MULTITSFRAME": MultiTSFrameMetadata, -1029} -1030 -1031 +1026 +1027 +1028METADATA_MODELS: dict[str, type[_TimeseriesMetadataBase]] = { +1029 "SERIES": SeriesMetadata, +1030 "SINGLETSFRAME": SingleTSFrameMetadata, +1031 "MULTITSFRAME": MultiTSFrameMetadata, +1032} +1033 +1034
[docs] -1032class MetadataIssueSeverity(StrEnum): -1033 """Severity of an issue found when checking metadata against the metadata conventions -1034 -1035 error: Violates the conventions, so helpers and components may misinterpret the metadata. -1036 warning: Likely a mistake, e.g. metadata not matching the data. -1037 """ -1038 -1039 ERROR = "error" -1040 WARNING = "warning"
- +1035class MetadataIssueSeverity(StrEnum): +1036 """Severity of an issue found when checking metadata against the metadata conventions +1037 +1038 error: Violates the conventions, so consumers like components may misinterpret the metadata. +1039 warning: Likely a mistake, e.g. metadata not matching the data. +1040 """ 1041 -1042 +1042 ERROR = "error" +1043 WARNING = "warning" + +1044 +1045
[docs] -1043class MetadataConventionIssue(BaseModel): -1044 """A metadata validation problem, reported as an error or warning. -1045 -1046 Attributes: -1047 severity (MetadataIssueSeverity): Error for a violation; warning for a likely mistake. -1048 location (str): Path to the affected field, e.g. "metrics[0].name"; empty for the -1049 metadata as a whole. -1050 message (str): Explanation of the problem. -1051 """ -1052 -1053 severity: MetadataIssueSeverity -1054 location: str = Field( -1055 ..., -1056 description="Where the issue occurs, e.g. metrics[0].value_dimensions[1].column in the" -1057 " checked metadata. Empty for the checked object as a whole.", -1058 ) -1059 message: str
+1046class MetadataConventionIssue(BaseModel): +1047 """A problem found when checking metadata against the metadata conventions +1048 +1049 An error for a violation of the conventions, a warning for a likely mistake. +1050 """ +1051 +1052 severity: MetadataIssueSeverity +1053 location: str = Field( +1054 ..., +1055 description="Where the issue occurs, e.g. metrics[0].value_dimensions[1].column in the" +1056 " checked metadata. Empty for the checked object as a whole.", +1057 ) +1058 message: str = Field(..., description="Explanation of the problem") +1059 1060 -1061
[docs] -1062class MetadataConventionError(ValueError): -1063 """Raised when required metadata is missing, invalid or of the wrong type. -1064 -1065 Attributes: -1066 issues (list[MetadataConventionIssue]): Individual problems with severity, location -1067 and message. -1068 """ -1069 -1070 def __init__(self, issues: list[MetadataConventionIssue]) -> None: -1071 self.issues = issues -1072 super().__init__( -1073 "Violates the metadata conventions: " -1074 + _listing([f"{issue.location}: {issue.message}" for issue in issues], 5) -1075 )
+1061class MetadataConventionError(ValueError): +1062 """Raised when required metadata is missing, invalid or of the wrong type. +1063 +1064 Attributes: +1065 issues (list[MetadataConventionIssue]): Individual problems with severity, location +1066 and message. +1067 """ +1068 +1069 def __init__(self, issues: list[MetadataConventionIssue]) -> None: +1070 self.issues = issues +1071 super().__init__( +1072 "Violates the metadata conventions: " +1073 + _listing([f"{issue.location}: {issue.message}" for issue in issues], 5) +1074 ) +1075 1076 -1077 -1078def _error(location: str, message: str) -> MetadataConventionIssue: -1079 return MetadataConventionIssue( -1080 severity=MetadataIssueSeverity.ERROR, location=location, message=message -1081 ) +1077def _error(location: str, message: str) -> MetadataConventionIssue: +1078 return MetadataConventionIssue( +1079 severity=MetadataIssueSeverity.ERROR, location=location, message=message +1080 ) +1081 1082 -1083 -1084def _warning(location: str, message: str) -> MetadataConventionIssue: -1085 return MetadataConventionIssue( -1086 severity=MetadataIssueSeverity.WARNING, location=location, message=message -1087 ) +1083def _warning(location: str, message: str) -> MetadataConventionIssue: +1084 return MetadataConventionIssue( +1085 severity=MetadataIssueSeverity.WARNING, location=location, message=message +1086 ) +1087 1088 -1089 -1090def _location(loc: tuple[int | str, ...]) -> str: -1091 """Format a pydantic error location like metrics[0].value_dimensions[1].column""" -1092 location = "" -1093 for part in loc: -1094 if isinstance(part, int): -1095 location += f"[{part}]" -1096 else: -1097 location += f".{part}" if location != "" else part -1098 return location +1089def _location(loc: tuple[int | str, ...]) -> str: +1090 """Format a pydantic error location like metrics[0].value_dimensions[1].column""" +1091 location = "" +1092 for part in loc: +1093 if isinstance(part, int): +1094 location += f"[{part}]" +1095 else: +1096 location += f".{part}" if location != "" else part +1097 return location +1098 1099 -1100 -1101def _listing(items: list[str], max_items: int = 10) -> str: -1102 listed = ", ".join(f'"{item}"' for item in items[:max_items]) -1103 if len(items) > max_items: -1104 listed += f" and {len(items) - max_items} more" -1105 return listed +1100def _listing(items: list[str], max_items: int = 10) -> str: +1101 listed = ", ".join(f'"{item}"' for item in items[:max_items]) +1102 if len(items) > max_items: +1103 listed += f" and {len(items) - max_items} more" +1104 return listed +1105 1106 -1107 -1108def _check_duplicate_columns( -1109 value_dimensions: list[dict[str, Any]], location: str -1110) -> list[MetadataConventionIssue]: -1111 counts = Counter(value_dimension["column"] for value_dimension in value_dimensions) -1112 return [ -1113 _error(location, f'Value dimension column "{column}" occurs more than once.') -1114 for column, count in counts.items() -1115 if count > 1 -1116 ] +1107def _check_duplicate_columns( +1108 value_dimensions: list[dict[str, Any]], location: str +1109) -> list[MetadataConventionIssue]: +1110 counts = Counter(value_dimension["column"] for value_dimension in value_dimensions) +1111 return [ +1112 _error(location, f'Value dimension column "{column}" occurs more than once.') +1113 for column, count in counts.items() +1114 if count > 1 +1115 ] +1116 1117 -1118 -1119def _check_interval( -1120 obj: dict[str, Any], location: str, start_key: str, end_key: str -1121) -> list[MetadataConventionIssue]: -1122 """Check an interval given by start and end fields of the object at location""" -1123 start = obj.get(start_key) -1124 end = obj.get(end_key) -1125 if (start is None) != (end is None): -1126 missing_key = start_key if start is None else end_key -1127 return [ -1128 _error( -1129 f"{location}.{missing_key}", -1130 f"Start and end of an interval must be provided together, but {missing_key}" -1131 " is missing.", -1132 ) -1133 ] -1134 if start is not None and end is not None and pd.Timestamp(start) > pd.Timestamp(end): -1135 return [_error(f"{location}.{start_key}", f"Start is after end {end}.")] -1136 return [] +1118def _check_interval( +1119 obj: dict[str, Any], location: str, start_key: str, end_key: str +1120) -> list[MetadataConventionIssue]: +1121 """Check an interval given by start and end fields of the object at location""" +1122 start = obj.get(start_key) +1123 end = obj.get(end_key) +1124 if (start is None) != (end is None): +1125 missing_key = start_key if start is None else end_key +1126 return [ +1127 _error( +1128 f"{location}.{missing_key}", +1129 f"Start and end of an interval must be provided together, but {missing_key}" +1130 " is missing.", +1131 ) +1132 ] +1133 if start is not None and end is not None and pd.Timestamp(start) > pd.Timestamp(end): +1134 return [_error(f"{location}.{start_key}", f"Start is after end {end}.")] +1135 return [] +1136 1137 -1138 -1139@lru_cache(maxsize=1024) -1140def _is_frequency(value: str) -> bool: -1141 """Whether value is a Pandas date offset alias or a timedelta string -1142 -1143 A unit is required, since e.g. "1" would be a timedelta string of 1 nanosecond. -1144 """ -1145 if re.search("[A-Za-z]", value) is None: -1146 return False -1147 try: -1148 pd.tseries.frequencies.to_offset(value) -1149 except ValueError: -1150 try: -1151 # e.g. "nan" gives NaT -1152 return bool(pd.notna(pd.to_timedelta(value))) -1153 except ValueError: -1154 return False -1155 return True +1138@lru_cache(maxsize=1024) +1139def _is_frequency(value: str) -> bool: +1140 """Whether value is a Pandas date offset alias or a timedelta string +1141 +1142 A unit is required, since e.g. "1" would be a timedelta string of 1 nanosecond. +1143 """ +1144 if re.search("[A-Za-z]", value) is None: +1145 return False +1146 try: +1147 pd.tseries.frequencies.to_offset(value) +1148 except ValueError: +1149 try: +1150 # e.g. "nan" gives NaT +1151 return bool(pd.notna(pd.to_timedelta(value))) +1152 except ValueError: +1153 return False +1154 return True +1155 1156 -1157 -1158@lru_cache(maxsize=1024) -1159def _duration(value: str) -> pd.Timedelta | None: -1160 """The fixed duration of a frequency, None if calendar based like "MS" or no frequency""" -1161 try: -1162 offset = pd.tseries.frequencies.to_offset(value) -1163 except ValueError: -1164 try: -1165 # e.g. "P1D" or "1 days" -1166 duration = pd.to_timedelta(value) -1167 except ValueError: -1168 return None -1169 return None if pd.isna(duration) else duration -1170 if isinstance(offset, pd.offsets.Day): -1171 # a calendar day since pandas 3, i.e. 24 hours in UTC -1172 return pd.Timedelta(days=offset.n) -1173 if isinstance(offset, pd.offsets.Tick): -1174 return pd.Timedelta(offset) -1175 # calendar based, e.g. months -1176 return None +1157@lru_cache(maxsize=1024) +1158def _duration(value: str) -> pd.Timedelta | None: +1159 """The fixed duration of a frequency, None if calendar based like "MS" or no frequency""" +1160 try: +1161 offset = pd.tseries.frequencies.to_offset(value) +1162 except ValueError: +1163 try: +1164 # e.g. "P1D" or "1 days" +1165 duration = pd.to_timedelta(value) +1166 except ValueError: +1167 return None +1168 return None if pd.isna(duration) else duration +1169 if isinstance(offset, pd.offsets.Day): +1170 # a calendar day since pandas 3, i.e. 24 hours in UTC +1171 return pd.Timedelta(days=offset.n) +1172 if isinstance(offset, pd.offsets.Tick): +1173 return pd.Timedelta(offset) +1174 # calendar based, e.g. months +1175 return None +1176 1177 -1178 -1179@lru_cache(maxsize=1024) -1180def _offset_count(value: str) -> int: -1181 """The count n of a calendar based frequency, e.g. 3 for "3MS\"""" -1182 return int(pd.tseries.frequencies.to_offset(value).n) +1178@lru_cache(maxsize=1024) +1179def _offset_count(value: str) -> int: +1180 """The count n of a calendar based frequency, e.g. 3 for "3MS\"""" +1181 return int(pd.tseries.frequencies.to_offset(value).n) +1182 1183 -1184 -1185def _not_a_frequency(location: str, value: str) -> MetadataConventionIssue: -1186 return _error( -1187 location, -1188 f'"{value}" is not a Pandas date offset alias or timedelta string, e.g. "15min".', -1189 ) +1184def _not_a_frequency(location: str, value: str) -> MetadataConventionIssue: +1185 return _error( +1186 location, +1187 f'"{value}" is not a Pandas date offset alias or timedelta string, e.g. "15min".', +1188 ) +1189 1190 -1191 -1192def _check_frequency( -1193 obj: dict[str, Any], field: str, location: str -1194) -> list[MetadataConventionIssue]: -1195 """Check that a frequency field is a positive frequency""" -1196 value = obj.get(field) -1197 if value is None: -1198 return [] -1199 if not _is_frequency(value): -1200 return [_not_a_frequency(f"{location}.{field}", value)] -1201 duration = _duration(value) -1202 positive = duration > pd.Timedelta(0) if duration is not None else _offset_count(value) > 0 -1203 return [] if positive else [_error(f"{location}.{field}", f'"{value}" is not positive.')] +1191def _check_frequency( +1192 obj: dict[str, Any], field: str, location: str +1193) -> list[MetadataConventionIssue]: +1194 """Check that a frequency field is a positive frequency""" +1195 value = obj.get(field) +1196 if value is None: +1197 return [] +1198 if not _is_frequency(value): +1199 return [_not_a_frequency(f"{location}.{field}", value)] +1200 duration = _duration(value) +1201 positive = duration > pd.Timedelta(0) if duration is not None else _offset_count(value) > 0 +1202 return [] if positive else [_error(f"{location}.{field}", f'"{value}" is not positive.')] +1203 1204 -1205 -1206def _check_offset( -1207 obj: dict[str, Any], frequency_field: str, offset_field: str, location: str -1208) -> list[MetadataConventionIssue]: -1209 """Check the offset of the timestamps against the intervals of a valid frequency""" -1210 offset = obj[offset_field] -1211 offset_location = f"{location}.{offset_field}" -1212 if not _is_frequency(offset): -1213 return [_not_a_frequency(offset_location, offset)] -1214 offset_duration = _duration(offset) -1215 if offset_duration is None or offset_duration < pd.Timedelta(0): -1216 return [ -1217 _error(offset_location, f'"{offset}" is not a non-negative timedelta, e.g. "4min".') -1218 ] -1219 # None for calendar based frequencies, which offsets cannot be compared with -1220 frequency_duration = _duration(obj[frequency_field]) -1221 if frequency_duration is not None and offset_duration >= frequency_duration: -1222 return [ -1223 _error( -1224 offset_location, -1225 f'"{offset}" is not smaller than {frequency_field} "{obj[frequency_field]}".', -1226 ) -1227 ] -1228 return [] +1205def _check_offset( +1206 obj: dict[str, Any], frequency_field: str, offset_field: str, location: str +1207) -> list[MetadataConventionIssue]: +1208 """Check the offset of the timestamps against the intervals of a valid frequency""" +1209 offset = obj[offset_field] +1210 offset_location = f"{location}.{offset_field}" +1211 if not _is_frequency(offset): +1212 return [_not_a_frequency(offset_location, offset)] +1213 offset_duration = _duration(offset) +1214 if offset_duration is None or offset_duration < pd.Timedelta(0): +1215 return [ +1216 _error(offset_location, f'"{offset}" is not a non-negative timedelta, e.g. "4min".') +1217 ] +1218 # None for calendar based frequencies, which offsets cannot be compared with +1219 frequency_duration = _duration(obj[frequency_field]) +1220 if frequency_duration is not None and offset_duration >= frequency_duration: +1221 return [ +1222 _error( +1223 offset_location, +1224 f'"{offset}" is not smaller than {frequency_field} "{obj[frequency_field]}".', +1225 ) +1226 ] +1227 return [] +1228 1229 -1230 -1231def _check_frequency_with_alignment( -1232 obj: dict[str, Any], frequency_field: str, offset_field: str, timezone_field: str, location: str -1233) -> list[MetadataConventionIssue]: -1234 """Check a frequency and the fields aligning its intervals, the offset and the time zone -1235 -1236 They are only meaningful together with the frequency on the same level, since the frequency -1237 of a metric falls back to the one of the dataset metadata as a whole. -1238 """ -1239 issues = _check_frequency(obj, frequency_field, location) -1240 valid_frequency = len(issues) == 0 and obj.get(frequency_field) is not None -1241 for field in (offset_field, timezone_field): -1242 if obj.get(field) is not None and obj.get(frequency_field) is None: -1243 issues.append(_error(f"{location}.{field}", f"Requires {frequency_field}.")) -1244 issues += _check_timezone(obj, location, timezone_field) -1245 if obj.get(offset_field) is not None and valid_frequency: -1246 issues += _check_offset(obj, frequency_field, offset_field, location) -1247 return issues +1230def _check_frequency_with_alignment( +1231 obj: dict[str, Any], frequency_field: str, offset_field: str, timezone_field: str, location: str +1232) -> list[MetadataConventionIssue]: +1233 """Check a frequency and the fields aligning its intervals, the offset and the time zone +1234 +1235 They are only meaningful together with the frequency on the same level, since the frequency +1236 of a metric falls back to the one of the dataset metadata as a whole. +1237 """ +1238 issues = _check_frequency(obj, frequency_field, location) +1239 valid_frequency = len(issues) == 0 and obj.get(frequency_field) is not None +1240 for field in (offset_field, timezone_field): +1241 if obj.get(field) is not None and obj.get(frequency_field) is None: +1242 issues.append(_error(f"{location}.{field}", f"Requires {frequency_field}.")) +1243 issues += _check_timezone(obj, location, timezone_field) +1244 if obj.get(offset_field) is not None and valid_frequency: +1245 issues += _check_offset(obj, frequency_field, offset_field, location) +1246 return issues +1247 1248 -1249 -1250_UTC_OFFSET_PATTERN = re.compile(r"^[+-](0[0-9]|1[0-4]):[0-5][0-9]$") +1249_UTC_OFFSET_PATTERN = re.compile(r"^[+-](0[0-9]|1[0-4]):[0-5][0-9]$") +1250 1251 -1252 -1253@cache -1254def _timezone_names() -> frozenset[str]: -1255 """The IANA time zone names of the tzdata package, independent of the system -1256 -1257 Not zoneinfo.available_timezones, which also lists the files of the system's time zone -1258 database, e.g. "localtime" on Debian or Ubuntu, which is no time zone name. -1259 """ -1260 with resources.files("tzdata").joinpath("zones").open(encoding="utf-8") as zones: -1261 return frozenset(line.strip() for line in zones if line.strip() != "") +1252@cache +1253def _timezone_names() -> frozenset[str]: +1254 """The IANA time zone names of the tzdata package, independent of the system +1255 +1256 Not zoneinfo.available_timezones, which also lists the files of the system's time zone +1257 database, e.g. "localtime" on Debian or Ubuntu, which is no time zone name. +1258 """ +1259 with resources.files("tzdata").joinpath("zones").open(encoding="utf-8") as zones: +1260 return frozenset(line.strip() for line in zones if line.strip() != "") +1261 1262 -1263 -1264def _is_timezone(value: str) -> bool: -1265 """Whether value is an IANA time zone name or a fixed UTC offset, e.g. "+02:00" """ -1266 return _UTC_OFFSET_PATTERN.match(value) is not None or value in _timezone_names() +1263def _is_timezone(value: str) -> bool: +1264 """Whether value is an IANA time zone name or a fixed UTC offset, e.g. "+02:00" """ +1265 return _UTC_OFFSET_PATTERN.match(value) is not None or value in _timezone_names() +1266 1267 -1268 -1269def _check_timezone( -1270 obj: dict[str, Any], location: str, field: str = "default_timezone" -1271) -> list[MetadataConventionIssue]: -1272 timezone = obj.get(field) -1273 if timezone is None or _is_timezone(timezone): -1274 return [] -1275 return [ -1276 _error( -1277 f"{location}.{field}", -1278 f'"{timezone}" is neither an IANA time zone name like "Europe/Berlin" nor a UTC' -1279 ' offset like "+02:00".', -1280 ) -1281 ] +1268def _check_timezone( +1269 obj: dict[str, Any], location: str, field: str = "default_timezone" +1270) -> list[MetadataConventionIssue]: +1271 timezone = obj.get(field) +1272 if timezone is None or _is_timezone(timezone): +1273 return [] +1274 return [ +1275 _error( +1276 f"{location}.{field}", +1277 f'"{timezone}" is neither an IANA time zone name like "Europe/Berlin" nor a UTC' +1278 ' offset like "+02:00".', +1279 ) +1280 ] +1281 1282 -1283 -1284def _check_recording( -1285 metric: dict[str, Any], dataset_metadata: dict[str, Any], location: str -1286) -> list[MetadataConventionIssue]: -1287 """Check that the recording information of a metric fits its recording mode""" -1288 issues = [] -1289 recording_mode = metric.get("recording_mode") -1290 if ( -1291 metric.get("recording_aggregation_method") is not None -1292 and recording_mode != RecordingMode.PERIODIC -1293 ): -1294 issues.append( -1295 _error( -1296 f"{location}.recording_aggregation_method", -1297 "Only meaningful for recording_mode periodic.", -1298 ) -1299 ) -1300 if ( -1301 recording_mode == RecordingMode.PERIODIC -1302 and metric.get("data_frequency") is None -1303 and dataset_metadata.get("ref_data_frequency") is None -1304 ): -1305 issues.append( -1306 _error( -1307 f"{location}.recording_mode", -1308 "periodic requires data_frequency or dataset_metadata.ref_data_frequency, the" -1309 " interval of the values.", -1310 ) -1311 ) -1312 if recording_mode != RecordingMode.ON_CHANGE: -1313 issues += [ -1314 _warning( -1315 f"{location}.value_dimensions[{vd_index}].deadband", -1316 "Only meaningful if the recording_mode of the metric is on_change.", -1317 ) -1318 for vd_index, value_dimension in enumerate(metric["value_dimensions"]) -1319 if value_dimension.get("deadband") is not None -1320 ] -1321 return issues +1283def _check_recording( +1284 metric: dict[str, Any], dataset_metadata: dict[str, Any], location: str +1285) -> list[MetadataConventionIssue]: +1286 """Check that the recording information of a metric fits its recording mode""" +1287 issues = [] +1288 recording_mode = metric.get("recording_mode") +1289 if ( +1290 metric.get("recording_aggregation_method") is not None +1291 and recording_mode != RecordingMode.PERIODIC +1292 ): +1293 issues.append( +1294 _error( +1295 f"{location}.recording_aggregation_method", +1296 "Only meaningful for recording_mode periodic.", +1297 ) +1298 ) +1299 if ( +1300 recording_mode == RecordingMode.PERIODIC +1301 and metric.get("data_frequency") is None +1302 and dataset_metadata.get("ref_data_frequency") is None +1303 ): +1304 issues.append( +1305 _error( +1306 f"{location}.recording_mode", +1307 "periodic requires data_frequency or dataset_metadata.ref_data_frequency, the" +1308 " interval of the values.", +1309 ) +1310 ) +1311 if recording_mode != RecordingMode.ON_CHANGE: +1312 issues += [ +1313 _warning( +1314 f"{location}.value_dimensions[{vd_index}].deadband", +1315 "Only meaningful if the recording_mode of the metric is on_change.", +1316 ) +1317 for vd_index, value_dimension in enumerate(metric["value_dimensions"]) +1318 if value_dimension.get("deadband") is not None +1319 ] +1320 return issues +1321 1322 -1323 -1324def _known_fields(model: type[BaseModel]) -> set[str]: -1325 return set(model.model_fields) +1323def _known_fields(model: type[BaseModel]) -> set[str]: +1324 return set(model.model_fields) +1325 1326 -1327 -1328# hints for fields placed where the conventions do not expect them -1329_MISPLACED_FIELD_HINTS: dict[tuple[type[BaseModel], str], str] = { -1330 (MetricMetadata, "unit"): 'Provide the unit of the value dimension "value" under' -1331 " value_dimensions instead.", -1332 (MetricMetadata, "measurement"): 'Provide the measurement of the value dimension "value"' -1333 " under value_dimensions instead.", -1334} +1327# hints for fields placed where the conventions do not expect them +1328_MISPLACED_FIELD_HINTS: dict[tuple[type[BaseModel], str], str] = { +1329 (MetricMetadata, "unit"): 'Provide the unit of the value dimension "value" under' +1330 " value_dimensions instead.", +1331 (MetricMetadata, "measurement"): 'Provide the measurement of the value dimension "value"' +1332 " under value_dimensions instead.", +1333} +1334 1335 -1336 -1337def _unknown_field_issues( -1338 obj: dict[str, Any], model: type[BaseModel], location: str, ignored: set[str] | None = None -1339) -> list[MetadataConventionIssue]: -1340 """Warnings for fields not defined by the conventions, which may be misspelled""" -1341 known = _known_fields(model) -1342 issues = [] -1343 for field in obj: -1344 if field in known or (ignored is not None and field in ignored): -1345 continue -1346 hint = _MISPLACED_FIELD_HINTS.get((model, field)) -1347 if hint is None: -1348 snake_case = re.sub(r"(?<!^)(?=[A-Z])", "_", field).lower() -1349 close_matches = ( -1350 [snake_case] -1351 if snake_case in known -1352 else difflib.get_close_matches(field, known, n=1, cutoff=0.8) -1353 ) -1354 hint = ( -1355 f'Did you mean "{close_matches[0]}"?' -1356 if len(close_matches) > 0 -1357 else "Additional fields are allowed, but make sure it is not misspelled." -1358 ) -1359 issues.append( -1360 _warning( -1361 f"{location}.{field}" if location != "" else field, -1362 f'Unknown field "{field}". {hint}', -1363 ) -1364 ) -1365 return issues +1336def _unknown_field_issues( +1337 obj: dict[str, Any], model: type[BaseModel], location: str, ignored: set[str] | None = None +1338) -> list[MetadataConventionIssue]: +1339 """Warnings for fields not defined by the conventions, which may be misspelled""" +1340 known = _known_fields(model) +1341 issues = [] +1342 for field in obj: +1343 if field in known or (ignored is not None and field in ignored): +1344 continue +1345 hint = _MISPLACED_FIELD_HINTS.get((model, field)) +1346 if hint is None: +1347 snake_case = re.sub(r"(?<!^)(?=[A-Z])", "_", field).lower() +1348 close_matches = ( +1349 [snake_case] +1350 if snake_case in known +1351 else difflib.get_close_matches(field, known, n=1, cutoff=0.8) +1352 ) +1353 hint = ( +1354 f'Did you mean "{close_matches[0]}"?' +1355 if len(close_matches) > 0 +1356 else "Additional fields are allowed, but make sure it is not misspelled." +1357 ) +1358 issues.append( +1359 _warning( +1360 f"{location}.{field}" if location != "" else field, +1361 f'Unknown field "{field}". {hint}', +1362 ) +1363 ) +1364 return issues +1365 1366 -1367 -1368def _dicts(value: Any) -> list[tuple[int, dict[str, Any]]]: -1369 """The dicts in value with their index, if value is a list""" -1370 if not isinstance(value, list): -1371 return [] -1372 return [(index, item) for index, item in enumerate(value) if isinstance(item, dict)] +1367def _dicts(value: Any) -> list[tuple[int, dict[str, Any]]]: +1368 """The dicts in value with their index, if value is a list""" +1369 if not isinstance(value, list): +1370 return [] +1371 return [(index, item) for index, item in enumerate(value) if isinstance(item, dict)] +1372 1373 -1374 -1375def _check_unknown_fields( -1376 attrs: Any, model: type[_TimeseriesMetadataBase] -1377) -> list[MetadataConventionIssue]: -1378 """Warnings for fields not defined by the conventions -1379 -1380 Also applied to structurally invalid metadata as far as possible, since a misspelled field -1381 often explains a missing required one. -1382 """ -1383 if not isinstance(attrs, dict): -1384 return [] -1385 issues = _unknown_field_issues(attrs, model, "") -1386 -1387 dataset_metadata = attrs.get("dataset_metadata") -1388 metric_key = None -1389 if isinstance(dataset_metadata, dict): -1390 metric_key = dataset_metadata.get("metric_key") -1391 dataset_metadata_model = model.model_fields["dataset_metadata"].annotation -1392 assert isinstance(dataset_metadata_model, type) # for mypy # noqa: S101 -1393 issues += _unknown_field_issues( -1394 dataset_metadata, dataset_metadata_model, "dataset_metadata" -1395 ) -1396 if isinstance(dataset_metadata.get("signal_group"), dict): -1397 issues += _unknown_field_issues( -1398 dataset_metadata["signal_group"], -1399 SignalGroupMetadata, -1400 "dataset_metadata.signal_group", -1401 ) -1402 -1403 for index, value_dimension in _dicts(attrs.get("value_dimensions_shared")): -1404 issues += _unknown_field_issues( -1405 value_dimension, SharedValueDimensionMetadata, f"value_dimensions_shared[{index}]" -1406 ) -1407 for index, metric in _dicts(attrs.get("metrics")): -1408 location = f"metrics[{index}]" -1409 # the field identifying the metric may be any field -1410 issues += _unknown_field_issues(metric, MetricMetadata, location, ignored={str(metric_key)}) -1411 for vd_index, value_dimension in _dicts(metric.get("value_dimensions")): -1412 issues += _unknown_field_issues( -1413 value_dimension, ValueDimensionMetadata, f"{location}.value_dimensions[{vd_index}]" -1414 ) -1415 for comment_index, comment in _dicts(metric.get("comments")): -1416 comment_location = f"{location}.comments[{comment_index}]" -1417 issues += _unknown_field_issues(comment, MetricComment, comment_location) -1418 for reply_index, reply in _dicts(comment.get("replies")): -1419 issues += _unknown_field_issues( -1420 reply, MetricCommentReply, f"{comment_location}.replies[{reply_index}]" -1421 ) -1422 return issues +1374def _check_unknown_fields( +1375 attrs: Any, model: type[_TimeseriesMetadataBase] +1376) -> list[MetadataConventionIssue]: +1377 """Warnings for fields not defined by the conventions +1378 +1379 Also applied to structurally invalid metadata as far as possible, since a misspelled field +1380 often explains a missing required one. +1381 """ +1382 if not isinstance(attrs, dict): +1383 return [] +1384 issues = _unknown_field_issues(attrs, model, "") +1385 +1386 dataset_metadata = attrs.get("dataset_metadata") +1387 metric_key = None +1388 if isinstance(dataset_metadata, dict): +1389 metric_key = dataset_metadata.get("metric_key") +1390 dataset_metadata_model = model.model_fields["dataset_metadata"].annotation +1391 assert isinstance(dataset_metadata_model, type) # for mypy # noqa: S101 +1392 issues += _unknown_field_issues( +1393 dataset_metadata, dataset_metadata_model, "dataset_metadata" +1394 ) +1395 if isinstance(dataset_metadata.get("signal_group"), dict): +1396 issues += _unknown_field_issues( +1397 dataset_metadata["signal_group"], +1398 SignalGroupMetadata, +1399 "dataset_metadata.signal_group", +1400 ) +1401 +1402 for index, value_dimension in _dicts(attrs.get("value_dimensions_shared")): +1403 issues += _unknown_field_issues( +1404 value_dimension, SharedValueDimensionMetadata, f"value_dimensions_shared[{index}]" +1405 ) +1406 for index, metric in _dicts(attrs.get("metrics")): +1407 location = f"metrics[{index}]" +1408 # the field identifying the metric may be any field +1409 issues += _unknown_field_issues(metric, MetricMetadata, location, ignored={str(metric_key)}) +1410 for vd_index, value_dimension in _dicts(metric.get("value_dimensions")): +1411 issues += _unknown_field_issues( +1412 value_dimension, ValueDimensionMetadata, f"{location}.value_dimensions[{vd_index}]" +1413 ) +1414 for comment_index, comment in _dicts(metric.get("comments")): +1415 comment_location = f"{location}.comments[{comment_index}]" +1416 issues += _unknown_field_issues(comment, MetricComment, comment_location) +1417 for reply_index, reply in _dicts(comment.get("replies")): +1418 issues += _unknown_field_issues( +1419 reply, MetricCommentReply, f"{comment_location}.replies[{reply_index}]" +1420 ) +1421 return issues +1422 1423 -1424 -1425_DEPRECATED_METRIC_FIELDS = { -1426 name: field.deprecated -1427 for name, field in MetricMetadata.model_fields.items() -1428 if field.deprecated is not None -1429} +1424_DEPRECATED_METRIC_FIELDS = { +1425 name: field.deprecated +1426 for name, field in MetricMetadata.model_fields.items() +1427 if field.deprecated is not None +1428} +1429 1430 -1431 -1432def _check_consistency(attrs: dict[str, Any]) -> list[MetadataConventionIssue]: -1433 """Check the rules of the conventions which the JSON Schema cannot express -1434 -1435 Expects attrs to be valid against the JSON Schema of the data type. -1436 """ -1437 issues: list[MetadataConventionIssue] = [] -1438 dataset_metadata = attrs["dataset_metadata"] -1439 metrics = attrs["metrics"] -1440 metric_key = dataset_metadata["metric_key"] -1441 -1442 metric_ids = [] -1443 for index, metric in enumerate(metrics): -1444 metric_id = metric.get(metric_key) -1445 if not isinstance(metric_id, str): -1446 issues.append( -1447 _error( -1448 f"metrics[{index}]", -1449 f'Metric has no string field "{metric_key}", which identifies metrics' -1450 " according to dataset_metadata.metric_key.", -1451 ) -1452 ) -1453 else: -1454 metric_ids.append(metric_id) -1455 issues += [ -1456 _warning(f"metrics[{index}].{name}", f"Deprecated. {deprecated}") -1457 for name, deprecated in _DEPRECATED_METRIC_FIELDS.items() -1458 if metric.get(name) is not None -1459 ] -1460 issues += _check_duplicate_columns( -1461 metric["value_dimensions"], f"metrics[{index}].value_dimensions" -1462 ) -1463 issues += _check_frequency_with_alignment( -1464 metric, -1465 "data_frequency", -1466 "data_frequency_offset", -1467 "data_frequency_timezone", -1468 f"metrics[{index}]", -1469 ) -1470 issues += _check_timezone(metric, f"metrics[{index}]") -1471 issues += _check_recording(metric, dataset_metadata, f"metrics[{index}]") -1472 for comment_index, comment in enumerate(metric.get("comments") or []): -1473 issues += _check_interval( -1474 comment, f"metrics[{index}].comments[{comment_index}]", "start_date", "end_date" -1475 ) -1476 issues += [ -1477 _error("metrics", f'Metric "{metric_id}" occurs more than once.') -1478 for metric_id, count in Counter(metric_ids).items() -1479 if count > 1 -1480 ] -1481 issues += _check_duplicate_columns( -1482 attrs.get("value_dimensions_shared") or [], "value_dimensions_shared" -1483 ) -1484 -1485 single_metric = dataset_metadata.get("single_metric") -1486 if single_metric is not None and len(metrics) > 1: -1487 issues.append( -1488 _error( -1489 "dataset_metadata.single_metric", -1490 f"Must not be set, since {len(metrics)} metrics are present under metrics.", -1491 ) -1492 ) -1493 elif single_metric is not None and len(metric_ids) == 1 and single_metric not in metric_ids: -1494 issues.append( -1495 _error( -1496 "dataset_metadata.single_metric", -1497 f'"{single_metric}" is not the "{metric_key}" of the metric under metrics.', -1498 ) -1499 ) -1500 elif single_metric is not None and single_metric not in dataset_metadata["queried_metrics"]: -1501 issues.append(_error("dataset_metadata.single_metric", "Missing in queried_metrics.")) -1502 -1503 if dataset_metadata.get("only_invalidate") is True and ( -1504 dataset_metadata.get("invalidate_dataset") is False -1505 ): -1506 issues.append( -1507 _error( -1508 "dataset_metadata.only_invalidate", -1509 "Cannot be true if invalidate_dataset is false.", -1510 ) -1511 ) -1512 -1513 if dataset_metadata.get("ref_dataset_discrete") is not True: -1514 issues += [ -1515 _error( -1516 f"dataset_metadata.{field}", -1517 "Required unless dataset_metadata.ref_dataset_discrete is true.", -1518 ) -1519 for field in REF_INTERVAL_FIELDS -1520 if dataset_metadata.get(field) is None -1521 ] -1522 issues += _check_interval( -1523 dataset_metadata, -1524 "dataset_metadata", -1525 "ref_interval_start_timestamp", -1526 "ref_interval_end_timestamp", -1527 ) -1528 issues += _check_interval( -1529 dataset_metadata, -1530 "dataset_metadata", -1531 "invalidation_interval_start", -1532 "invalidation_interval_end", -1533 ) -1534 issues += _check_frequency_with_alignment( -1535 dataset_metadata, -1536 "ref_data_frequency", -1537 "ref_data_frequency_offset", -1538 "ref_data_frequency_timezone", -1539 "dataset_metadata", -1540 ) -1541 issues += _check_frequency(dataset_metadata, "default_aggregation_interval", "dataset_metadata") -1542 issues += _check_timezone(dataset_metadata, "dataset_metadata") -1543 return issues +1431def _check_consistency(attrs: dict[str, Any]) -> list[MetadataConventionIssue]: +1432 """Check the rules of the conventions which the JSON Schema cannot express +1433 +1434 Expects attrs to be valid against the JSON Schema of the data type. +1435 """ +1436 issues: list[MetadataConventionIssue] = [] +1437 dataset_metadata = attrs["dataset_metadata"] +1438 metrics = attrs["metrics"] +1439 metric_key = dataset_metadata["metric_key"] +1440 +1441 metric_ids = [] +1442 for index, metric in enumerate(metrics): +1443 metric_id = metric.get(metric_key) +1444 if not isinstance(metric_id, str): +1445 issues.append( +1446 _error( +1447 f"metrics[{index}]", +1448 f'Metric has no string field "{metric_key}", which identifies metrics' +1449 " according to dataset_metadata.metric_key.", +1450 ) +1451 ) +1452 else: +1453 metric_ids.append(metric_id) +1454 issues += [ +1455 _warning(f"metrics[{index}].{name}", f"Deprecated. {deprecated}") +1456 for name, deprecated in _DEPRECATED_METRIC_FIELDS.items() +1457 if metric.get(name) is not None +1458 ] +1459 issues += _check_duplicate_columns( +1460 metric["value_dimensions"], f"metrics[{index}].value_dimensions" +1461 ) +1462 issues += _check_frequency_with_alignment( +1463 metric, +1464 "data_frequency", +1465 "data_frequency_offset", +1466 "data_frequency_timezone", +1467 f"metrics[{index}]", +1468 ) +1469 issues += _check_timezone(metric, f"metrics[{index}]") +1470 issues += _check_recording(metric, dataset_metadata, f"metrics[{index}]") +1471 for comment_index, comment in enumerate(metric.get("comments") or []): +1472 issues += _check_interval( +1473 comment, f"metrics[{index}].comments[{comment_index}]", "start_date", "end_date" +1474 ) +1475 issues += [ +1476 _error("metrics", f'Metric "{metric_id}" occurs more than once.') +1477 for metric_id, count in Counter(metric_ids).items() +1478 if count > 1 +1479 ] +1480 issues += _check_duplicate_columns( +1481 attrs.get("value_dimensions_shared") or [], "value_dimensions_shared" +1482 ) +1483 +1484 single_metric = dataset_metadata.get("single_metric") +1485 if single_metric is not None and len(metrics) > 1: +1486 issues.append( +1487 _error( +1488 "dataset_metadata.single_metric", +1489 f"Must not be set, since {len(metrics)} metrics are present under metrics.", +1490 ) +1491 ) +1492 elif single_metric is not None and len(metric_ids) == 1 and single_metric not in metric_ids: +1493 issues.append( +1494 _error( +1495 "dataset_metadata.single_metric", +1496 f'"{single_metric}" is not the "{metric_key}" of the metric under metrics.', +1497 ) +1498 ) +1499 elif single_metric is not None and single_metric not in dataset_metadata["queried_metrics"]: +1500 issues.append(_error("dataset_metadata.single_metric", "Missing in queried_metrics.")) +1501 +1502 if dataset_metadata.get("only_invalidate") is True and ( +1503 dataset_metadata.get("invalidate_dataset") is False +1504 ): +1505 issues.append( +1506 _error( +1507 "dataset_metadata.only_invalidate", +1508 "Cannot be true if invalidate_dataset is false.", +1509 ) +1510 ) +1511 +1512 if dataset_metadata.get("ref_dataset_discrete") is not True: +1513 issues += [ +1514 _error( +1515 f"dataset_metadata.{field}", +1516 "Required unless dataset_metadata.ref_dataset_discrete is true.", +1517 ) +1518 for field in REF_INTERVAL_FIELDS +1519 if dataset_metadata.get(field) is None +1520 ] +1521 issues += _check_interval( +1522 dataset_metadata, +1523 "dataset_metadata", +1524 "ref_interval_start_timestamp", +1525 "ref_interval_end_timestamp", +1526 ) +1527 issues += _check_interval( +1528 dataset_metadata, +1529 "dataset_metadata", +1530 "invalidation_interval_start", +1531 "invalidation_interval_end", +1532 ) +1533 issues += _check_frequency_with_alignment( +1534 dataset_metadata, +1535 "ref_data_frequency", +1536 "ref_data_frequency_offset", +1537 "ref_data_frequency_timezone", +1538 "dataset_metadata", +1539 ) +1540 issues += _check_frequency(dataset_metadata, "default_aggregation_interval", "dataset_metadata") +1541 issues += _check_timezone(dataset_metadata, "dataset_metadata") +1542 return issues +1543 1544 -1545 -1546_CLOSED_BY_INTERVAL_TYPE: dict[str, Literal["both", "neither", "left", "right"]] = { -1547 "closed": "both", -1548 "open": "neither", -1549 "left_closed": "left", -1550 "right_open": "left", -1551 "right_closed": "right", -1552 "left_open": "right", -1553} +1545_CLOSED_BY_INTERVAL_TYPE: dict[str, Literal["both", "neither", "left", "right"]] = { +1546 "closed": "both", +1547 "open": "neither", +1548 "left_closed": "left", +1549 "right_open": "left", +1550 "right_closed": "right", +1551 "left_open": "right", +1552} +1553 1554 -1555 -1556def _check_data_in_ref_interval( -1557 dataset_metadata: dict[str, Any], data: pd.Series | pd.DataFrame -1558) -> list[MetadataConventionIssue]: -1559 """Warn if the data has timestamps outside of the ref interval""" -1560 start = dataset_metadata.get("ref_interval_start_timestamp") -1561 end = dataset_metadata.get("ref_interval_end_timestamp") -1562 interval_type = dataset_metadata.get("ref_interval_type") -1563 timestamps = data.index if isinstance(data, pd.Series) else data.get("timestamp") -1564 if ( -1565 dataset_metadata.get("ref_dataset_discrete") is True -1566 or start is None -1567 or end is None -1568 or interval_type is None -1569 or pd.Timestamp(start) > pd.Timestamp(end) -1570 or timestamps is None -1571 or len(timestamps) == 0 -1572 or not pd.api.types.is_datetime64_any_dtype(timestamps) -1573 ): -1574 return [] -1575 ref_interval = pd.Interval( -1576 pd.Timestamp(start), pd.Timestamp(end), closed=_CLOSED_BY_INTERVAL_TYPE[interval_type] -1577 ) -1578 # naive timestamps are considered to be UTC -1579 first, last = ( -1580 timestamp if timestamp.tzinfo is not None else timestamp.tz_localize("UTC") -1581 for timestamp in (timestamps.min(), timestamps.max()) -1582 ) -1583 if first in ref_interval and last in ref_interval: -1584 return [] -1585 return [ -1586 _warning( -1587 "dataset_metadata", -1588 f"The data has timestamps outside of the {interval_type} ref interval: The first is" -1589 f" {first.isoformat()}, the last {last.isoformat()}.", -1590 ) -1591 ] +1555def _check_data_in_ref_interval( +1556 dataset_metadata: dict[str, Any], data: pd.Series | pd.DataFrame +1557) -> list[MetadataConventionIssue]: +1558 """Warn if the data has timestamps outside of the ref interval""" +1559 start = dataset_metadata.get("ref_interval_start_timestamp") +1560 end = dataset_metadata.get("ref_interval_end_timestamp") +1561 interval_type = dataset_metadata.get("ref_interval_type") +1562 timestamps = data.index if isinstance(data, pd.Series) else data.get("timestamp") +1563 if ( +1564 dataset_metadata.get("ref_dataset_discrete") is True +1565 or start is None +1566 or end is None +1567 or interval_type is None +1568 or pd.Timestamp(start) > pd.Timestamp(end) +1569 or timestamps is None +1570 or len(timestamps) == 0 +1571 or not pd.api.types.is_datetime64_any_dtype(timestamps) +1572 ): +1573 return [] +1574 ref_interval = pd.Interval( +1575 pd.Timestamp(start), pd.Timestamp(end), closed=_CLOSED_BY_INTERVAL_TYPE[interval_type] +1576 ) +1577 # naive timestamps are considered to be UTC +1578 first, last = ( +1579 timestamp if timestamp.tzinfo is not None else timestamp.tz_localize("UTC") +1580 for timestamp in (timestamps.min(), timestamps.max()) +1581 ) +1582 if first in ref_interval and last in ref_interval: +1583 return [] +1584 return [ +1585 _warning( +1586 "dataset_metadata", +1587 f"The data has timestamps outside of the {interval_type} ref interval: The first is" +1588 f" {first.isoformat()}, the last {last.isoformat()}.", +1589 ) +1590 ] +1591 1592 -1593 -1594def _check_metrics_of_data( -1595 attrs: dict[str, Any], data_type: TimeseriesDataType, data: pd.Series | pd.DataFrame | None -1596) -> list[MetadataConventionIssue]: -1597 """Check that the metrics of the data and the queried metrics are described""" -1598 issues: list[MetadataConventionIssue] = [] -1599 dataset_metadata = attrs["dataset_metadata"] -1600 # metrics without a string id are reported by _check_consistency -1601 metric_ids = { -1602 metric_id -1603 for metric in attrs["metrics"] -1604 if isinstance(metric_id := metric.get(dataset_metadata["metric_key"]), str) -1605 } -1606 data_metrics: list[str] = [] -1607 if data_type == "MULTITSFRAME" and isinstance(data, pd.DataFrame) and "metric" in data.columns: -1608 data_metrics = [str(metric) for metric in data["metric"].unique()] -1609 if len(missing := sorted(set(data_metrics) - metric_ids)) > 0: -1610 issues.append( -1611 _error( -1612 "metrics", -1613 f"Metrics of the metric column missing under metrics: {_listing(missing)}", -1614 ) -1615 ) -1616 queried_metrics = dataset_metadata["queried_metrics"] -1617 if len(unqueried := sorted(set(data_metrics) - set(queried_metrics))) > 0: -1618 issues.append( -1619 _error( -1620 "dataset_metadata.queried_metrics", -1621 f"Metrics of the metric column missing in queried_metrics:" -1622 f" {_listing(unqueried)}", -1623 ) -1624 ) -1625 # queried metrics occurring in the data are reported as missing under metrics above -1626 undescribed_metrics = sorted( -1627 set(dataset_metadata["queried_metrics"]) - metric_ids - set(data_metrics) -1628 ) -1629 if len(undescribed_metrics) > 0: -1630 issues.append( -1631 _warning( -1632 "metrics", -1633 f"Queried metrics without metadata under metrics: {_listing(undescribed_metrics)}", -1634 ) -1635 ) -1636 return issues +1593def _check_metrics_of_data( +1594 attrs: dict[str, Any], data_type: TimeseriesDataType, data: pd.Series | pd.DataFrame | None +1595) -> list[MetadataConventionIssue]: +1596 """Check that the metrics of the data and the queried metrics are described""" +1597 issues: list[MetadataConventionIssue] = [] +1598 dataset_metadata = attrs["dataset_metadata"] +1599 # metrics without a string id are reported by _check_consistency +1600 metric_ids = { +1601 metric_id +1602 for metric in attrs["metrics"] +1603 if isinstance(metric_id := metric.get(dataset_metadata["metric_key"]), str) +1604 } +1605 data_metrics: list[str] = [] +1606 if data_type == "MULTITSFRAME" and isinstance(data, pd.DataFrame) and "metric" in data.columns: +1607 data_metrics = [str(metric) for metric in data["metric"].unique()] +1608 if len(missing := sorted(set(data_metrics) - metric_ids)) > 0: +1609 issues.append( +1610 _error( +1611 "metrics", +1612 f"Metrics of the metric column missing under metrics: {_listing(missing)}", +1613 ) +1614 ) +1615 queried_metrics = dataset_metadata["queried_metrics"] +1616 if len(unqueried := sorted(set(data_metrics) - set(queried_metrics))) > 0: +1617 issues.append( +1618 _error( +1619 "dataset_metadata.queried_metrics", +1620 f"Metrics of the metric column missing in queried_metrics:" +1621 f" {_listing(unqueried)}", +1622 ) +1623 ) +1624 # queried metrics occurring in the data are reported as missing under metrics above +1625 undescribed_metrics = sorted( +1626 set(dataset_metadata["queried_metrics"]) - metric_ids - set(data_metrics) +1627 ) +1628 if len(undescribed_metrics) > 0: +1629 issues.append( +1630 _warning( +1631 "metrics", +1632 f"Queried metrics without metadata under metrics: {_listing(undescribed_metrics)}", +1633 ) +1634 ) +1635 return issues +1636 1637 -1638 -1639def _check_consistency_with_data_type( -1640 attrs: dict[str, Any], data_type: TimeseriesDataType, data: pd.Series | pd.DataFrame | None -1641) -> list[MetadataConventionIssue]: -1642 """Check the rules specific to the data type and whether the metadata matches the data""" -1643 metrics = attrs["metrics"] -1644 issues = _check_metrics_of_data(attrs, data_type, data) -1645 if data is not None: -1646 issues += _check_data_in_ref_interval(attrs["dataset_metadata"], data) -1647 -1648 # the value columns of frames are only known from their data -1649 value_columns: set[str] | None = None -1650 if data_type == "SERIES": -1651 value_columns = {"value"} -1652 elif isinstance(data, pd.DataFrame): -1653 value_columns = set(data.columns) - ( -1654 {"timestamp", "metric"} if data_type == "MULTITSFRAME" else {"timestamp"} -1655 ) -1656 if value_columns is None: -1657 return issues -1658 -1659 for index, metric in enumerate(metrics): -1660 described_columns = { -1661 value_dimension["column"] for value_dimension in metric["value_dimensions"] -1662 } -1663 if len(undescribed := sorted(value_columns - described_columns)) > 0: -1664 issues.append( -1665 _error( -1666 f"metrics[{index}].value_dimensions", -1667 f"Value columns of the {data_type} without value dimension:" -1668 f" {_listing(undescribed)}", -1669 ) -1670 ) -1671 -1672 value_dimension_lists = [ -1673 (f"metrics[{index}].value_dimensions", metric["value_dimensions"]) -1674 for index, metric in enumerate(metrics) -1675 ] + [("value_dimensions_shared", attrs.get("value_dimensions_shared") or [])] -1676 for location, value_dimensions in value_dimension_lists: -1677 for index, value_dimension in enumerate(value_dimensions): -1678 if value_dimension["column"] not in value_columns: -1679 issues.append( -1680 _warning( -1681 f"{location}[{index}].column", -1682 f'Column "{value_dimension["column"]}" is not a value column of the' -1683 f" {data_type}.", -1684 ) -1685 ) -1686 return issues +1638def _check_consistency_with_data_type( +1639 attrs: dict[str, Any], data_type: TimeseriesDataType, data: pd.Series | pd.DataFrame | None +1640) -> list[MetadataConventionIssue]: +1641 """Check the rules specific to the data type and whether the metadata matches the data""" +1642 metrics = attrs["metrics"] +1643 issues = _check_metrics_of_data(attrs, data_type, data) +1644 if data is not None: +1645 issues += _check_data_in_ref_interval(attrs["dataset_metadata"], data) +1646 +1647 # the value columns of frames are only known from their data +1648 value_columns: set[str] | None = None +1649 if data_type == "SERIES": +1650 value_columns = {"value"} +1651 elif isinstance(data, pd.DataFrame): +1652 value_columns = set(data.columns) - ( +1653 {"timestamp", "metric"} if data_type == "MULTITSFRAME" else {"timestamp"} +1654 ) +1655 if value_columns is None: +1656 return issues +1657 +1658 for index, metric in enumerate(metrics): +1659 described_columns = { +1660 value_dimension["column"] for value_dimension in metric["value_dimensions"] +1661 } +1662 if len(undescribed := sorted(value_columns - described_columns)) > 0: +1663 issues.append( +1664 _error( +1665 f"metrics[{index}].value_dimensions", +1666 f"Value columns of the {data_type} without value dimension:" +1667 f" {_listing(undescribed)}", +1668 ) +1669 ) +1670 +1671 value_dimension_lists = [ +1672 (f"metrics[{index}].value_dimensions", metric["value_dimensions"]) +1673 for index, metric in enumerate(metrics) +1674 ] + [("value_dimensions_shared", attrs.get("value_dimensions_shared") or [])] +1675 for location, value_dimensions in value_dimension_lists: +1676 for index, value_dimension in enumerate(value_dimensions): +1677 if value_dimension["column"] not in value_columns: +1678 issues.append( +1679 _warning( +1680 f"{location}[{index}].column", +1681 f'Column "{value_dimension["column"]}" is not a value column of the' +1682 f" {data_type}.", +1683 ) +1684 ) +1685 return issues +1686 1687 -1688 -1689_SKIP_RULES = "skip_rules_of_metadata_conventions" +1688_SKIP_RULES = "skip_rules_of_metadata_conventions" +1689 1690 -1691 -1692def _rule_errors( -1693 data: Any, model: _TimeseriesMetadataBase, data_type: TimeseriesDataType -1694) -> list[MetadataConventionIssue]: -1695 """The errors of the rules which JSON Schema cannot express, for valid structure -1696 -1697 The rules are checked on the input if it is in JSON form, otherwise on the model's dump, e.g. -1698 for input with nested models. -1699 """ -1700 -1701 def errors(attrs: dict[str, Any]) -> list[MetadataConventionIssue]: -1702 issues = _check_consistency(attrs) + _check_consistency_with_data_type( -1703 attrs, data_type, None -1704 ) -1705 return [issue for issue in issues if issue.severity == MetadataIssueSeverity.ERROR] -1706 -1707 if isinstance(data, dict): -1708 try: -1709 return errors(data) -1710 except (AttributeError, KeyError, TypeError): -1711 pass -1712 return errors(model.model_dump(mode="json")) +1691def _rule_errors( +1692 data: Any, model: _TimeseriesMetadataBase, data_type: TimeseriesDataType +1693) -> list[MetadataConventionIssue]: +1694 """The errors of the rules which JSON Schema cannot express, for valid structure +1695 +1696 The rules are checked on the input if it is in JSON form, otherwise on the model's dump, e.g. +1697 for input with nested models. +1698 """ +1699 +1700 def errors(attrs: dict[str, Any]) -> list[MetadataConventionIssue]: +1701 issues = _check_consistency(attrs) + _check_consistency_with_data_type( +1702 attrs, data_type, None +1703 ) +1704 return [issue for issue in issues if issue.severity == MetadataIssueSeverity.ERROR] +1705 +1706 if isinstance(data, dict): +1707 try: +1708 return errors(data) +1709 except (AttributeError, KeyError, TypeError): +1710 pass +1711 return errors(model.model_dump(mode="json")) +1712 1713 -1714 -1715def _metric_rule_errors( -1716 meta: _TimeseriesMetadataBase, metric: MetricMetadata -1717) -> list[MetadataConventionIssue]: -1718 """The rule errors concerning a single metric, together with the dataset metadata""" -1719 attrs = { -1720 "dataset_metadata": meta.dataset_metadata.model_dump(mode="json"), -1721 "metrics": [metric.model_dump(mode="json")], -1722 } -1723 if meta.value_dimensions_shared is not None: -1724 attrs["value_dimensions_shared"] = [ -1725 value_dimension.model_dump(mode="json") -1726 for value_dimension in meta.value_dimensions_shared -1727 ] -1728 issues = _check_consistency(attrs) + _check_consistency_with_data_type( -1729 attrs, meta._data_type, None -1730 ) -1731 return [issue for issue in issues if issue.severity == MetadataIssueSeverity.ERROR] +1714def _metric_rule_errors( +1715 meta: _TimeseriesMetadataBase, metric: MetricMetadata +1716) -> list[MetadataConventionIssue]: +1717 """The rule errors concerning a single metric, together with the dataset metadata""" +1718 attrs = { +1719 "dataset_metadata": meta.dataset_metadata.model_dump(mode="json"), +1720 "metrics": [metric.model_dump(mode="json")], +1721 } +1722 if meta.value_dimensions_shared is not None: +1723 attrs["value_dimensions_shared"] = [ +1724 value_dimension.model_dump(mode="json") +1725 for value_dimension in meta.value_dimensions_shared +1726 ] +1727 issues = _check_consistency(attrs) + _check_consistency_with_data_type( +1728 attrs, meta._data_type, None +1729 ) +1730 return [issue for issue in issues if issue.severity == MetadataIssueSeverity.ERROR] +1731 1732 -1733
[docs] -1734def check_metadata_conventions( -1735 obj: pd.Series | pd.DataFrame | Mapping[str, Any], -1736 data_type: TimeseriesDataType | None = None, -1737) -> list[MetadataConventionIssue]: -1738 """Report metadata errors and warnings, optionally checking against pandas data. -1739 -1740 For pandas inputs, also check metric IDs, value columns and time bounds. An empty list -1741 means no issues; warnings alone do not make metadata invalid. -1742 -1743 Args: -1744 obj: Series/DataFrame to check, or a mapping containing metadata. -1745 data_type: -1746 Convention to check; required for mappings. None infers it from pandas input; -1747 see :ref:`metadata-type-selection`. -1748 -1749 Returns: -1750 list[MetadataConventionIssue]: -1751 Issues with severity, location and message. Invalid metadata is reported as -1752 issues. -1753 -1754 Raises: -1755 ValueError: A metadata mapping is provided without data_type. -1756 -1757 Code example: -1758 -1759 Using :ref:`metadata-example-data`, validate the data and its metadata together, or pass -1760 plain metadata with an explicit data type. Issues describe the severity, location and cause: -1761 -1762 .. doctest:: metadata.check_metadata_conventions -1763 -1764 >>> md.check_metadata_conventions(series) -1765 [] -1766 >>> md.check_metadata_conventions(series.attrs, "SERIES") -1767 [] -1768 >>> invalid = series.copy() -1769 >>> invalid.attrs["metrics"][0]["default_timezone"] = "Mars/Base" -1770 >>> issues = md.check_metadata_conventions(invalid) -1771 >>> [(issue.severity.value, issue.location) for issue in issues] -1772 [('error', 'metrics[0].default_timezone')] -1773 >>> print(issues[0].message) # doctest: +ELLIPSIS -1774 "Mars/Base" is neither an IANA time zone name like "Europe/Berlin" nor a UTC offset... -1775 """ -1776 # imported here, since hdhelpers.metadata.attached imports this module -1777 from hdhelpers.metadata.attached import attrs_to_json # noqa: PLC0415 -1778 -1779 data = obj if isinstance(obj, pd.Series | pd.DataFrame) else None -1780 attrs = data.attrs if data is not None else obj -1781 if isinstance(attrs, Mapping): -1782 # attached metadata in its JSON form -1783 attrs = attrs_to_json(attrs) -1784 if data_type is None and isinstance(data, pd.Series): -1785 data_type = "SERIES" -1786 elif data_type is None and isinstance(data, pd.DataFrame): -1787 data_type = "MULTITSFRAME" if "metric" in data.columns else "SINGLETSFRAME" -1788 elif data_type is None: -1789 raise ValueError("The data type is required for checking metadata without data.") -1790 model = METADATA_MODELS[data_type] -1791 -1792 try: -1793 attrs_json = json.dumps(attrs, allow_nan=False) -1794 except (TypeError, ValueError) as e: -1795 return [ -1796 _error( -1797 "", -1798 f"Metadata is not JSON serializable, so it cannot be exchanged with adapters: {e}", -1799 ) -1800 ] -1801 -1802 try: -1803 model.model_validate_json(attrs_json, context={_SKIP_RULES: True}) -1804 except ValidationError as e: -1805 issues = [] -1806 for error in e.errors(): -1807 message = error["msg"].removeprefix("Value error, ") -1808 if (error.get("ctx") or {}).get("pattern") == UTC_TIMESTAMP_PATTERN: -1809 message = ( -1810 "Should be a UTC isoformat timestamp with explicit offset, i.e. ending with Z" -1811 f" or +00:00, got {error['input']!r}." -1812 ) -1813 elif ( -1814 error["loc"] == ("metrics",) -1815 and isinstance(attrs, Mapping) -1816 and isinstance(attrs.get("metrics"), dict) -1817 ): -1818 message += ( -1819 ". Metrics as a mapping is an older metadata convention, the current one" -1820 " expects a list of metric objects." -1821 ) -1822 issues.append(_error(_location(error["loc"]), message)) -1823 return issues + _check_unknown_fields(json.loads(attrs_json), model) -1824 -1825 json_attrs = json.loads(attrs_json) -1826 return ( -1827 _check_consistency(json_attrs) -1828 + _check_consistency_with_data_type(json_attrs, data_type, data) -1829 + _check_unknown_fields(json_attrs, model) -1830 )
+1733def check_metadata_conventions( +1734 obj: pd.Series | pd.DataFrame | Mapping[str, Any], +1735 data_type: TimeseriesDataType | None = None, +1736) -> list[MetadataConventionIssue]: +1737 """Report metadata errors and warnings, optionally checking against pandas data. +1738 +1739 For pandas inputs, also check metric IDs, value columns and time bounds. An empty list +1740 means no issues; warnings alone do not make metadata invalid. +1741 +1742 Args: +1743 obj: Series/DataFrame to check, or a mapping containing metadata. +1744 data_type: +1745 Convention to check; required for mappings. None infers it from pandas input; +1746 see :ref:`metadata-type-selection`. +1747 +1748 Returns: +1749 list[MetadataConventionIssue]: +1750 Issues with severity, location and message. Invalid metadata is reported as +1751 issues. +1752 +1753 Raises: +1754 ValueError: A metadata mapping is provided without data_type. +1755 +1756 Code example: +1757 +1758 Using :ref:`metadata-example-data`, validate the data and its metadata together, or pass +1759 plain metadata with an explicit data type. Issues describe the severity, location and cause: +1760 +1761 .. doctest:: metadata.check_metadata_conventions +1762 +1763 >>> md.check_metadata_conventions(series) +1764 [] +1765 >>> md.check_metadata_conventions(series.attrs, "SERIES") +1766 [] +1767 >>> invalid = series.copy() +1768 >>> invalid.attrs["metrics"][0]["default_timezone"] = "Mars/Base" +1769 >>> issues = md.check_metadata_conventions(invalid) +1770 >>> [(issue.severity.value, issue.location) for issue in issues] +1771 [('error', 'metrics[0].default_timezone')] +1772 >>> print(issues[0].message) # doctest: +ELLIPSIS +1773 "Mars/Base" is neither an IANA time zone name like "Europe/Berlin" nor a UTC offset... +1774 """ +1775 # imported here, since hdhelpers.metadata.attached imports this module +1776 from hdhelpers.metadata.attached import attrs_to_json # noqa: PLC0415 +1777 +1778 data = obj if isinstance(obj, pd.Series | pd.DataFrame) else None +1779 attrs = data.attrs if data is not None else obj +1780 if isinstance(attrs, Mapping): +1781 # attached metadata in its JSON form +1782 attrs = attrs_to_json(attrs) +1783 if data_type is None and isinstance(data, pd.Series): +1784 data_type = "SERIES" +1785 elif data_type is None and isinstance(data, pd.DataFrame): +1786 data_type = "MULTITSFRAME" if "metric" in data.columns else "SINGLETSFRAME" +1787 elif data_type is None: +1788 raise ValueError("The data type is required for checking metadata without data.") +1789 model = METADATA_MODELS[data_type] +1790 +1791 try: +1792 attrs_json = json.dumps(attrs, allow_nan=False) +1793 except (TypeError, ValueError) as e: +1794 return [ +1795 _error( +1796 "", +1797 f"Metadata is not JSON serializable, so it cannot be exchanged with adapters: {e}", +1798 ) +1799 ] +1800 +1801 try: +1802 model.model_validate_json(attrs_json, context={_SKIP_RULES: True}) +1803 except ValidationError as e: +1804 issues = [] +1805 for error in e.errors(): +1806 message = error["msg"].removeprefix("Value error, ") +1807 if (error.get("ctx") or {}).get("pattern") == UTC_TIMESTAMP_PATTERN: +1808 message = ( +1809 "Should be a UTC isoformat timestamp with explicit offset, i.e. ending with Z" +1810 f" or +00:00, got {error['input']!r}." +1811 ) +1812 elif ( +1813 error["loc"] == ("metrics",) +1814 and isinstance(attrs, Mapping) +1815 and isinstance(attrs.get("metrics"), dict) +1816 ): +1817 message += ( +1818 ". Metrics as a mapping is an older metadata convention, the current one" +1819 " expects a list of metric objects." +1820 ) +1821 issues.append(_error(_location(error["loc"]), message)) +1822 return issues + _check_unknown_fields(json.loads(attrs_json), model) +1823 +1824 json_attrs = json.loads(attrs_json) +1825 return ( +1826 _check_consistency(json_attrs) +1827 + _check_consistency_with_data_type(json_attrs, data_type, data) +1828 + _check_unknown_fields(json_attrs, model) +1829 ) diff --git a/docs/_sources/metadata_models.rst.txt b/docs/_sources/metadata_models.rst.txt index 8aece16..52677e0 100644 --- a/docs/_sources/metadata_models.rst.txt +++ b/docs/_sources/metadata_models.rst.txt @@ -154,6 +154,9 @@ where the conventions permit unknown values. .. autoclass:: InheritedMetricMetadata(**data) + The reading methods of the root models use these properties as fallbacks only with + ``include_inherited=True``. + .. autoclass:: MetricComment(**data) .. autoclass:: MetricCommentReply(**data) @@ -180,6 +183,21 @@ Allowed values and validation issues .. autoclass:: MetadataConventionIssue(**data) + .. attribute:: severity + :type: MetadataIssueSeverity + + Error for a violation; warning for a likely mistake. + + .. attribute:: location + :type: str + + Path to the affected field, e.g. ``"metrics[0].name"``; empty for the metadata as a whole. + + .. attribute:: message + :type: str + + Explanation of the problem. + Obtain validation issues with :func:`hdhelpers.metadata.check_metadata_conventions`. When required metadata cannot be read, :func:`hdhelpers.metadata.read_metadata` raises :class:`~hdhelpers.metadata.MetadataConventionError`, whose ``issues`` attribute diff --git a/docs/metadata_models.html b/docs/metadata_models.html index 3e87146..ee6265f 100644 --- a/docs/metadata_models.html +++ b/docs/metadata_models.html @@ -120,9 +120,9 @@

Root classes and methods
class hdhelpers.metadata.conventions.SeriesMetadata(**data)[source]
-

Immutable metadata for a SERIES: one metric with a single “value” dimension.

-

Read from a pandas Series with read_metadata. See the module documentation for storage, -adapter exchange and common field rules.

+

Metadata of a SERIES: one metric with a single “value” dimension.

+

Exchanged with adapters, e.g. in the Data-Attributes header of generic REST adapter endpoints +or the __metadata__ field of the direct provisioning adapter’s wrapped format.

as_singletsframe(column: str = 'value') → SingleTSFrameMetadata[source]
@@ -285,9 +285,9 @@

Root classes and methods
class hdhelpers.metadata.conventions.SingleTSFrameMetadata(**data)[source]
-

Immutable metadata for a SINGLETSFRAME: one metric with one or more value columns.

-

Read from a pandas DataFrame with read_metadata. See the module documentation for storage, -adapter exchange and common field rules.

+

Metadata of a SINGLETSFRAME: one metric with one or more value columns.

+

Exchanged with adapters, e.g. in the Data-Attributes header of generic REST adapter endpoints +or the __metadata__ field of the direct provisioning adapter’s wrapped format.

as_series(column: str = 'value') → SeriesMetadata[source]
@@ -310,10 +310,10 @@

Root classes and methods
class hdhelpers.metadata.conventions.MultiTSFrameMetadata(**data)[source]
-

Immutable metadata for a MULTITSFRAME: multiple metrics identified by a metric column.

+

Metadata of a MULTITSFRAME: multiple metrics identified by a metric column.

Each metric can have several value dimensions; a MULTITSFRAME may also hold one metric. -Read from a pandas DataFrame with read_metadata. See the module documentation for storage, -adapter exchange and common field rules.

+Exchanged with adapters, e.g. in the Data-Attributes header of generic REST adapter endpoints +or the __metadata__ field of the direct provisioning adapter’s wrapped format.

narrowed_to(metric_id: str) → SingleTSFrameMetadata[source]
@@ -396,7 +396,7 @@

Nested metadatacolumn, unit and measurement. Unit and measurement may be None. Optional fields include names, data type, deadband and display precision.

@@ -441,12 +441,14 @@

Nested metadataclass hdhelpers.metadata.conventions.InheritedMetricMetadata(**data)[source]

Properties supplied by a source system for a metric and its parent assets.

Stores already-resolved values, e.g. a plant’s latitude; the source system resolves the -hierarchy. Metadata lookups use these properties as fallbacks only with -include_inherited=True. Convention-defined fields such as name, id and unit belong in -the metric or value-dimension metadata.

+hierarchy. Consumers use these properties only as last resort and not for information +describing the metric itself, like its unit or names. Convention-defined fields such as name, +id and unit belong in the metric or value-dimension metadata.

Use JSON types matching each value’s meaning: numbers, booleans and UTC ISO timestamp strings for points in time. For example, hetida platform supplies properties by label, using a signal’s own value or that of the nearest parent asset providing one.

+

The reading methods of the root models use these properties as fallbacks only with +include_inherited=True.

@@ -487,7 +489,7 @@

Allowed values and validation issues class hdhelpers.metadata.conventions.AggregationMethod(*values)[source]

Method for aggregating timeseries values into coarser intervals.

-

Metadata lookups normalize AVG to MEAN; they are distinct enum values.

+

AVG is an alias of MEAN.

@@ -512,46 +514,31 @@

Allowed values and validation issues class hdhelpers.metadata.conventions.MetadataIssueSeverity(*values)[source]

Severity of an issue found when checking metadata against the metadata conventions

-

error: Violates the conventions, so helpers and components may misinterpret the metadata. +

error: Violates the conventions, so consumers like components may misinterpret the metadata. warning: Likely a mistake, e.g. metadata not matching the data.

class hdhelpers.metadata.conventions.MetadataConventionIssue(**data)[source]
-

A metadata validation problem, reported as an error or warning.

+

A problem found when checking metadata against the metadata conventions

+

An error for a violation of the conventions, a warning for a likely mistake.

-severity
+severity: MetadataIssueSeverity

Error for a violation; warning for a likely mistake.

-
-
Type:
-

MetadataIssueSeverity

-
-
-location
-

Path to the affected field, e.g. “metrics[0].name”; empty for the -metadata as a whole.

-
-
Type:
-

str

-
-
+location: str +

Path to the affected field, e.g. "metrics[0].name"; empty for the metadata as a whole.

-message
+message: str

Explanation of the problem.

-
-
Type:
-

str

-
-
diff --git a/docs/searchindex.js b/docs/searchindex.js index 798657e..feb611f 100644 --- a/docs/searchindex.js +++ b/docs/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles":{"Allowed values and validation issues":[[4,"allowed-values-and-validation-issues"]],"Automatic metadata type selection":[[3,"automatic-metadata-type-selection"]],"Core package":[[2,"core-package"]],"Default aggregation methods":[[3,"default-aggregation-methods"]],"Default time zones":[[3,"default-time-zones"]],"First steps":[[0,null]],"Functions":[[1,"functions"]],"Further Information":[[1,"further-information"]],"How to get metadata with hdhelpers?":[[0,"how-to-get-metadata-with-hdhelpers"]],"How to query nested information?":[[0,"how-to-query-nested-information"]],"How to use hdhelpers for plotting? (tbd)":[[0,"how-to-use-hdhelpers-for-plotting-tbd"]],"Installation":[[2,null]],"Introduction":[[1,"introduction"]],"Metadata as model":[[0,"metadata-as-model"]],"Metadata examples":[[3,null]],"Metadata lookup basics":[[3,"metadata-lookup-basics"]],"Metadata model reference":[[4,null]],"Metric fields and value-dimension fields":[[3,"metric-fields-and-value-dimension-fields"]],"Names and display-name fallbacks":[[3,"names-and-display-name-fallbacks"]],"Nested metadata":[[4,"nested-metadata"]],"Plotting helpers":[[2,"plotting-helpers"]],"Reading and updating":[[4,"reading-and-updating"]],"Reading method guide":[[4,"reading-method-guide"]],"Recording information":[[3,"recording-information"]],"Root classes and methods":[[4,"root-classes-and-methods"]],"Root models":[[4,"id1"]],"Shared data setup":[[3,"shared-data-setup"]],"Shared methods":[[4,"id2"]],"Units, measurements and return shapes":[[3,"units-measurements-and-return-shapes"]],"Verify the installation":[[2,"verify-the-installation"]],"Where does the information come from?":[[0,"where-does-the-information-come-from"]],"Which time zone to compute in?":[[0,"which-time-zone-to-compute-in"]],"exceptions":[[1,"exceptions"]],"hdhelpers":[[1,null]],"helpers":[[1,"module-hdhelpers.helpers"]],"metadata":[[1,"metadata"]],"plotting":[[1,"module-hdhelpers.plotting"]]},"docnames":["first_steps","index","installation","metadata_examples","metadata_models"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1},"filenames":["first_steps.rst","index.rst","installation.rst","metadata_examples.rst","metadata_models.rst"],"indexentries":{"aggregationmethod (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.AggregationMethod",false]],"as_multitsframe() (hdhelpers.metadata.conventions.singletsframemetadata method)":[[4,"hdhelpers.metadata.conventions.SingleTSFrameMetadata.as_multitsframe",false]],"as_series() (hdhelpers.metadata.conventions.singletsframemetadata method)":[[4,"hdhelpers.metadata.conventions.SingleTSFrameMetadata.as_series",false]],"as_singletsframe() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.as_singletsframe",false]],"attach_metadata() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.attach_metadata",false]],"attrs_to_json() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.attrs_to_json",false]],"check_metadata_conventions() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.check_metadata_conventions",false]],"copy_without_metadata() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.copy_without_metadata",false]],"data_frequency (hdhelpers.metadata.recording attribute)":[[1,"hdhelpers.metadata.Recording.data_frequency",false]],"data_frequency_offset (hdhelpers.metadata.recording attribute)":[[1,"hdhelpers.metadata.Recording.data_frequency_offset",false]],"data_frequency_timezone (hdhelpers.metadata.recording attribute)":[[1,"hdhelpers.metadata.Recording.data_frequency_timezone",false]],"default_aggregation_method() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.default_aggregation_method",false]],"default_aggregation_methods() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.default_aggregation_methods",false]],"default_timezone() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.default_timezone",false]],"default_timezones() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.default_timezones",false]],"display_name() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.display_name",false]],"display_names() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.display_names",false]],"error_code (hdhelpers.exceptions.helperexception attribute)":[[1,"hdhelpers.exceptions.HelperException.error_code",false]],"extra_information (hdhelpers.exceptions.helperexception attribute)":[[1,"hdhelpers.exceptions.HelperException.extra_information",false]],"get_default_aggregation_methods() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_default_aggregation_methods",false]],"get_default_timezones() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_default_timezones",false]],"get_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_display_names",false]],"get_locale() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.get_locale",false]],"get_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_measurements",false]],"get_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_metric_info",false]],"get_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_names",false]],"get_queried_interval() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_queried_interval",false]],"get_recordings() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_recordings",false]],"get_series_default_aggregation_method() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_default_aggregation_method",false]],"get_series_default_timezone() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_default_timezone",false]],"get_series_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_display_name",false]],"get_series_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_info",false]],"get_series_measurement() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_measurement",false]],"get_series_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_name",false]],"get_series_recording() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_recording",false]],"get_series_short_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_short_display_name",false]],"get_series_unit() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_unit",false]],"get_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_short_display_names",false]],"get_singlets_default_aggregation_method() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_default_aggregation_method",false]],"get_singlets_default_timezone() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_default_timezone",false]],"get_singlets_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_display_names",false]],"get_singlets_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_info",false]],"get_singlets_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_measurements",false]],"get_singlets_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_metric_info",false]],"get_singlets_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_names",false]],"get_singlets_recording() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_recording",false]],"get_singlets_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_short_display_names",false]],"get_singlets_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_units",false]],"get_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_units",false]],"hdhelpers.exceptions":[[1,"module-hdhelpers.exceptions",false]],"hdhelpers.helpers":[[1,"module-hdhelpers.helpers",false]],"hdhelpers.metadata":[[1,"module-hdhelpers.metadata",false]],"hdhelpers.plotting":[[1,"module-hdhelpers.plotting",false]],"helperexception":[[1,"hdhelpers.exceptions.HelperException",false]],"inheritedmetricmetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.InheritedMetricMetadata",false]],"insufficientplottingdata":[[1,"hdhelpers.exceptions.InsufficientPlottingData",false]],"intervaltype (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.IntervalType",false]],"issues (hdhelpers.metadata.metadataconventionerror attribute)":[[1,"hdhelpers.metadata.MetadataConventionError.issues",false]],"location (hdhelpers.metadata.conventions.metadataconventionissue attribute)":[[4,"hdhelpers.metadata.conventions.MetadataConventionIssue.location",false]],"measurement() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.measurement",false]],"measurements() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.measurements",false]],"message (hdhelpers.metadata.conventions.metadataconventionissue attribute)":[[4,"hdhelpers.metadata.conventions.MetadataConventionIssue.message",false]],"metadataconventionerror":[[1,"hdhelpers.metadata.MetadataConventionError",false]],"metadataconventionissue (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.MetadataConventionIssue",false]],"metadataissueseverity (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.MetadataIssueSeverity",false]],"metric() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.metric",false]],"metric_ids() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.metric_ids",false]],"metric_info() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.metric_info",false]],"metric_infos() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.metric_infos",false]],"metriccomment (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.MetricComment",false]],"metriccommentreply (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.MetricCommentReply",false]],"metricmetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.MetricMetadata",false]],"modify_timezone() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.modify_timezone",false]],"module":[[1,"module-hdhelpers.exceptions",false],[1,"module-hdhelpers.helpers",false],[1,"module-hdhelpers.metadata",false],[1,"module-hdhelpers.plotting",false]],"multitsframedatasetmetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.MultiTSFrameDatasetMetadata",false]],"multitsframemetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.MultiTSFrameMetadata",false]],"name() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.name",false]],"names() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.names",false]],"narrowed_to() (hdhelpers.metadata.conventions.multitsframemetadata method)":[[4,"hdhelpers.metadata.conventions.MultiTSFrameMetadata.narrowed_to",false]],"queried_interval() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.queried_interval",false]],"read_metadata() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.read_metadata",false]],"recording (class in hdhelpers.metadata)":[[1,"hdhelpers.metadata.Recording",false]],"recording() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.recording",false]],"recording_aggregation_method (hdhelpers.metadata.recording attribute)":[[1,"hdhelpers.metadata.Recording.recording_aggregation_method",false]],"recording_mode (hdhelpers.metadata.recording attribute)":[[1,"hdhelpers.metadata.Recording.recording_mode",false]],"recordingmode (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.RecordingMode",false]],"recordings() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.recordings",false]],"resolve_color() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.resolve_color",false]],"seriesmetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata",false]],"set_agnostic_theme() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_agnostic_theme",false]],"set_dt_ticks() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_dt_ticks",false]],"set_metadata() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.set_metadata",false]],"severity (hdhelpers.metadata.conventions.metadataconventionissue attribute)":[[4,"hdhelpers.metadata.conventions.MetadataConventionIssue.severity",false]],"sharedvaluedimensionmetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.SharedValueDimensionMetadata",false]],"short_display_name() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.short_display_name",false]],"short_display_names() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.short_display_names",false]],"signalgroupmetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.SignalGroupMetadata",false]],"singlemetricdatasetmetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.SingleMetricDatasetMetadata",false]],"singletsframemetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.SingleTSFrameMetadata",false]],"unit() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.unit",false]],"units() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.units",false]],"value_dimension_info() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.value_dimension_info",false]],"value_dimension_infos() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.value_dimension_infos",false]],"valuedatatype (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.ValueDataType",false]],"valuedimensionmetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.ValueDimensionMetadata",false]],"with_dataset_metadata() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.with_dataset_metadata",false]],"with_metric() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.with_metric",false]],"with_value_dimension() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.with_value_dimension",false]]},"objects":{"hdhelpers":[[1,0,0,"-","exceptions"],[1,0,0,"-","helpers"],[1,0,0,"-","metadata"],[1,0,0,"-","plotting"]],"hdhelpers.exceptions":[[1,1,1,"","HelperException"],[1,1,1,"","InsufficientPlottingData"]],"hdhelpers.exceptions.HelperException":[[1,2,1,"","error_code"],[1,2,1,"","extra_information"]],"hdhelpers.helpers":[[1,3,1,"","get_locale"],[1,3,1,"","modify_timezone"]],"hdhelpers.metadata":[[1,1,1,"","MetadataConventionError"],[1,4,1,"","Recording"],[1,3,1,"","attach_metadata"],[1,3,1,"","attrs_to_json"],[1,3,1,"","check_metadata_conventions"],[1,3,1,"","copy_without_metadata"],[1,3,1,"","get_default_aggregation_methods"],[1,3,1,"","get_default_timezones"],[1,3,1,"","get_display_names"],[1,3,1,"","get_measurements"],[1,3,1,"","get_metric_info"],[1,3,1,"","get_names"],[1,3,1,"","get_queried_interval"],[1,3,1,"","get_recordings"],[1,3,1,"","get_series_default_aggregation_method"],[1,3,1,"","get_series_default_timezone"],[1,3,1,"","get_series_display_name"],[1,3,1,"","get_series_info"],[1,3,1,"","get_series_measurement"],[1,3,1,"","get_series_name"],[1,3,1,"","get_series_recording"],[1,3,1,"","get_series_short_display_name"],[1,3,1,"","get_series_unit"],[1,3,1,"","get_short_display_names"],[1,3,1,"","get_singlets_default_aggregation_method"],[1,3,1,"","get_singlets_default_timezone"],[1,3,1,"","get_singlets_display_names"],[1,3,1,"","get_singlets_info"],[1,3,1,"","get_singlets_measurements"],[1,3,1,"","get_singlets_metric_info"],[1,3,1,"","get_singlets_names"],[1,3,1,"","get_singlets_recording"],[1,3,1,"","get_singlets_short_display_names"],[1,3,1,"","get_singlets_units"],[1,3,1,"","get_units"],[1,3,1,"","read_metadata"],[1,3,1,"","set_metadata"]],"hdhelpers.metadata.MetadataConventionError":[[1,2,1,"","issues"]],"hdhelpers.metadata.Recording":[[1,2,1,"","data_frequency"],[1,2,1,"","data_frequency_offset"],[1,2,1,"","data_frequency_timezone"],[1,2,1,"","recording_aggregation_method"],[1,2,1,"","recording_mode"]],"hdhelpers.metadata.conventions":[[4,4,1,"","AggregationMethod"],[4,4,1,"","InheritedMetricMetadata"],[4,4,1,"","IntervalType"],[4,4,1,"","MetadataConventionIssue"],[4,4,1,"","MetadataIssueSeverity"],[4,4,1,"","MetricComment"],[4,4,1,"","MetricCommentReply"],[4,4,1,"","MetricMetadata"],[4,4,1,"","MultiTSFrameDatasetMetadata"],[4,4,1,"","MultiTSFrameMetadata"],[4,4,1,"","RecordingMode"],[4,4,1,"","SeriesMetadata"],[4,4,1,"","SharedValueDimensionMetadata"],[4,4,1,"","SignalGroupMetadata"],[4,4,1,"","SingleMetricDatasetMetadata"],[4,4,1,"","SingleTSFrameMetadata"],[4,4,1,"","ValueDataType"],[4,4,1,"","ValueDimensionMetadata"]],"hdhelpers.metadata.conventions.MetadataConventionIssue":[[4,2,1,"","location"],[4,2,1,"","message"],[4,2,1,"","severity"]],"hdhelpers.metadata.conventions.MultiTSFrameMetadata":[[4,5,1,"","narrowed_to"]],"hdhelpers.metadata.conventions.SeriesMetadata":[[4,5,1,"","as_singletsframe"],[4,5,1,"","default_aggregation_method"],[4,5,1,"","default_aggregation_methods"],[4,5,1,"","default_timezone"],[4,5,1,"","default_timezones"],[4,5,1,"","display_name"],[4,5,1,"","display_names"],[4,5,1,"","measurement"],[4,5,1,"","measurements"],[4,5,1,"","metric"],[4,5,1,"","metric_ids"],[4,5,1,"","metric_info"],[4,5,1,"","metric_infos"],[4,5,1,"","name"],[4,5,1,"","names"],[4,5,1,"","queried_interval"],[4,5,1,"","recording"],[4,5,1,"","recordings"],[4,5,1,"","short_display_name"],[4,5,1,"","short_display_names"],[4,5,1,"","unit"],[4,5,1,"","units"],[4,5,1,"","value_dimension_info"],[4,5,1,"","value_dimension_infos"],[4,5,1,"","with_dataset_metadata"],[4,5,1,"","with_metric"],[4,5,1,"","with_value_dimension"]],"hdhelpers.metadata.conventions.SingleTSFrameMetadata":[[4,5,1,"","as_multitsframe"],[4,5,1,"","as_series"]],"hdhelpers.plotting":[[1,3,1,"","resolve_color"],[1,3,1,"","set_agnostic_theme"],[1,3,1,"","set_dt_ticks"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","attribute","Python attribute"],"3":["py","function","Python function"],"4":["py","class","Python class"],"5":["py","method","Python method"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:attribute","3":"py:function","4":"py:class","5":"py:method"},"terms":{"00z":3,"01t00":[1,3],"01t01":1,"01t08":0,"02t00":3,"15min":[1,3],"1c2d3e4f5a6b":0,"1d":0,"1ms":1,"4c1f0e2a":0,"4e5f":0,"5min":1,"7b3d":0,"9a8b":0,"A":[0,1,3,4],"All":4,"An":[0,1,3,4],"Both":[0,4],"DO":0,"Each":[0,3,4],"For":[0,1,2,3,4],"Here":4,"How":[1,4],"If":[0,1,4],"In":[0,3],"It":[0,1,4],"Its":[1,4],"NOT":0,"No":1,"Other":[0,1,3,4],"Our":0,"So":0,"That":0,"The":[0,1,2,3,4],"These":[3,4],"They":[0,1,3,4],"This":[1,2],"To":[0,2,3],"We":0,"When":[3,4],"Where":1,"Which":[1,4],"With":[0,1,2],"_":1,"__data__":0,"__hd_wrapped_data_object__":0,"__metadata__":0,"__version__":2,"abc":[1,3],"abov":[0,1],"absent":[1,3],"accept":[0,4],"access":[0,1,3,4],"accord":0,"acm":0,"across":[3,4],"actual":4,"adapt":[3,4],"add":2,"addit":[0,3,4],"address":0,"adit":1,"adjust":1,"affect":4,"aggreg":[0,1,4],"aggregationmethod":4,"alia":1,"align":1,"allow":[0,1,3],"alon":1,"alreadi":[1,3,4],"also":[1,2,3,4],"alway":1,"america":0,"among":4,"ani":[1,4],"anyway":0,"api":[1,3],"appli":[0,1,4],"arbitrari":[1,4],"arg":1,"argument":[0,3,4],"as_multitsfram":4,"as_seri":4,"as_singletsfram":4,"asset":[0,3,4],"associ":4,"attach":[0,1,3,4],"attach_metadata":[0,1,3],"attr":[0,1,3,4],"attribut":[1,4],"attrs_to_json":[0,1,3,4],"automat":[1,4],"avail":4,"avg":[1,3,4],"awar":1,"axe":1,"axi":0,"back":[0,3,4],"background":1,"bar":[3,4],"base":1,"basic":[1,4],"becom":[1,4],"befor":[0,3,4],"behavior":1,"belong":[0,1,4],"berlin":[0,1,3],"bool":[1,4],"boolean":[1,3,4],"bound":1,"boundari":4,"bucket":0,"built":4,"c":[1,2,3,4],"calcul":[1,4],"calendar":0,"calibr":0,"calibration_d":0,"call":3,"can":[0,1,4],"caus":1,"celsius":3,"centimetr":1,"chang":[0,1,3,4],"check":[1,3,4],"check_metadata_convent":[0,1,3,4],"choos":3,"class":[1,3],"close":[0,4],"cm":[0,1,4],"coarser":4,"code":[0,1],"collect":1,"color":[0,1],"color_str":1,"column":[0,1,3,4],"column_nam":1,"come":1,"command":2,"comment":[0,4],"common":4,"compar":1,"compat":1,"complet":3,"compli":1,"compon":[0,1,2,3,4],"comput":[1,3],"concern":4,"configur":1,"construct":4,"contain":[0,1,4],"context":1,"continu":2,"contract":0,"contrast":0,"convent":[0,1,3,4],"convers":[3,4],"convert":[1,3,4],"convert_index":1,"copi":[0,1,3,4],"copy_without_metadata":1,"core":1,"correspond":[0,4],"count":[0,3],"creat":[0,1,3],"current":3,"daili":[0,3],"daily_sum":0,"dark":1,"dashboard":0,"data":[0,1,2,4],"data_frequ":[1,3,4],"data_frequency_offset":1,"data_frequency_timezon":1,"data_typ":[1,3],"datafram":[0,1,3,4],"dataset":[0,1,3,4],"dataset_metadata":[0,1,3,4],"date":[0,1],"datetim":[1,4],"day":0,"daylight":0,"de":1,"de_d":1,"deadband":[1,4],"decim":[1,3],"dedic":0,"deepcopi":3,"def":0,"default":[0,1,4],"default_aggregation_method":[1,4],"default_timezon":[0,1,4],"defaultdict":1,"defin":[0,4],"degre":3,"delet":4,"depend":[0,1,4],"deriv":0,"describ":[0,1,3,4],"descript":4,"design":[0,1,2,3],"detail":1,"dict":[1,3,4],"dictionari":[1,4],"differ":[0,4],"dimens":[0,1,4],"dimensionless":1,"direct":[0,3,4],"disabl":3,"display":[0,1,4],"display_nam":[0,1,4],"distinct":[3,4],"distinguish":3,"docstr":4,"document":[3,4],"doe":[1,4],"dot":[0,1,3,4],"dump":[0,1],"e":[0,1,4],"eas":1,"eb6962":1,"edit":0,"either":1,"empti":[0,1,3,4],"enabl":[0,1,2,4],"encod":1,"end":[0,1,3,4],"energi":1,"engin":1,"entrypoint":0,"enum":4,"environ":2,"equal":4,"equival":0,"error":[1,4],"error_cod":1,"especi":1,"europ":[0,1,3],"evalu":0,"even":[0,1,3,4],"everi":[0,3,4],"exampl":[0,1,2,4],"exceed":[1,4],"exchang":[3,4],"execut":1,"exist":4,"expect":1,"explan":4,"explicit":[0,1,3,4],"explicit_local":1,"export":[1,2,3,4],"external_id":0,"extra":2,"extra_inform":1,"fail":0,"failur":1,"fall":[0,3,4],"fallback":[0,1,4],"fals":[0,1,4],"field":[0,1,4],"fig":[0,1],"figur":[0,1],"final":[0,4],"firmwar":0,"first":[1,2,3,4],"fix":0,"follow":[0,1,2,3],"form":0,"format":1,"found":4,"frame":1,"frequenc":[1,3,4],"frontend":1,"full_titl":0,"function":[0,2,3],"fuseki":1,"g":[0,1,4],"gap":[1,4],"germani":0,"get":1,"get_":1,"get_default_aggregation_method":[1,3],"get_default_timezon":[0,1,3],"get_display_nam":[1,3],"get_local":[0,1],"get_measur":[1,3],"get_metric_info":[0,1,3],"get_nam":[1,3],"get_queried_interv":[0,1,3],"get_record":[1,3],"get_series_default_aggregation_method":[1,3],"get_series_default_timezon":[0,1,3],"get_series_display_nam":[1,3],"get_series_info":[0,1,3],"get_series_measur":[1,3],"get_series_nam":[0,1,3],"get_series_record":[1,3],"get_series_short_display_nam":[1,3],"get_series_unit":[0,1,3,4],"get_short_display_nam":[1,3],"get_singlets_default_aggregation_method":[1,3],"get_singlets_default_timezon":[0,1,3],"get_singlets_display_nam":[1,3],"get_singlets_info":[0,1,3],"get_singlets_measur":[1,3],"get_singlets_metric_info":[0,1,3],"get_singlets_nam":[1,3],"get_singlets_record":[1,3],"get_singlets_short_display_nam":[1,3],"get_singlets_unit":[0,1,3],"get_unit":[0,1,3],"getter":[0,1,3],"give":[0,1,3],"given":[0,4],"go":[0,1],"graph_object":[0,1],"group":[0,3,4],"groupbi":0,"guid":1,"handl":[1,4],"handler":1,"hd_metadata":[1,3],"hdhelper":[2,3,4],"heartbeat":[1,4],"helper":[0,4],"helperexcept":1,"hetida":[0,1,2,3,4],"hierarchi":[0,4],"higher":0,"hold":[0,1,3,4],"hour":0,"hyphen":1,"iana":[0,1],"id":[0,1,4],"identifi":[1,3,4],"ignor":[1,4],"immut":[0,1,3,4],"import":[0,1,2,3],"includ":[1,2,4],"include_inherit":[0,1,3,4],"incompat":1,"independ":[1,3],"index":[0,1,3],"individu":1,"infer":[1,3,4],"inform":4,"inherit":[0,1,3,4],"inheritedmetricmetadata":4,"input":[0,1,3],"instal":1,"instead":[0,1,2],"insuffici":1,"insufficientplottingdata":1,"int":1,"interval":[0,1,4],"intervaltyp":4,"invalid":[1,3,4],"irregular":[1,4],"isinst":1,"iso":[3,4],"isoformat":[1,3],"issu":[0,1],"item":0,"json":[0,1,4],"keep":[1,3,4],"key":[0,3,4],"keyerror":[1,4],"keyword":[0,4],"ki":[0,1],"kwarg":1,"label":[0,1,4],"last":[0,1,3],"latitud":[0,4],"leav":1,"left":1,"left_clos":4,"left_open":4,"let":0,"level":[0,1,3],"lifecycl":3,"light":1,"like":[0,1,4],"limit":0,"line":0,"list":[0,1,4],"liter":1,"load":1,"local":1,"locat":[0,1,4],"long":4,"longitud":0,"look":[0,3],"lookup":[0,1,4],"lot":0,"m":[0,1,2,3,4],"main":0,"make":1,"manag":2,"mani":0,"manufactur":0,"map":[1,3,4],"mar":1,"marker":0,"match":[1,3,4],"max":[1,3],"maximum":1,"may":[0,3,4],"md":[1,3,4],"mean":[1,3,4],"meaning":1,"meant":0,"measur":[0,1,4],"messag":[1,4],"meta":[0,1,4],"metadata":2,"metadataconventionerror":[0,1,4],"metadataconventionissu":[1,4],"metadataissuesever":4,"method":[0,1],"metr":[1,3],"metric":[0,1,4],"metric_id":[1,4],"metric_info":[0,1,4],"metric_key":[0,1,4],"metriccom":4,"metriccommentrepli":4,"metricmetadata":4,"midnight":0,"might":0,"misinterpret":4,"miss":[0,1,3,4],"mistak":4,"mode":[1,4],"model":[1,3],"model_copi":4,"model_dump":4,"model_valid":4,"modelnam":4,"modifi":1,"modified_timezon":1,"modify_timezon":[0,1],"modul":[3,4],"multi":4,"multidimension":4,"multipl":4,"multitsfram":[0,1,3,4],"multitsframe_metadata_exampl":[1,3],"multitsframedatasetmetadata":4,"multitsframemetadata":[0,1,4],"must":[1,4],"naiv":1,"name":[0,1,4],"narrowed_to":4,"nearest":4,"neither":[0,1],"nest":[1,3],"never":0,"nevertheless":0,"new":[0,1,4],"new_york":0,"newer":2,"newest":4,"newli":1,"next":[0,1,4],"non":4,"none":[0,1,3,4],"nonempti":1,"normal":[1,3,4],"north":[0,1,3],"note":0,"now":1,"null":[0,4],"number":4,"obj":[0,1],"object":[0,1,3,4],"object_to_convert":1,"obtain":4,"occur":4,"offset":[0,1,3,4],"omit":[0,1,3,4],"on_chang":[1,3,4],"onc":[0,4],"one":[0,1,3,4],"onli":[0,1,3,4],"open":4,"oper":[1,4],"option":[1,2,3,4],"order":[0,1,3,4],"origin":[1,4],"otherwis":[1,3,4],"overlap":4,"overrid":[1,3],"packag":1,"palett":1,"panda":[0,1,2,3,4],"paramet":1,"parent":[3,4],"parquet":1,"pars":[0,1,3],"part":0,"pass":[0,1],"path":[0,1,3,4],"pd":[1,3],"per":[0,1,3,4],"period":[1,3,4],"permit":4,"physic":[1,3],"pip":2,"place":1,"plain":1,"plant":[0,1,3,4],"plant_a":[1,3,4],"platform":[0,1,4],"plot_target_set":1,"plural":[0,4],"point":[1,4],"possibl":4,"preced":4,"precis":4,"present":4,"preserv":1,"pressur":[1,3,4],"print":[1,2],"problem":[1,4],"produc":1,"project":2,"properti":[0,3,4],"provid":[0,1,4],"provis":0,"purpos":4,"python":[2,3],"quantiti":[1,3,4],"queri":[1,4],"queried_interv":[0,4],"queried_metr":[0,4],"rais":[0,1,4],"rang":[0,4],"raw":[1,3],"reach":0,"read":[1,3],"read_metadata":[0,1,3,4],"recommend":1,"record":[0,1,4],"recording_aggregation_method":1,"recording_mod":[1,3],"recordingmod":4,"red":1,"ref":4,"ref_interval_end_timestamp":0,"ref_interval_start_timestamp":0,"ref_interval_typ":0,"refer":[0,1,3],"relat":[0,1,3],"relev":4,"remain":4,"remov":1,"renam":4,"replac":[1,4],"repli":4,"report":[0,1,4],"repres":0,"request":[0,1,3,4],"requir":[0,1,2,4],"rerun":3,"resampl":0,"resolv":[0,1,4],"resolve_color":[0,1],"resort":0,"respect":0,"result":[0,1,3,4],"retriev":[0,4],"return":[0,1,4],"reus":1,"revalid":1,"right_clos":4,"right_open":4,"root":1,"rule":4,"run":[2,3],"runtim":[0,2],"s":[0,1,2,3,4],"sampl":[0,1,2,3],"save":0,"say":0,"scalar":1,"scale":3,"scatter":[0,1],"schema":3,"search":[0,1,4],"see":[0,1,3,4],"select":[1,4],"self":4,"sensor":[0,1],"sent":4,"separ":[0,4],"sequenc":4,"seri":[0,1,3,4],"serial":[1,3],"series_metadata_exampl":[1,3],"seriesmetadata":[0,1,4],"session":[2,3],"set":[0,1,3,4],"set_agnostic_them":[0,1],"set_dt_tick":[0,1],"set_index":0,"set_metadata":[0,1,3,4],"setup":[0,1,4],"sever":[1,3,4],"shape":1,"share":[0,1],"sharedvaluedimensionmetadata":4,"shift":1,"short":[0,1,3,4],"short_display_nam":[0,1,4],"show":[0,3],"signal":[3,4],"signalgroupmetadata":4,"sinc":[0,4],"singl":[1,3,4],"single_metr":[0,3,4],"singlemetricdatasetmetadata":4,"singletsfram":[0,1,3,4],"singletsframe_metadata_exampl":[1,3],"singletsframemetadata":[0,1,4],"site":3,"sole":4,"sort":1,"sourc":[1,3,4],"specif":[3,4],"specifi":4,"sql":4,"standard":[0,1],"start":[0,1,3,4],"state":[1,3],"stay":1,"step":[1,2,3],"still":3,"storag":4,"store":[1,3,4],"str":[1,4],"streamlin":1,"string":[1,3,4],"style":1,"sub":0,"suitabl":1,"sum":0,"suppli":[1,3,4],"support":4,"switch":0,"system":[3,4],"t":1,"take":[0,4],"taken":1,"target":[0,1],"tbd":1,"temperatur":[1,3,4],"templat":3,"test":3,"theme":1,"thin":1,"three":[1,3,4],"tickformatstop":1,"time":[1,2,4],"timedelta":1,"timeseri":[0,4],"timestamp":[0,1,3,4],"timezon":[0,1],"to_datetim":[1,3],"to_fram":1,"to_parquet":[0,1],"to_timezon":[0,1],"togeth":[1,3,4],"toggleabl":1,"total_second":1,"tri":[1,3,4],"true":[0,1,3,4],"tupl":[1,4],"turn":4,"type":[0,1,4],"typeerror":1,"tz_convert":0,"unavail":[1,3],"unchang":[0,1,3,4],"underscor":1,"uniqu":[3,4],"unit":[0,1,4],"unknown":[1,3,4],"unrel":1,"unresolv":1,"unset":1,"unspecifi":1,"unsupport":1,"updat":[1,3],"update_layout":0,"update_xax":0,"use":[1,2,3,4],"user":0,"usual":0,"utc":[0,1,3,4],"utcoffset":1,"uv":2,"valid":[0,1,3],"valu":[0,1],"value_data_typ":[1,3],"value_dim_info":1,"value_dimens":[0,1,4],"value_dimension_info":4,"value_dimensions_shar":[0,3,4],"valuedatatyp":4,"valuedimensionmetadata":4,"valueerror":1,"variabl":1,"variant":3,"verifi":1,"version":[0,4],"via":0,"view":0,"viewer":0,"violat":4,"vision":0,"want":0,"warn":[1,4],"water":[0,1,3],"well":3,"wherea":3,"whole":[0,4],"whose":[1,3,4],"win":[0,3],"with_dataset_metadata":4,"with_metr":4,"with_value_dimens":[0,1,4],"within":3,"without":[0,1,2,3],"work":[0,2],"workflow":0,"wrapper":1,"write":1,"written":1,"wrong":1,"x":[0,1],"y":[0,1],"yaxis_titl":0,"yield":[0,1],"york":0,"z":3,"zero":1,"zone":[1,2,4],"zoom":1},"titles":["First steps","hdhelpers","Installation","Metadata examples","Metadata model reference"],"titleterms":{"Further":1,"How":0,"Where":0,"Which":0,"aggreg":3,"allow":4,"automat":3,"basic":3,"class":4,"come":0,"comput":0,"core":2,"data":3,"default":3,"dimens":3,"display":3,"doe":0,"exampl":3,"except":1,"fallback":3,"field":3,"first":0,"function":1,"get":0,"guid":4,"hdhelper":[0,1],"helper":[1,2],"inform":[0,1,3],"instal":2,"introduct":1,"issu":4,"lookup":3,"measur":3,"metadata":[0,1,3,4],"method":[3,4],"metric":3,"model":[0,4],"name":3,"nest":[0,4],"packag":2,"plot":[0,1,2],"queri":0,"read":4,"record":3,"refer":4,"return":3,"root":4,"select":3,"setup":3,"shape":3,"share":[3,4],"step":0,"tbd":0,"time":[0,3],"type":3,"unit":3,"updat":4,"use":0,"valid":4,"valu":[3,4],"verifi":2,"zone":[0,3]}}) \ No newline at end of file +Search.setIndex({"alltitles":{"Allowed values and validation issues":[[4,"allowed-values-and-validation-issues"]],"Automatic metadata type selection":[[3,"automatic-metadata-type-selection"]],"Core package":[[2,"core-package"]],"Default aggregation methods":[[3,"default-aggregation-methods"]],"Default time zones":[[3,"default-time-zones"]],"First steps":[[0,null]],"Functions":[[1,"functions"]],"Further Information":[[1,"further-information"]],"How to get metadata with hdhelpers?":[[0,"how-to-get-metadata-with-hdhelpers"]],"How to query nested information?":[[0,"how-to-query-nested-information"]],"How to use hdhelpers for plotting? (tbd)":[[0,"how-to-use-hdhelpers-for-plotting-tbd"]],"Installation":[[2,null]],"Introduction":[[1,"introduction"]],"Metadata as model":[[0,"metadata-as-model"]],"Metadata examples":[[3,null]],"Metadata lookup basics":[[3,"metadata-lookup-basics"]],"Metadata model reference":[[4,null]],"Metric fields and value-dimension fields":[[3,"metric-fields-and-value-dimension-fields"]],"Names and display-name fallbacks":[[3,"names-and-display-name-fallbacks"]],"Nested metadata":[[4,"nested-metadata"]],"Plotting helpers":[[2,"plotting-helpers"]],"Reading and updating":[[4,"reading-and-updating"]],"Reading method guide":[[4,"reading-method-guide"]],"Recording information":[[3,"recording-information"]],"Root classes and methods":[[4,"root-classes-and-methods"]],"Root models":[[4,"id1"]],"Shared data setup":[[3,"shared-data-setup"]],"Shared methods":[[4,"id2"]],"Units, measurements and return shapes":[[3,"units-measurements-and-return-shapes"]],"Verify the installation":[[2,"verify-the-installation"]],"Where does the information come from?":[[0,"where-does-the-information-come-from"]],"Which time zone to compute in?":[[0,"which-time-zone-to-compute-in"]],"exceptions":[[1,"exceptions"]],"hdhelpers":[[1,null]],"helpers":[[1,"module-hdhelpers.helpers"]],"metadata":[[1,"metadata"]],"plotting":[[1,"module-hdhelpers.plotting"]]},"docnames":["first_steps","index","installation","metadata_examples","metadata_models"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1},"filenames":["first_steps.rst","index.rst","installation.rst","metadata_examples.rst","metadata_models.rst"],"indexentries":{"aggregationmethod (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.AggregationMethod",false]],"as_multitsframe() (hdhelpers.metadata.conventions.singletsframemetadata method)":[[4,"hdhelpers.metadata.conventions.SingleTSFrameMetadata.as_multitsframe",false]],"as_series() (hdhelpers.metadata.conventions.singletsframemetadata method)":[[4,"hdhelpers.metadata.conventions.SingleTSFrameMetadata.as_series",false]],"as_singletsframe() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.as_singletsframe",false]],"attach_metadata() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.attach_metadata",false]],"attrs_to_json() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.attrs_to_json",false]],"check_metadata_conventions() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.check_metadata_conventions",false]],"copy_without_metadata() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.copy_without_metadata",false]],"data_frequency (hdhelpers.metadata.recording attribute)":[[1,"hdhelpers.metadata.Recording.data_frequency",false]],"data_frequency_offset (hdhelpers.metadata.recording attribute)":[[1,"hdhelpers.metadata.Recording.data_frequency_offset",false]],"data_frequency_timezone (hdhelpers.metadata.recording attribute)":[[1,"hdhelpers.metadata.Recording.data_frequency_timezone",false]],"default_aggregation_method() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.default_aggregation_method",false]],"default_aggregation_methods() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.default_aggregation_methods",false]],"default_timezone() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.default_timezone",false]],"default_timezones() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.default_timezones",false]],"display_name() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.display_name",false]],"display_names() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.display_names",false]],"error_code (hdhelpers.exceptions.helperexception attribute)":[[1,"hdhelpers.exceptions.HelperException.error_code",false]],"extra_information (hdhelpers.exceptions.helperexception attribute)":[[1,"hdhelpers.exceptions.HelperException.extra_information",false]],"get_default_aggregation_methods() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_default_aggregation_methods",false]],"get_default_timezones() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_default_timezones",false]],"get_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_display_names",false]],"get_locale() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.get_locale",false]],"get_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_measurements",false]],"get_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_metric_info",false]],"get_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_names",false]],"get_queried_interval() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_queried_interval",false]],"get_recordings() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_recordings",false]],"get_series_default_aggregation_method() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_default_aggregation_method",false]],"get_series_default_timezone() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_default_timezone",false]],"get_series_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_display_name",false]],"get_series_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_info",false]],"get_series_measurement() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_measurement",false]],"get_series_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_name",false]],"get_series_recording() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_recording",false]],"get_series_short_display_name() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_short_display_name",false]],"get_series_unit() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_series_unit",false]],"get_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_short_display_names",false]],"get_singlets_default_aggregation_method() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_default_aggregation_method",false]],"get_singlets_default_timezone() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_default_timezone",false]],"get_singlets_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_display_names",false]],"get_singlets_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_info",false]],"get_singlets_measurements() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_measurements",false]],"get_singlets_metric_info() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_metric_info",false]],"get_singlets_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_names",false]],"get_singlets_recording() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_recording",false]],"get_singlets_short_display_names() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_short_display_names",false]],"get_singlets_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_singlets_units",false]],"get_units() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.get_units",false]],"hdhelpers.exceptions":[[1,"module-hdhelpers.exceptions",false]],"hdhelpers.helpers":[[1,"module-hdhelpers.helpers",false]],"hdhelpers.metadata":[[1,"module-hdhelpers.metadata",false]],"hdhelpers.plotting":[[1,"module-hdhelpers.plotting",false]],"helperexception":[[1,"hdhelpers.exceptions.HelperException",false]],"inheritedmetricmetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.InheritedMetricMetadata",false]],"insufficientplottingdata":[[1,"hdhelpers.exceptions.InsufficientPlottingData",false]],"intervaltype (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.IntervalType",false]],"issues (hdhelpers.metadata.metadataconventionerror attribute)":[[1,"hdhelpers.metadata.MetadataConventionError.issues",false]],"location (hdhelpers.metadata.conventions.metadataconventionissue attribute)":[[4,"hdhelpers.metadata.conventions.MetadataConventionIssue.location",false]],"measurement() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.measurement",false]],"measurements() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.measurements",false]],"message (hdhelpers.metadata.conventions.metadataconventionissue attribute)":[[4,"hdhelpers.metadata.conventions.MetadataConventionIssue.message",false]],"metadataconventionerror":[[1,"hdhelpers.metadata.MetadataConventionError",false]],"metadataconventionissue (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.MetadataConventionIssue",false]],"metadataissueseverity (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.MetadataIssueSeverity",false]],"metric() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.metric",false]],"metric_ids() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.metric_ids",false]],"metric_info() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.metric_info",false]],"metric_infos() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.metric_infos",false]],"metriccomment (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.MetricComment",false]],"metriccommentreply (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.MetricCommentReply",false]],"metricmetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.MetricMetadata",false]],"modify_timezone() (in module hdhelpers.helpers)":[[1,"hdhelpers.helpers.modify_timezone",false]],"module":[[1,"module-hdhelpers.exceptions",false],[1,"module-hdhelpers.helpers",false],[1,"module-hdhelpers.metadata",false],[1,"module-hdhelpers.plotting",false]],"multitsframedatasetmetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.MultiTSFrameDatasetMetadata",false]],"multitsframemetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.MultiTSFrameMetadata",false]],"name() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.name",false]],"names() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.names",false]],"narrowed_to() (hdhelpers.metadata.conventions.multitsframemetadata method)":[[4,"hdhelpers.metadata.conventions.MultiTSFrameMetadata.narrowed_to",false]],"queried_interval() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.queried_interval",false]],"read_metadata() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.read_metadata",false]],"recording (class in hdhelpers.metadata)":[[1,"hdhelpers.metadata.Recording",false]],"recording() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.recording",false]],"recording_aggregation_method (hdhelpers.metadata.recording attribute)":[[1,"hdhelpers.metadata.Recording.recording_aggregation_method",false]],"recording_mode (hdhelpers.metadata.recording attribute)":[[1,"hdhelpers.metadata.Recording.recording_mode",false]],"recordingmode (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.RecordingMode",false]],"recordings() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.recordings",false]],"resolve_color() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.resolve_color",false]],"seriesmetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata",false]],"set_agnostic_theme() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_agnostic_theme",false]],"set_dt_ticks() (in module hdhelpers.plotting)":[[1,"hdhelpers.plotting.set_dt_ticks",false]],"set_metadata() (in module hdhelpers.metadata)":[[1,"hdhelpers.metadata.set_metadata",false]],"severity (hdhelpers.metadata.conventions.metadataconventionissue attribute)":[[4,"hdhelpers.metadata.conventions.MetadataConventionIssue.severity",false]],"sharedvaluedimensionmetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.SharedValueDimensionMetadata",false]],"short_display_name() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.short_display_name",false]],"short_display_names() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.short_display_names",false]],"signalgroupmetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.SignalGroupMetadata",false]],"singlemetricdatasetmetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.SingleMetricDatasetMetadata",false]],"singletsframemetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.SingleTSFrameMetadata",false]],"unit() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.unit",false]],"units() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.units",false]],"value_dimension_info() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.value_dimension_info",false]],"value_dimension_infos() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.value_dimension_infos",false]],"valuedatatype (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.ValueDataType",false]],"valuedimensionmetadata (class in hdhelpers.metadata.conventions)":[[4,"hdhelpers.metadata.conventions.ValueDimensionMetadata",false]],"with_dataset_metadata() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.with_dataset_metadata",false]],"with_metric() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.with_metric",false]],"with_value_dimension() (hdhelpers.metadata.conventions.seriesmetadata method)":[[4,"hdhelpers.metadata.conventions.SeriesMetadata.with_value_dimension",false]]},"objects":{"hdhelpers":[[1,0,0,"-","exceptions"],[1,0,0,"-","helpers"],[1,0,0,"-","metadata"],[1,0,0,"-","plotting"]],"hdhelpers.exceptions":[[1,1,1,"","HelperException"],[1,1,1,"","InsufficientPlottingData"]],"hdhelpers.exceptions.HelperException":[[1,2,1,"","error_code"],[1,2,1,"","extra_information"]],"hdhelpers.helpers":[[1,3,1,"","get_locale"],[1,3,1,"","modify_timezone"]],"hdhelpers.metadata":[[1,1,1,"","MetadataConventionError"],[1,4,1,"","Recording"],[1,3,1,"","attach_metadata"],[1,3,1,"","attrs_to_json"],[1,3,1,"","check_metadata_conventions"],[1,3,1,"","copy_without_metadata"],[1,3,1,"","get_default_aggregation_methods"],[1,3,1,"","get_default_timezones"],[1,3,1,"","get_display_names"],[1,3,1,"","get_measurements"],[1,3,1,"","get_metric_info"],[1,3,1,"","get_names"],[1,3,1,"","get_queried_interval"],[1,3,1,"","get_recordings"],[1,3,1,"","get_series_default_aggregation_method"],[1,3,1,"","get_series_default_timezone"],[1,3,1,"","get_series_display_name"],[1,3,1,"","get_series_info"],[1,3,1,"","get_series_measurement"],[1,3,1,"","get_series_name"],[1,3,1,"","get_series_recording"],[1,3,1,"","get_series_short_display_name"],[1,3,1,"","get_series_unit"],[1,3,1,"","get_short_display_names"],[1,3,1,"","get_singlets_default_aggregation_method"],[1,3,1,"","get_singlets_default_timezone"],[1,3,1,"","get_singlets_display_names"],[1,3,1,"","get_singlets_info"],[1,3,1,"","get_singlets_measurements"],[1,3,1,"","get_singlets_metric_info"],[1,3,1,"","get_singlets_names"],[1,3,1,"","get_singlets_recording"],[1,3,1,"","get_singlets_short_display_names"],[1,3,1,"","get_singlets_units"],[1,3,1,"","get_units"],[1,3,1,"","read_metadata"],[1,3,1,"","set_metadata"]],"hdhelpers.metadata.MetadataConventionError":[[1,2,1,"","issues"]],"hdhelpers.metadata.Recording":[[1,2,1,"","data_frequency"],[1,2,1,"","data_frequency_offset"],[1,2,1,"","data_frequency_timezone"],[1,2,1,"","recording_aggregation_method"],[1,2,1,"","recording_mode"]],"hdhelpers.metadata.conventions":[[4,4,1,"","AggregationMethod"],[4,4,1,"","InheritedMetricMetadata"],[4,4,1,"","IntervalType"],[4,4,1,"","MetadataConventionIssue"],[4,4,1,"","MetadataIssueSeverity"],[4,4,1,"","MetricComment"],[4,4,1,"","MetricCommentReply"],[4,4,1,"","MetricMetadata"],[4,4,1,"","MultiTSFrameDatasetMetadata"],[4,4,1,"","MultiTSFrameMetadata"],[4,4,1,"","RecordingMode"],[4,4,1,"","SeriesMetadata"],[4,4,1,"","SharedValueDimensionMetadata"],[4,4,1,"","SignalGroupMetadata"],[4,4,1,"","SingleMetricDatasetMetadata"],[4,4,1,"","SingleTSFrameMetadata"],[4,4,1,"","ValueDataType"],[4,4,1,"","ValueDimensionMetadata"]],"hdhelpers.metadata.conventions.MetadataConventionIssue":[[4,2,1,"","location"],[4,2,1,"","message"],[4,2,1,"","severity"]],"hdhelpers.metadata.conventions.MultiTSFrameMetadata":[[4,5,1,"","narrowed_to"]],"hdhelpers.metadata.conventions.SeriesMetadata":[[4,5,1,"","as_singletsframe"],[4,5,1,"","default_aggregation_method"],[4,5,1,"","default_aggregation_methods"],[4,5,1,"","default_timezone"],[4,5,1,"","default_timezones"],[4,5,1,"","display_name"],[4,5,1,"","display_names"],[4,5,1,"","measurement"],[4,5,1,"","measurements"],[4,5,1,"","metric"],[4,5,1,"","metric_ids"],[4,5,1,"","metric_info"],[4,5,1,"","metric_infos"],[4,5,1,"","name"],[4,5,1,"","names"],[4,5,1,"","queried_interval"],[4,5,1,"","recording"],[4,5,1,"","recordings"],[4,5,1,"","short_display_name"],[4,5,1,"","short_display_names"],[4,5,1,"","unit"],[4,5,1,"","units"],[4,5,1,"","value_dimension_info"],[4,5,1,"","value_dimension_infos"],[4,5,1,"","with_dataset_metadata"],[4,5,1,"","with_metric"],[4,5,1,"","with_value_dimension"]],"hdhelpers.metadata.conventions.SingleTSFrameMetadata":[[4,5,1,"","as_multitsframe"],[4,5,1,"","as_series"]],"hdhelpers.plotting":[[1,3,1,"","resolve_color"],[1,3,1,"","set_agnostic_theme"],[1,3,1,"","set_dt_ticks"]]},"objnames":{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","attribute","Python attribute"],"3":["py","function","Python function"],"4":["py","class","Python class"],"5":["py","method","Python method"]},"objtypes":{"0":"py:module","1":"py:exception","2":"py:attribute","3":"py:function","4":"py:class","5":"py:method"},"terms":{"00z":3,"01t00":[1,3],"01t01":1,"01t08":0,"02t00":3,"15min":[1,3],"1c2d3e4f5a6b":0,"1d":0,"1ms":1,"4c1f0e2a":0,"4e5f":0,"5min":1,"7b3d":0,"9a8b":0,"A":[0,1,3,4],"All":4,"An":[0,1,3,4],"Both":[0,4],"DO":0,"Each":[0,3,4],"For":[0,1,2,3,4],"Here":4,"How":[1,4],"If":[0,1,4],"In":[0,3],"It":[0,1,4],"Its":[1,4],"NOT":0,"No":1,"Other":[0,1,3,4],"Our":0,"So":0,"That":0,"The":[0,1,2,3,4],"These":[3,4],"They":[0,1,3,4],"This":[1,2],"To":[0,2,3],"We":0,"When":[3,4],"Where":1,"Which":[1,4],"With":[0,1,2],"_":1,"__data__":0,"__hd_wrapped_data_object__":0,"__metadata__":[0,4],"__version__":2,"abc":[1,3],"abov":[0,1],"absent":[1,3],"accept":[0,4],"access":[0,1,3,4],"accord":0,"acm":0,"across":[3,4],"actual":4,"adapt":[3,4],"add":2,"addit":[0,3,4],"address":0,"adit":1,"adjust":1,"affect":4,"aggreg":[0,1,4],"aggregationmethod":4,"alia":[1,4],"align":1,"allow":[0,1,3],"alon":1,"alreadi":[1,3,4],"also":[1,2,3,4],"alway":1,"america":0,"among":4,"ani":[1,4],"anyway":0,"api":[1,3],"appli":[0,1,4],"arbitrari":[1,4],"arg":1,"argument":[0,3,4],"as_multitsfram":4,"as_seri":4,"as_singletsfram":4,"asset":[0,3,4],"associ":4,"attach":[0,1,3,4],"attach_metadata":[0,1,3],"attr":[0,1,3,4],"attribut":[1,4],"attrs_to_json":[0,1,3,4],"automat":[1,4],"avail":4,"avg":[1,3,4],"awar":1,"axe":1,"axi":0,"back":[0,3,4],"background":1,"bar":[3,4],"base":1,"basic":[1,4],"becom":[1,4],"befor":[0,3,4],"behavior":1,"belong":[0,1,4],"berlin":[0,1,3],"bool":[1,4],"boolean":[1,3,4],"bound":1,"boundari":4,"bucket":0,"built":4,"c":[1,2,3,4],"calcul":[1,4],"calendar":0,"calibr":0,"calibration_d":0,"call":3,"can":[0,1,4],"caus":1,"celsius":3,"centimetr":1,"chang":[0,1,3,4],"check":[1,3,4],"check_metadata_convent":[0,1,3,4],"choos":3,"class":[1,3],"close":[0,4],"cm":[0,1,4],"coarser":4,"code":[0,1],"collect":1,"color":[0,1],"color_str":1,"column":[0,1,3,4],"column_nam":1,"come":1,"command":2,"comment":[0,4],"compar":1,"compat":1,"complet":3,"compli":1,"compon":[0,1,2,3,4],"comput":[1,3],"concern":4,"configur":1,"construct":4,"consum":4,"contain":[0,1,4],"context":1,"continu":2,"contract":0,"contrast":0,"convent":[0,1,3,4],"convers":[3,4],"convert":[1,3,4],"convert_index":1,"copi":[0,1,3,4],"copy_without_metadata":1,"core":1,"correspond":[0,4],"count":[0,3],"creat":[0,1,3],"current":3,"daili":[0,3],"daily_sum":0,"dark":1,"dashboard":0,"data":[0,1,2,4],"data_frequ":[1,3,4],"data_frequency_offset":1,"data_frequency_timezon":1,"data_typ":[1,3],"datafram":[0,1,3,4],"dataset":[0,1,3,4],"dataset_metadata":[0,1,3,4],"date":[0,1],"datetim":[1,4],"day":0,"daylight":0,"de":1,"de_d":1,"deadband":[1,4],"decim":[1,3],"dedic":0,"deepcopi":3,"def":0,"default":[0,1,4],"default_aggregation_method":[1,4],"default_timezon":[0,1,4],"defaultdict":1,"defin":[0,4],"degre":3,"delet":4,"depend":[0,1,4],"deriv":0,"describ":[0,1,3,4],"descript":4,"design":[0,1,2,3],"detail":1,"dict":[1,3,4],"dictionari":[1,4],"differ":[0,4],"dimens":[0,1,4],"dimensionless":1,"direct":[0,3,4],"disabl":3,"display":[0,1,4],"display_nam":[0,1,4],"distinct":3,"distinguish":3,"docstr":4,"document":[3,4],"doe":[1,4],"dot":[0,1,3,4],"dump":[0,1],"e":[0,1,4],"eas":1,"eb6962":1,"edit":0,"either":1,"empti":[0,1,3,4],"enabl":[0,1,2,4],"encod":1,"end":[0,1,3,4],"endpoint":4,"energi":1,"engin":1,"entrypoint":0,"environ":2,"equal":4,"equival":0,"error":[1,4],"error_cod":1,"especi":1,"europ":[0,1,3],"evalu":0,"even":[0,1,3,4],"everi":[0,3,4],"exampl":[0,1,2,4],"exceed":[1,4],"exchang":[3,4],"execut":1,"exist":4,"expect":1,"explan":4,"explicit":[0,1,3,4],"explicit_local":1,"export":[1,2,3,4],"external_id":0,"extra":2,"extra_inform":1,"fail":0,"failur":1,"fall":[0,3,4],"fallback":[0,1,4],"fals":[0,1,4],"field":[0,1,4],"fig":[0,1],"figur":[0,1],"final":[0,4],"firmwar":0,"first":[1,2,3,4],"fix":0,"follow":[0,1,2,3],"form":0,"format":[1,4],"found":4,"frame":1,"frequenc":[1,3,4],"frontend":1,"full_titl":0,"function":[0,2,3],"fuseki":1,"g":[0,1,4],"gap":[1,4],"generic":4,"germani":0,"get":1,"get_":1,"get_default_aggregation_method":[1,3],"get_default_timezon":[0,1,3],"get_display_nam":[1,3],"get_local":[0,1],"get_measur":[1,3],"get_metric_info":[0,1,3],"get_nam":[1,3],"get_queried_interv":[0,1,3],"get_record":[1,3],"get_series_default_aggregation_method":[1,3],"get_series_default_timezon":[0,1,3],"get_series_display_nam":[1,3],"get_series_info":[0,1,3],"get_series_measur":[1,3],"get_series_nam":[0,1,3],"get_series_record":[1,3],"get_series_short_display_nam":[1,3],"get_series_unit":[0,1,3,4],"get_short_display_nam":[1,3],"get_singlets_default_aggregation_method":[1,3],"get_singlets_default_timezon":[0,1,3],"get_singlets_display_nam":[1,3],"get_singlets_info":[0,1,3],"get_singlets_measur":[1,3],"get_singlets_metric_info":[0,1,3],"get_singlets_nam":[1,3],"get_singlets_record":[1,3],"get_singlets_short_display_nam":[1,3],"get_singlets_unit":[0,1,3],"get_unit":[0,1,3],"getter":[0,1,3],"give":[0,1,3],"given":[0,4],"go":[0,1],"graph_object":[0,1],"group":[0,3,4],"groupbi":0,"guid":1,"handl":[1,4],"handler":1,"hd_metadata":[1,3],"hdhelper":[2,3,4],"header":4,"heartbeat":[1,4],"helper":0,"helperexcept":1,"hetida":[0,1,2,3,4],"hierarchi":[0,4],"higher":0,"hold":[0,1,3,4],"hour":0,"hyphen":1,"iana":[0,1],"id":[0,1,4],"identifi":[1,3,4],"ignor":[1,4],"immut":[0,1,3,4],"import":[0,1,2,3],"includ":[1,2,4],"include_inherit":[0,1,3,4],"incompat":1,"independ":[1,3],"index":[0,1,3],"individu":1,"infer":[1,3,4],"inform":4,"inherit":[0,1,3,4],"inheritedmetricmetadata":4,"input":[0,1,3],"instal":1,"instead":[0,1,2],"insuffici":1,"insufficientplottingdata":1,"int":1,"interval":[0,1,4],"intervaltyp":4,"invalid":[1,3,4],"irregular":[1,4],"isinst":1,"iso":[3,4],"isoformat":[1,3],"issu":[0,1],"item":0,"json":[0,1,4],"keep":[1,3,4],"key":[0,3,4],"keyerror":[1,4],"keyword":[0,4],"ki":[0,1],"kwarg":1,"label":[0,1,4],"last":[0,1,3,4],"latitud":[0,4],"leav":1,"left":1,"left_clos":4,"left_open":4,"let":0,"level":[0,1,3],"lifecycl":3,"light":1,"like":[0,1,4],"limit":0,"line":0,"list":[0,1,4],"liter":1,"load":1,"local":1,"locat":[0,1,4],"long":4,"longitud":0,"look":[0,3],"lookup":[0,1,4],"lot":0,"m":[0,1,2,3,4],"main":0,"make":1,"manag":2,"mani":0,"manufactur":0,"map":[1,3,4],"mar":1,"marker":0,"match":[1,3,4],"max":[1,3],"maximum":1,"may":[0,3,4],"md":[1,3,4],"mean":[1,3,4],"meaning":1,"meant":0,"measur":[0,1,4],"messag":[1,4],"meta":[0,1,4],"metadata":2,"metadataconventionerror":[0,1,4],"metadataconventionissu":[1,4],"metadataissuesever":4,"method":[0,1],"metr":[1,3],"metric":[0,1,4],"metric_id":[1,4],"metric_info":[0,1,4],"metric_key":[0,1,4],"metriccom":4,"metriccommentrepli":4,"metricmetadata":4,"midnight":0,"might":0,"misinterpret":4,"miss":[0,1,3,4],"mistak":4,"mode":[1,4],"model":[1,3],"model_copi":4,"model_dump":4,"model_valid":4,"modelnam":4,"modifi":1,"modified_timezon":1,"modify_timezon":[0,1],"modul":[3,4],"multi":4,"multidimension":4,"multipl":4,"multitsfram":[0,1,3,4],"multitsframe_metadata_exampl":[1,3],"multitsframedatasetmetadata":4,"multitsframemetadata":[0,1,4],"must":[1,4],"naiv":1,"name":[0,1,4],"narrowed_to":4,"nearest":4,"neither":[0,1],"nest":[1,3],"never":0,"nevertheless":0,"new":[0,1,4],"new_york":0,"newer":2,"newest":4,"newli":1,"next":[0,1,4],"non":4,"none":[0,1,3,4],"nonempti":1,"normal":[1,3,4],"north":[0,1,3],"note":0,"now":1,"null":[0,4],"number":4,"obj":[0,1],"object":[0,1,3,4],"object_to_convert":1,"obtain":4,"occur":4,"offset":[0,1,3,4],"omit":[0,1,3,4],"on_chang":[1,3,4],"onc":[0,4],"one":[0,1,3,4],"onli":[0,1,3,4],"open":4,"oper":[1,4],"option":[1,2,3,4],"order":[0,1,3,4],"origin":[1,4],"otherwis":[1,3,4],"overlap":4,"overrid":[1,3],"packag":1,"palett":1,"panda":[0,1,2,3,4],"paramet":1,"parent":[3,4],"parquet":1,"pars":[0,1,3],"part":0,"pass":[0,1],"path":[0,1,3,4],"pd":[1,3],"per":[0,1,3,4],"period":[1,3,4],"permit":4,"physic":[1,3],"pip":2,"place":1,"plain":1,"plant":[0,1,3,4],"plant_a":[1,3,4],"platform":[0,1,4],"plot_target_set":1,"plural":[0,4],"point":[1,4],"possibl":4,"preced":4,"precis":4,"present":4,"preserv":1,"pressur":[1,3,4],"print":[1,2],"problem":[1,4],"produc":1,"project":2,"properti":[0,3,4],"provid":[0,1,4],"provis":[0,4],"purpos":4,"python":[2,3],"quantiti":[1,3,4],"queri":[1,4],"queried_interv":[0,4],"queried_metr":[0,4],"rais":[0,1,4],"rang":[0,4],"raw":[1,3],"reach":0,"read":[1,3],"read_metadata":[0,1,3,4],"recommend":1,"record":[0,1,4],"recording_aggregation_method":1,"recording_mod":[1,3],"recordingmod":4,"red":1,"ref":4,"ref_interval_end_timestamp":0,"ref_interval_start_timestamp":0,"ref_interval_typ":0,"refer":[0,1,3],"relat":[0,1,3],"relev":4,"remain":4,"remov":1,"renam":4,"replac":[1,4],"repli":4,"report":[0,1],"repres":0,"request":[0,1,3,4],"requir":[0,1,2,4],"rerun":3,"resampl":0,"resolv":[0,1,4],"resolve_color":[0,1],"resort":[0,4],"respect":0,"rest":4,"result":[0,1,3,4],"retriev":[0,4],"return":[0,1,4],"reus":1,"revalid":1,"right_clos":4,"right_open":4,"root":1,"rule":4,"run":[2,3],"runtim":[0,2],"s":[0,1,2,3,4],"sampl":[0,1,2,3],"save":0,"say":0,"scalar":1,"scale":3,"scatter":[0,1],"schema":3,"search":[0,1,4],"see":[0,1,3,4],"select":[1,4],"self":4,"sensor":[0,1],"sent":4,"separ":[0,4],"sequenc":4,"seri":[0,1,3,4],"serial":[1,3],"series_metadata_exampl":[1,3],"seriesmetadata":[0,1,4],"session":[2,3],"set":[0,1,3,4],"set_agnostic_them":[0,1],"set_dt_tick":[0,1],"set_index":0,"set_metadata":[0,1,3,4],"setup":[0,1,4],"sever":[1,3,4],"shape":1,"share":[0,1],"sharedvaluedimensionmetadata":4,"shift":1,"short":[0,1,3,4],"short_display_nam":[0,1,4],"show":[0,3],"signal":[3,4],"signalgroupmetadata":4,"sinc":[0,4],"singl":[1,3,4],"single_metr":[0,3,4],"singlemetricdatasetmetadata":4,"singletsfram":[0,1,3,4],"singletsframe_metadata_exampl":[1,3],"singletsframemetadata":[0,1,4],"site":3,"sole":4,"sort":1,"sourc":[1,3,4],"specif":[3,4],"specifi":4,"sql":4,"standard":[0,1],"start":[0,1,3,4],"state":[1,3],"stay":1,"step":[1,2,3],"still":3,"storag":4,"store":[1,3,4],"str":[1,4],"streamlin":1,"string":[1,3,4],"style":1,"sub":0,"suitabl":1,"sum":0,"suppli":[1,3,4],"support":4,"switch":0,"system":[3,4],"t":1,"take":[0,4],"taken":1,"target":[0,1],"tbd":1,"temperatur":[1,3,4],"templat":3,"test":3,"theme":1,"thin":1,"three":[1,3,4],"tickformatstop":1,"time":[1,2,4],"timedelta":1,"timeseri":[0,4],"timestamp":[0,1,3,4],"timezon":[0,1],"to_datetim":[1,3],"to_fram":1,"to_parquet":[0,1],"to_timezon":[0,1],"togeth":[1,3,4],"toggleabl":1,"total_second":1,"tri":[1,3,4],"true":[0,1,3,4],"tupl":[1,4],"turn":4,"type":[0,1,4],"typeerror":1,"tz_convert":0,"unavail":[1,3],"unchang":[0,1,3,4],"underscor":1,"uniqu":[3,4],"unit":[0,1,4],"unknown":[1,3,4],"unrel":1,"unresolv":1,"unset":1,"unspecifi":1,"unsupport":1,"updat":[1,3],"update_layout":0,"update_xax":0,"use":[1,2,3,4],"user":0,"usual":0,"utc":[0,1,3,4],"utcoffset":1,"uv":2,"valid":[0,1,3],"valu":[0,1],"value_data_typ":[1,3],"value_dim_info":1,"value_dimens":[0,1,4],"value_dimension_info":4,"value_dimensions_shar":[0,3,4],"valuedatatyp":4,"valuedimensionmetadata":4,"valueerror":1,"variabl":1,"variant":3,"verifi":1,"version":[0,4],"via":0,"view":0,"viewer":0,"violat":4,"vision":0,"want":0,"warn":[1,4],"water":[0,1,3],"well":3,"wherea":3,"whole":[0,4],"whose":[1,3,4],"win":[0,3],"with_dataset_metadata":4,"with_metr":4,"with_value_dimens":[0,1,4],"within":3,"without":[0,1,2,3],"work":[0,2],"workflow":0,"wrap":4,"wrapper":1,"write":1,"written":1,"wrong":1,"x":[0,1],"y":[0,1],"yaxis_titl":0,"yield":[0,1],"york":0,"z":3,"zero":1,"zone":[1,2,4],"zoom":1},"titles":["First steps","hdhelpers","Installation","Metadata examples","Metadata model reference"],"titleterms":{"Further":1,"How":0,"Where":0,"Which":0,"aggreg":3,"allow":4,"automat":3,"basic":3,"class":4,"come":0,"comput":0,"core":2,"data":3,"default":3,"dimens":3,"display":3,"doe":0,"exampl":3,"except":1,"fallback":3,"field":3,"first":0,"function":1,"get":0,"guid":4,"hdhelper":[0,1],"helper":[1,2],"inform":[0,1,3],"instal":2,"introduct":1,"issu":4,"lookup":3,"measur":3,"metadata":[0,1,3,4],"method":[3,4],"metric":3,"model":[0,4],"name":3,"nest":[0,4],"packag":2,"plot":[0,1,2],"queri":0,"read":4,"record":3,"refer":4,"return":3,"root":4,"select":3,"setup":3,"shape":3,"share":[3,4],"step":0,"tbd":0,"time":[0,3],"type":3,"unit":3,"updat":4,"use":0,"valid":4,"valu":[3,4],"verifi":2,"zone":[0,3]}}) \ No newline at end of file diff --git a/sphinx/source/metadata_models.rst b/sphinx/source/metadata_models.rst index 8aece16..52677e0 100644 --- a/sphinx/source/metadata_models.rst +++ b/sphinx/source/metadata_models.rst @@ -154,6 +154,9 @@ where the conventions permit unknown values. .. autoclass:: InheritedMetricMetadata(**data) + The reading methods of the root models use these properties as fallbacks only with + ``include_inherited=True``. + .. autoclass:: MetricComment(**data) .. autoclass:: MetricCommentReply(**data) @@ -180,6 +183,21 @@ Allowed values and validation issues .. autoclass:: MetadataConventionIssue(**data) + .. attribute:: severity + :type: MetadataIssueSeverity + + Error for a violation; warning for a likely mistake. + + .. attribute:: location + :type: str + + Path to the affected field, e.g. ``"metrics[0].name"``; empty for the metadata as a whole. + + .. attribute:: message + :type: str + + Explanation of the problem. + Obtain validation issues with :func:`hdhelpers.metadata.check_metadata_conventions`. When required metadata cannot be read, :func:`hdhelpers.metadata.read_metadata` raises :class:`~hdhelpers.metadata.MetadataConventionError`, whose ``issues`` attribute diff --git a/src/hdhelpers/metadata/conventions.py b/src/hdhelpers/metadata/conventions.py index d5d9c5b..93b6596 100644 --- a/src/hdhelpers/metadata/conventions.py +++ b/src/hdhelpers/metadata/conventions.py @@ -18,6 +18,9 @@ The models document the request body of the hetida designer backend endpoint for checking metadata. Hence they occur as JSON Schemas in the openapi.json of the backend, so that third party projects, e.g. adapters providing or receiving such metadata, can validate against them. +Their docstrings and field descriptions become the descriptions of these schemas, so they are +written for JSON users: null, true and false instead of their Python counterparts and no Python +names, e.g. of functions or parameters. Additional fields are allowed at every level. The models are immutable, see ConventionModel. Metadata timestamps are UTC ISO strings with an explicit Z or +00:00 offset. @@ -119,7 +122,7 @@ def _utc_isoformat(value: Any) -> Any: class AggregationMethod(StrEnum): """Method for aggregating timeseries values into coarser intervals. - Metadata lookups normalize AVG to MEAN; they are distinct enum values. + AVG is an alias of MEAN. """ MEAN = "MEAN" @@ -370,7 +373,7 @@ class ValueDimensionMetadata(SharedValueDimensionMetadata): Each metric describes its value columns; a SERIES uses "value". Non-null fields take precedence over shared metadata for the same column. Unit and measurement are required - fields, but may be None if unknown. + fields, but may be null if unknown. """ unit: str | None = Field( @@ -387,9 +390,9 @@ class InheritedMetricMetadata(ConventionModel): """Properties supplied by a source system for a metric and its parent assets. Stores already-resolved values, e.g. a plant's latitude; the source system resolves the - hierarchy. Metadata lookups use these properties as fallbacks only with - ``include_inherited=True``. Convention-defined fields such as name, id and unit belong in - the metric or value-dimension metadata. + hierarchy. Consumers use these properties only as last resort and not for information + describing the metric itself, like its unit or names. Convention-defined fields such as name, + id and unit belong in the metric or value-dimension metadata. Use JSON types matching each value's meaning: numbers, booleans and UTC ISO timestamp strings for points in time. For example, hetida platform supplies properties by label, @@ -581,12 +584,12 @@ class _DatasetMetadataBase(ConventionModel): " datapoints instead of an interval. If true, the ref interval is not required.", ) ref_interval_start_timestamp: UtcTimestamp | None = optional( - description="Start of the queried time interval (ref interval) in UTC isoformat." - " Required unless ref_dataset_discrete is true.", + description="Start of the queried time interval (ref interval) as ISO 8601 timestamp" + " in UTC. Required unless ref_dataset_discrete is true.", ) ref_interval_end_timestamp: UtcTimestamp | None = optional( - description="End of the queried time interval (ref interval) in UTC isoformat." - " Required unless ref_dataset_discrete is true.", + description="End of the queried time interval (ref interval) as ISO 8601 timestamp in" + " UTC. Required unless ref_dataset_discrete is true.", ) ref_interval_type: IntervalType | None = optional( description="Type of the ref interval. Required unless ref_dataset_discrete is true.", @@ -608,10 +611,10 @@ class _DatasetMetadataBase(ConventionModel): + " Applies to all metrics without their own data_frequency.", ) query_timestamp: UtcTimestamp | None = optional( - description="The point in time whose state of the data was delivered in UTC isoformat:" - " Data stored, changed or invalidated later is not included. A source supporting it" - " delivers the same data again when queried for this point in time, e.g. for" - " reproducing an execution.", + description="The point in time whose state of the data was delivered, as ISO 8601" + " timestamp in UTC: Data stored, changed or invalidated later is not included. A source" + " supporting it delivers the same data again when queried for this point in time, e.g." + " for reproducing an execution.", ) default_aggregation_interval: str | None = optional( description="Interval to aggregate the timeseries to by default, e.g. when components" @@ -628,12 +631,12 @@ class _DatasetMetadataBase(ConventionModel): + " Applies to all metrics without their own default_timezone.", ) invalidation_interval_start: UtcTimestamp | None = optional( - description="Start of an invalidation interval separate from the ref interval in UTC" - " isoformat", + description="Start of an invalidation interval separate from the ref interval, as ISO" + " 8601 timestamp in UTC", ) invalidation_interval_end: UtcTimestamp | None = optional( - description="End of an invalidation interval separate from the ref interval in UTC" - " isoformat", + description="End of an invalidation interval separate from the ref interval, as ISO" + " 8601 timestamp in UTC", ) invalidation_interval_type: IntervalType | None = optional( IntervalType.CLOSED, description="Type of the invalidation interval" @@ -901,10 +904,10 @@ def _with_value_column(metric: MetricMetadata, column: str, new_column: str) -> class SeriesMetadata(_TimeseriesMetadataBase): - """Immutable metadata for a SERIES: one metric with a single "value" dimension. + """Metadata of a SERIES: one metric with a single "value" dimension. - Read from a pandas Series with read_metadata. See the module documentation for storage, - adapter exchange and common field rules. + Exchanged with adapters, e.g. in the Data-Attributes header of generic REST adapter endpoints + or the __metadata__ field of the direct provisioning adapter's wrapped format. """ model_config = ConfigDict(json_schema_extra={"examples": [SERIES_METADATA_EXAMPLE]}) @@ -932,10 +935,10 @@ def as_singletsframe(self, column: str = "value") -> "SingleTSFrameMetadata": class SingleTSFrameMetadata(_TimeseriesMetadataBase): - """Immutable metadata for a SINGLETSFRAME: one metric with one or more value columns. + """Metadata of a SINGLETSFRAME: one metric with one or more value columns. - Read from a pandas DataFrame with read_metadata. See the module documentation for storage, - adapter exchange and common field rules. + Exchanged with adapters, e.g. in the Data-Attributes header of generic REST adapter endpoints + or the __metadata__ field of the direct provisioning adapter's wrapped format. """ model_config = ConfigDict(json_schema_extra={"examples": [SINGLETSFRAME_METADATA_EXAMPLE]}) @@ -989,11 +992,11 @@ def as_multitsframe(self, metric_id: str | None = None) -> "MultiTSFrameMetadata class MultiTSFrameMetadata(_TimeseriesMetadataBase): - """Immutable metadata for a MULTITSFRAME: multiple metrics identified by a metric column. + """Metadata of a MULTITSFRAME: multiple metrics identified by a metric column. Each metric can have several value dimensions; a MULTITSFRAME may also hold one metric. - Read from a pandas DataFrame with read_metadata. See the module documentation for storage, - adapter exchange and common field rules. + Exchanged with adapters, e.g. in the Data-Attributes header of generic REST adapter endpoints + or the __metadata__ field of the direct provisioning adapter's wrapped format. """ model_config = ConfigDict(json_schema_extra={"examples": [MULTITSFRAME_METADATA_EXAMPLE]}) @@ -1032,7 +1035,7 @@ def narrowed_to(self, metric_id: str) -> SingleTSFrameMetadata: class MetadataIssueSeverity(StrEnum): """Severity of an issue found when checking metadata against the metadata conventions - error: Violates the conventions, so helpers and components may misinterpret the metadata. + error: Violates the conventions, so consumers like components may misinterpret the metadata. warning: Likely a mistake, e.g. metadata not matching the data. """ @@ -1041,13 +1044,9 @@ class MetadataIssueSeverity(StrEnum): class MetadataConventionIssue(BaseModel): - """A metadata validation problem, reported as an error or warning. + """A problem found when checking metadata against the metadata conventions - Attributes: - severity (MetadataIssueSeverity): Error for a violation; warning for a likely mistake. - location (str): Path to the affected field, e.g. "metrics[0].name"; empty for the - metadata as a whole. - message (str): Explanation of the problem. + An error for a violation of the conventions, a warning for a likely mistake. """ severity: MetadataIssueSeverity @@ -1056,7 +1055,7 @@ class MetadataConventionIssue(BaseModel): description="Where the issue occurs, e.g. metrics[0].value_dimensions[1].column in the" " checked metadata. Empty for the checked object as a whole.", ) - message: str + message: str = Field(..., description="Explanation of the problem") class MetadataConventionError(ValueError): diff --git a/tests/data/convention_schemas.json b/tests/data/convention_schemas.json index 80fd3c9..a440bf3 100644 --- a/tests/data/convention_schemas.json +++ b/tests/data/convention_schemas.json @@ -1,7 +1,7 @@ { "$defs": { "AggregationMethod": { - "description": "Method for aggregating timeseries values into coarser intervals.\n\nMetadata lookups normalize AVG to MEAN; they are distinct enum values.", + "description": "Method for aggregating timeseries values into coarser intervals.\n\nAVG is an alias of MEAN.", "enum": [ "MEAN", "AVG", @@ -18,7 +18,7 @@ }, "InheritedMetricMetadata": { "additionalProperties": true, - "description": "Properties supplied by a source system for a metric and its parent assets.\n\nStores already-resolved values, e.g. a plant's latitude; the source system resolves the\nhierarchy. Metadata lookups use these properties as fallbacks only with\n``include_inherited=True``. Convention-defined fields such as name, id and unit belong in\nthe metric or value-dimension metadata.\n\nUse JSON types matching each value's meaning: numbers, booleans and UTC ISO timestamp\nstrings for points in time. For example, hetida platform supplies properties by label,\nusing a signal's own value or that of the nearest parent asset providing one.", + "description": "Properties supplied by a source system for a metric and its parent assets.\n\nStores already-resolved values, e.g. a plant's latitude; the source system resolves the\nhierarchy. Consumers use these properties only as last resort and not for information\ndescribing the metric itself, like its unit or names. Convention-defined fields such as name,\nid and unit belong in the metric or value-dimension metadata.\n\nUse JSON types matching each value's meaning: numbers, booleans and UTC ISO timestamp\nstrings for points in time. For example, hetida platform supplies properties by label,\nusing a signal's own value or that of the nearest parent asset providing one.", "properties": {}, "title": "InheritedMetricMetadata", "type": "object" @@ -603,7 +603,7 @@ } ], "default": null, - "description": "End of an invalidation interval separate from the ref interval in UTC isoformat", + "description": "End of an invalidation interval separate from the ref interval, as ISO 8601 timestamp in UTC", "title": "Invalidation Interval End" }, "invalidation_interval_start": { @@ -618,7 +618,7 @@ } ], "default": null, - "description": "Start of an invalidation interval separate from the ref interval in UTC isoformat", + "description": "Start of an invalidation interval separate from the ref interval, as ISO 8601 timestamp in UTC", "title": "Invalidation Interval Start" }, "invalidation_interval_type": { @@ -701,7 +701,7 @@ } ], "default": null, - "description": "The point in time whose state of the data was delivered in UTC isoformat: Data stored, changed or invalidated later is not included. A source supporting it delivers the same data again when queried for this point in time, e.g. for reproducing an execution.", + "description": "The point in time whose state of the data was delivered, as ISO 8601 timestamp in UTC: Data stored, changed or invalidated later is not included. A source supporting it delivers the same data again when queried for this point in time, e.g. for reproducing an execution.", "title": "Query Timestamp" }, "ref_data_frequency": { @@ -768,7 +768,7 @@ } ], "default": null, - "description": "End of the queried time interval (ref interval) in UTC isoformat. Required unless ref_dataset_discrete is true.", + "description": "End of the queried time interval (ref interval) as ISO 8601 timestamp in UTC. Required unless ref_dataset_discrete is true.", "title": "Ref Interval End Timestamp" }, "ref_interval_start_timestamp": { @@ -783,7 +783,7 @@ } ], "default": null, - "description": "Start of the queried time interval (ref interval) in UTC isoformat. Required unless ref_dataset_discrete is true.", + "description": "Start of the queried time interval (ref interval) as ISO 8601 timestamp in UTC. Required unless ref_dataset_discrete is true.", "title": "Ref Interval Start Timestamp" }, "ref_interval_type": { @@ -832,7 +832,7 @@ }, "MultiTSFrameMetadata": { "additionalProperties": true, - "description": "Immutable metadata for a MULTITSFRAME: multiple metrics identified by a metric column.\n\nEach metric can have several value dimensions; a MULTITSFRAME may also hold one metric.\nRead from a pandas DataFrame with read_metadata. See the module documentation for storage,\nadapter exchange and common field rules.", + "description": "Metadata of a MULTITSFRAME: multiple metrics identified by a metric column.\n\nEach metric can have several value dimensions; a MULTITSFRAME may also hold one metric.\nExchanged with adapters, e.g. in the Data-Attributes header of generic REST adapter endpoints\nor the __metadata__ field of the direct provisioning adapter's wrapped format.", "examples": [ { "dataset_metadata": { @@ -928,7 +928,7 @@ }, "SeriesMetadata": { "additionalProperties": true, - "description": "Immutable metadata for a SERIES: one metric with a single \"value\" dimension.\n\nRead from a pandas Series with read_metadata. See the module documentation for storage,\nadapter exchange and common field rules.", + "description": "Metadata of a SERIES: one metric with a single \"value\" dimension.\n\nExchanged with adapters, e.g. in the Data-Attributes header of generic REST adapter endpoints\nor the __metadata__ field of the direct provisioning adapter's wrapped format.", "examples": [ { "dataset_metadata": { @@ -1283,7 +1283,7 @@ } ], "default": null, - "description": "End of an invalidation interval separate from the ref interval in UTC isoformat", + "description": "End of an invalidation interval separate from the ref interval, as ISO 8601 timestamp in UTC", "title": "Invalidation Interval End" }, "invalidation_interval_start": { @@ -1298,7 +1298,7 @@ } ], "default": null, - "description": "Start of an invalidation interval separate from the ref interval in UTC isoformat", + "description": "Start of an invalidation interval separate from the ref interval, as ISO 8601 timestamp in UTC", "title": "Invalidation Interval Start" }, "invalidation_interval_type": { @@ -1381,7 +1381,7 @@ } ], "default": null, - "description": "The point in time whose state of the data was delivered in UTC isoformat: Data stored, changed or invalidated later is not included. A source supporting it delivers the same data again when queried for this point in time, e.g. for reproducing an execution.", + "description": "The point in time whose state of the data was delivered, as ISO 8601 timestamp in UTC: Data stored, changed or invalidated later is not included. A source supporting it delivers the same data again when queried for this point in time, e.g. for reproducing an execution.", "title": "Query Timestamp" }, "ref_data_frequency": { @@ -1448,7 +1448,7 @@ } ], "default": null, - "description": "End of the queried time interval (ref interval) in UTC isoformat. Required unless ref_dataset_discrete is true.", + "description": "End of the queried time interval (ref interval) as ISO 8601 timestamp in UTC. Required unless ref_dataset_discrete is true.", "title": "Ref Interval End Timestamp" }, "ref_interval_start_timestamp": { @@ -1463,7 +1463,7 @@ } ], "default": null, - "description": "Start of the queried time interval (ref interval) in UTC isoformat. Required unless ref_dataset_discrete is true.", + "description": "Start of the queried time interval (ref interval) as ISO 8601 timestamp in UTC. Required unless ref_dataset_discrete is true.", "title": "Ref Interval Start Timestamp" }, "ref_interval_type": { @@ -1505,7 +1505,7 @@ }, "SingleTSFrameMetadata": { "additionalProperties": true, - "description": "Immutable metadata for a SINGLETSFRAME: one metric with one or more value columns.\n\nRead from a pandas DataFrame with read_metadata. See the module documentation for storage,\nadapter exchange and common field rules.", + "description": "Metadata of a SINGLETSFRAME: one metric with one or more value columns.\n\nExchanged with adapters, e.g. in the Data-Attributes header of generic REST adapter endpoints\nor the __metadata__ field of the direct provisioning adapter's wrapped format.", "examples": [ { "dataset_metadata": { @@ -1591,7 +1591,7 @@ }, "ValueDimensionMetadata": { "additionalProperties": true, - "description": "Metadata for one metric's value column, such as its unit and measured quantity.\n\nEach metric describes its value columns; a SERIES uses \"value\". Non-null fields take\nprecedence over shared metadata for the same column. Unit and measurement are required\nfields, but may be None if unknown.", + "description": "Metadata for one metric's value column, such as its unit and measured quantity.\n\nEach metric describes its value columns; a SERIES uses \"value\". Non-null fields take\nprecedence over shared metadata for the same column. Unit and measurement are required\nfields, but may be null if unknown.", "properties": { "column": { "description": "The actual column name in the MULTITSFRAME or SINGLETSFRAME, connecting the value dimension to its column. \"value\" for a SERIES.",