diff --git a/lib/charms/data_platform_libs/v0/data_interfaces.py b/lib/charms/data_platform_libs/v0/data_interfaces.py index 5be1d931..f4a65fdd 100644 --- a/lib/charms/data_platform_libs/v0/data_interfaces.py +++ b/lib/charms/data_platform_libs/v0/data_interfaces.py @@ -433,6 +433,11 @@ def _on_subject_requested(self, event: SubjectRequestedEvent): overload, ) +try: + from cryptography.fernet import Fernet, InvalidToken +except ImportError: + Fernet = None + InvalidToken = None from ops import JujuVersion, Model, Secret, SecretInfo, SecretNotFoundError from ops.charm import ( CharmBase, @@ -453,7 +458,7 @@ def _on_subject_requested(self, event: SubjectRequestedEvent): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 58 +LIBPATCH = 59 PYDEPS = ["ops>=2.0.0"] @@ -489,6 +494,10 @@ def _on_subject_requested(self, event: SubjectRequestedEvent): "owner_no_refresh": "ERROR secret owner cannot use --refresh", } +CROSS_MODEL_RELATION_CONSUMER_SECRETS = [ + "mtls-cert", +] + ############################################################################## # Exceptions @@ -1227,6 +1236,15 @@ def _load_secrets_from_databag(self, relation: Relation) -> None: """Load secrets from the databag.""" raise NotImplementedError + def _get_encryption_key(self, relation: Relation) -> Optional[str]: + """Fetch the encryption key from the encryption secret if available.""" + if not (encryption_secret := relation.data[relation.app].get("encryption-secret")): + return None + + # get the encryption secret created on provider side + secret = self._model.get_secret(id=encryption_secret) + return secret.get_content().get("encryption-key") + def _fetch_specific_relation_data( self, relation: Relation, fields: Optional[List[str]] ) -> Dict[str, str]: @@ -1586,11 +1604,48 @@ def _fetch_relation_data_without_secrets( return {} if fields: - return { + relation_data = { k: relation.data[component][k] for k in fields if k in relation.data[component] } else: - return dict(relation.data[component]) + relation_data = dict(relation.data[component]) + + try: + remote_model_uuid = relation.remote_model.uuid + except (FileNotFoundError, ModelError, RuntimeError) as e: + # access to remote model added in Juju 3.6.2, fails with 2.9 + logger.warning("Access to remote model failed: %s", e) + remote_model_uuid = "" + + # if not a cross-model relation we can return data as-is + if remote_model_uuid == "" or self._model.uuid == remote_model_uuid: + return relation_data + + if not (encryption_key := self._get_encryption_key(relation)): + return relation_data + + if not Fernet: + logger.warning( + "Cryptography module not installed. Could not decrypt sensitive field in cross-model relation" + ) + return relation_data + + # still here means sensitive data needs to be decrypted + for key, value in relation_data.items(): + if key in CROSS_MODEL_RELATION_CONSUMER_SECRETS: + try: + f = Fernet(encryption_key) + decrypted_value = f.decrypt(value.encode()).decode() + relation_data[key] = decrypted_value + except ( + AttributeError, + InvalidToken, + TypeError, + ValueError, + ): # pyright: ignore [reportGeneralTypeIssues] + logger.warning("Could not decrypt sensitive field in cross-model relation") + + return relation_data def _fetch_relation_data_with_secrets( self, @@ -1637,8 +1692,40 @@ def _update_relation_data_without_secrets( if component not in relation.data or relation.data[component] is None: return - if relation: - relation.data[component].update(data) + if not relation: + return + + # ensure no sensitive information is stored in relation data + encryption_key = self._get_encryption_key(relation) + + try: + remote_model_uuid = relation.remote_model.uuid + except (FileNotFoundError, ModelError, RuntimeError): + # access to remote model added in Juju 3.6.2, fails with 2.9 + remote_model_uuid = "" + + if encryption_key and remote_model_uuid != "" and self._model.uuid != remote_model_uuid: + for key, value in data.items(): + try: + f = Fernet(encryption_key) # pyright: ignore [reportOptionalCall] + if key in CROSS_MODEL_RELATION_CONSUMER_SECRETS: + encrypted_value = f.encrypt(value.encode()).decode() + data[key] = encrypted_value + except ( + AttributeError, + InvalidToken, + ValueError, + ): # pyright: ignore [reportGeneralTypeIssues] + logger.warning("Could not encrypt sensitive field in cross-model relation") + data[key] = "" + except TypeError: + # "TypeError: 'NoneType' object is not callable" raised when Fernet is `None` + logger.warning( + "Cryptography module not installed. Could not decrypt sensitive field in cross-model relation" + ) + data[key] = "" + + relation.data[component].update(data) def _delete_relation_data_without_secrets( self, component: Union[Application, Unit], relation: Relation, fields: List[str] @@ -1899,7 +1986,7 @@ def _update_relation_data(self, relation: Relation, data: Dict[str, str]) -> Non """Set values for fields not caring whether it's a secret or not.""" keys = set(data.keys()) if self.fetch_relation_field(relation.id, self.RESOURCE_FIELD) is None and ( - keys - {"endpoints", "read-only-endpoints", "replset"} + keys - {"endpoints", "read-only-endpoints", "replset", "encryption-secret"} ): raise PrematureDataAccessError( "Premature access to relation data, update is forbidden before the connection is initialized." @@ -2112,12 +2199,31 @@ def __init__( field for field in self.SECRET_LABEL_MAP.keys() if field not in self._remote_secret_fields + and not ( + field in CROSS_MODEL_RELATION_CONSUMER_SECRETS and self.is_cross_model_relation + ) ] if additional_secret_fields: self._remote_secret_fields += additional_secret_fields self.data_component = self.local_unit # Internal functions + @property + def is_cross_model_relation(self) -> bool: + """Determines whether the relation is a cross-model relation or not.""" + if len(self.relations) == 0: + return False + + try: + remote_model_uuid = self.relations[0].remote_model.uuid + except (FileNotFoundError, ModelError, RuntimeError): + # access to remote model added in Juju 3.6.2, fails with 2.9 + return False + + if self._model.uuid != remote_model_uuid: + return True + + return False def _is_resource_created_for_relation(self, relation: Relation) -> bool: if not relation.app: @@ -2389,6 +2495,44 @@ def _validate_entity_consistency(event: RelationEvent, diff: Diff) -> None: raise ValueError(f"Cannot change {key} after relation has already been created") # Event handlers + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: + """Event emitted when a relation is created.""" + if not self.relation_data.local_unit.is_leader(): + return + + try: + remote_model_uuid = event.relation.remote_model.uuid + except (FileNotFoundError, ModelError, RuntimeError): + # access to remote model added in Juju 3.6.2, fails with 2.9 + return + + if self.model.uuid == remote_model_uuid: + return + + if not Fernet: + logger.warning( + "Cryptography module not installed. Could not decrypt sensitive field in cross-model relation" + ) + return + + # in cross-model relations, generate an encryption key and share it with the requirer as a secret + event_data = {} + secret_label = f"{self.model.uuid}-{event.relation.id}-encryption-secret" + + try: + # check if secret was already created to avoid duplicates + secret = self.charm.model.get_secret(label=secret_label) + except SecretNotFoundError: + encryption_key = Fernet.generate_key() # pyright: ignore [reportOptionalMemberAccess] + content = {"encryption-key": encryption_key.decode()} + secret = self.charm.app.add_secret(content, label=secret_label) + + secret.grant(event.relation) + if not secret.id: + raise SecretError("Encryption secret is missing secred id") + event_data["encryption-secret"] = secret.id + + self.relation_data.update_relation_data(event.relation.id, event_data) def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: """Event emitted when the relation data has changed.""" @@ -4412,6 +4556,33 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: # Check which data has changed to emit customs events. diff = self._diff(event) + # send request again if encryption secret was added from provider side + if "encryption-secret" in diff.added and self.relation_data.local_unit.is_leader(): + relation_data = { + "topic": self.relation_data.topic, + "encryption-secret": event.relation.data[event.relation.app].get( + "encryption-secret" + ), + } + + if self.relation_data.mtls_cert: + relation_data["mtls-cert"] = self.relation_data.mtls_cert + + if self.relation_data.consumer_group_prefix: + relation_data["consumer-group-prefix"] = self.relation_data.consumer_group_prefix + + if self.relation_data.extra_user_roles: + relation_data["extra-user-roles"] = self.relation_data.extra_user_roles + if self.relation_data.extra_group_roles: + relation_data["extra-group-roles"] = self.relation_data.extra_group_roles + if self.relation_data.entity_type: + relation_data["entity-type"] = self.relation_data.entity_type + if self.relation_data.entity_permissions: + relation_data["entity-permissions"] = self.relation_data.entity_permissions + + self.relation_data.update_relation_data(event.relation.id, relation_data) + return + # Check if the topic is created # (the Kafka charm shared the credentials). @@ -5689,6 +5860,24 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: # Check which data has changed to emit customs events. diff = self._diff(event) + + # send request again if encryption secret was added from provider side + if "encryption-secret" in diff.added and self.relation_data.local_unit.is_leader(): + payload = { + "prefix": self.relation_data.prefix, + "encryption-secret": event.relation.data[event.relation.app].get( + "encryption-secret" + ), + } + if self.relation_data.mtls_cert: + payload["mtls-cert"] = self.relation_data.mtls_cert + + self.relation_data.update_relation_data( + event.relation.id, + payload, + ) + return + # Register all new secrets with their labels if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): self.relation_data._register_secrets_to_relation(event.relation, diff.added) diff --git a/lib/charms/grafana_k8s/v0/grafana_dashboard.py b/lib/charms/grafana_k8s/v0/grafana_dashboard.py index 9886fc2b..29ede242 100644 --- a/lib/charms/grafana_k8s/v0/grafana_dashboard.py +++ b/lib/charms/grafana_k8s/v0/grafana_dashboard.py @@ -184,7 +184,7 @@ def __init__(self, *args): import re import subprocess import tempfile -import uuid + from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Tuple import yaml @@ -217,7 +217,7 @@ def __init__(self, *args): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 49 +LIBPATCH = 52 PYDEPS = ["cosl >= 0.0.50"] @@ -392,6 +392,13 @@ def __init__(self, *args): } +def _data_hash(data: Any) -> str: + """Deterministic hash of a template dict for use as a stable relation data key.""" + return hashlib.shake_128( + json.dumps(data, sort_keys=True).encode() + ).digest(8).hex() + + class RelationNotFoundError(Exception): """Raised if there is no relation with the given name.""" @@ -1349,11 +1356,29 @@ def _on_grafana_dashboard_relation_changed(self, event: RelationChangedEvent) -> def _upset_dashboards_on_relation(self, relation: Relation) -> None: """Update the dashboards in the relation data bucket.""" + new_templates = type_convert_stored(self._stored.dashboard_templates) # pyright: ignore + + # Check if the templates have actually changed before updating. + # This avoids generating a new UUID on every event, which would cause + # unnecessary relation-changed events on the consumer side. + # See: https://github.com/canonical/opentelemetry-collector-operator/issues/331 + existing_data_str = relation.data[self._charm.app].get("dashboards", "{}") + try: + existing_data = json.loads(existing_data_str) + existing_templates = existing_data.get("templates", {}) + except json.JSONDecodeError: + existing_templates = {} + + if new_templates == existing_templates: + return # No change in templates, don't update the databag + # It's completely ridiculous to add a UUID, but if we don't have some - # pseudo-random value, this never makes it across 'juju set-state' + # pseudo-random value, this never makes it across 'juju set-state'. + # Use a deterministic hash of the templates so the value is stable when + # templates haven't changed, avoiding spurious relation-changed events. stored_data = { - "templates": type_convert_stored(self._stored.dashboard_templates), # pyright: ignore - "uuid": str(uuid.uuid4()), + "templates": new_templates, + "uuid": _data_hash(new_templates), } relation.data[self._charm.app]["dashboards"] = json.dumps(stored_data) @@ -1495,6 +1520,41 @@ def update_dashboards(self, relation: Optional[Relation] = None) -> None: for relation in relations: self._render_dashboards_and_signal_changed(relation) + def has_invalid_dashboards(self) -> bool: + """Check whether any relation reported invalid dashboards. + + Validation errors written to relation app data by this consumer (see + :meth:`_render_dashboards_and_signal_changed`) are read back to determine + whether the relationship currently carries an invalid dashboard. + + Returns: + True if any related dashboard provider reported dashboard validation + errors, False otherwise. + """ + if not self._charm.unit.is_leader(): + return False + + for relation in self._charm.model.relations.get(self._relation_name, []): + app_data = relation.data.get(self._charm.app) + if not app_data: + continue + + event_raw = app_data.get("event", "{}") + try: + event_data = json.loads(event_raw) + except (json.JSONDecodeError, TypeError): + continue + + if event_data.get("errors"): + logger.error( + "Invalid dashboards on relation %s: %s", + relation.id, + event_data["errors"], + ) + return True + + return False + def _on_grafana_dashboard_relation_broken(self, event: RelationBrokenEvent) -> None: """Update job config when providers depart. @@ -1567,7 +1627,11 @@ def _render_dashboards_and_signal_changed(self, relation: Relation) -> bool: # except json.JSONDecodeError as e: error = str(e.msg) logger.warning("Invalid JSON in Grafana dashboard '{}': {}".format(fname, error)) - continue + relation_has_invalid_dashboards = True + except (KeyError, TypeError, AttributeError) as e: + error = str(e) + logger.warning("Invalid Grafana dashboard '{}': {}".format(fname, error)) + relation_has_invalid_dashboards = True # Prepend the relation name and ID to the dashboard ID to avoid clashes with # multiple relations with apps from the same charm, or having dashboards with @@ -1615,6 +1679,12 @@ def _render_dashboards_and_signal_changed(self, relation: Relation) -> bool: # # Dropping dashboards for a relation needs to be signalled return True + # Clear any stale validation errors so the charm returns to Active once fixed + event_data = json.loads(relation.data[self._charm.app].get("event", "{}")) + if event_data.get("errors"): + event_data.pop("errors") + relation.data[self._charm.app]["event"] = json.dumps(event_data) + stored_data = rendered_dashboards currently_stored_data = self._get_stored_dashboards(relation.id) @@ -1717,7 +1787,7 @@ def set_peer_data(self, key: str, data: Any) -> None: if not peers or not peers.data: logger.info("set_peer_data: no peer relation. Is the charm being installed/removed?") return - peers.data[self._charm.app][key] = json.dumps(data) # type: ignore[attr-defined] + peers.data[self._charm.app][key] = json.dumps(data, sort_keys=True) # type: ignore[attr-defined] def get_peer_data(self, key: str) -> Any: """Retrieve information from the peer data bucket instead of `StoredState`.""" @@ -1831,14 +1901,30 @@ def _upset_dashboards_on_event(self, event: RelationEvent) -> None: def _update_remote_grafana(self, _: Optional[RelationEvent] = None) -> None: """Push dashboards to the downstream Grafana relation.""" - # It's still ridiculous to add a UUID here, but needed - stored_data = { - "templates": type_convert_stored(self._stored.dashboard_templates), # pyright: ignore - "uuid": str(uuid.uuid4()), - } + new_templates = type_convert_stored(self._stored.dashboard_templates) # pyright: ignore if self._charm.unit.is_leader(): for grafana_relation in self.model.relations[self._grafana_relation]: + # Check if the templates have actually changed before updating. + # This avoids generating a new UUID on every event, which would cause + # unnecessary relation-changed events on the consumer side. + existing_data_str = grafana_relation.data[self._charm.app].get("dashboards", "{}") + try: + existing_data = json.loads(existing_data_str) + existing_templates = existing_data.get("templates", {}) + except json.JSONDecodeError: + existing_templates = {} + + if new_templates == existing_templates: + continue # No change in templates, don't update the databag + + # It's still ridiculous to add a UUID here, but needed. + # Use a deterministic hash of the templates so the value is stable when + # templates haven't changed, avoiding spurious relation-changed events. + stored_data = { + "templates": new_templates, + "uuid": _data_hash(new_templates), + } grafana_relation.data[self._charm.app]["dashboards"] = json.dumps(stored_data) def remove_dashboards(self, event: RelationBrokenEvent) -> None: @@ -1853,9 +1939,10 @@ def remove_dashboards(self, event: RelationBrokenEvent) -> None: for id in app_ids: del self._stored.dashboard_templates[id] # type: ignore + remaining_templates = type_convert_stored(self._stored.dashboard_templates) # pyright: ignore stored_data = { - "templates": type_convert_stored(self._stored.dashboard_templates), # pyright: ignore - "uuid": str(uuid.uuid4()), + "templates": remaining_templates, + "uuid": _data_hash(remaining_templates), } if self._charm.unit.is_leader(): @@ -2100,11 +2187,11 @@ def validate_alert_rules(self, rules: dict) -> Tuple[bool, str]: # - alert: OtherAlert # expr: up transformed_rules = {"groups": []} # type: ignore - for rule in rules["groups"]: - transformed = {"name": str(uuid.uuid4()), "rules": [rule]} + for i, rule in enumerate(rules["groups"]): + transformed = {"name": f"group_{i}", "rules": [rule]} transformed_rules["groups"].append(transformed) - rule_path.write_text(yaml.dump(transformed_rules)) + rule_path.write_text(yaml.safe_dump(transformed_rules, sort_keys=True)) # databag-order: ignore args = [str(self.path), "validate", str(rule_path)] # noinspection PyBroadException diff --git a/lib/charms/hydra/v0/oauth.py b/lib/charms/hydra/v0/oauth.py index c0b35a3a..f3725536 100644 --- a/lib/charms/hydra/v0/oauth.py +++ b/lib/charms/hydra/v0/oauth.py @@ -46,6 +46,17 @@ def _set_client_config(self): ) self.oauth.update_client_config(client_config) ``` + +## Provider + +Besides the `client_created`/`client_changed` events, a provider can read a requirer's +published configuration at any time with `OAuthProvider.get_client_config(relation)`. It +returns `None` when the requirer has published nothing yet and raises `DataValidationError` +when what it published does not match the requirer schema. Use it to reconcile registered +clients holistically rather than relying on an event having been delivered. + +Note that `client_created`/`client_changed` are not emitted when the requirer's data fails +validation; the failure is logged and the relation is skipped rather than erroring the hook. """ import json @@ -67,7 +78,7 @@ def _set_client_config(self): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 11 +LIBPATCH = 13 PYDEPS = ["jsonschema"] @@ -165,7 +176,24 @@ def _set_client_config(self): "default": "client_secret_basic", }, }, - "required": ["redirect_uri", "audience", "scope", "grant_types", "token_endpoint_auth_method"], + "required": ["audience", "scope", "grant_types", "token_endpoint_auth_method"], + "allOf": [ + { + "if": { + "properties": { + "grant_types": { + "contains": { + "const": "authorization_code", + } + } + }, + "required": ["grant_types"], + }, + "then": { + "required": ["redirect_uri"], + }, + } + ], } @@ -264,7 +292,7 @@ def _validate_data(data: Dict, schema: Dict) -> None: class ClientConfig: """Helper class containing a client's configuration.""" - redirect_uri: str + redirect_uri: str | None scope: str grant_types: List[str] audience: List[str] = field(default_factory=lambda: []) @@ -273,18 +301,23 @@ class ClientConfig: def validate(self) -> None: """Validate the client configuration.""" - # Validate redirect_uri - if not re.match(url_regex, self.redirect_uri): + if "authorization_code" in self.grant_types and not self.redirect_uri: + raise ClientConfigError( + "redirect_uri is required when using authorization_code grant_type" + ) + + # Validate redirect_uri when configured + if self.redirect_uri is not None and not re.match(url_regex, self.redirect_uri): raise ClientConfigError(f"Invalid URL {self.redirect_uri}") - if self.redirect_uri.startswith("http://"): + if self.redirect_uri is not None and self.redirect_uri.startswith("http://"): logger.warning("Provided Redirect URL uses http scheme. Don't do this in production") # Validate grant_types for grant_type in self.grant_types: if grant_type not in ALLOWED_GRANT_TYPES: raise ClientConfigError( - f"Invalid grant_type {grant_type}, must be one " f"of {ALLOWED_GRANT_TYPES}" + f"Invalid grant_type {grant_type}, must be one of {ALLOWED_GRANT_TYPES}" ) # Validate client authentication methods @@ -508,7 +541,9 @@ def get_provider_info( client_secret_id = data.get("client_secret_id") if client_secret_id: _client_secret = self.get_client_secret(client_secret_id) - client_secret = _client_secret.get_content()[CLIENT_SECRET_FIELD] + # `refresh=True`: the provider cuts a new revision when it rotates the secret, + # and nothing here observes `secret-changed`, so the tracked revision goes stale. + client_secret = _client_secret.get_content(refresh=True)[CLIENT_SECRET_FIELD] data["client_secret"] = client_secret oauth_provider = OauthProviderConfig.from_dict(data) @@ -693,7 +728,11 @@ def _get_client_config_from_relation_data(self, event: RelationChangedEvent) -> logger.info("No requirer relation data available.") return - client_data = _load_data(data, OAUTH_REQUIRER_JSON_SCHEMA) + try: + client_data = _load_data(data, OAUTH_REQUIRER_JSON_SCHEMA) + except DataValidationError: + logger.warning("The requirer relation data is not valid yet.") + return redirect_uri = client_data.get("redirect_uri") scope = client_data.get("scope") grant_types = client_data.get("grant_types") @@ -704,7 +743,13 @@ def _get_client_config_from_relation_data(self, event: RelationChangedEvent) -> if not data: logger.info("No provider relation data available.") return - provider_data = _load_data(data, OAUTH_PROVIDER_JSON_SCHEMA) + try: + provider_data = _load_data(data, OAUTH_PROVIDER_JSON_SCHEMA) + except DataValidationError: + # A partially written provider databag must not wedge the hook: the charm can + # only repair it from a later hook, which would never run. + logger.warning("The provider relation data is not complete yet.") + return client_id = provider_data.get("client_id") relation_id = event.relation.id @@ -739,9 +784,15 @@ def _on_relation_broken(self, event: RelationBrokenEvent) -> None: self.on.client_deleted.emit(event.relation.id) def _create_juju_secret(self, client_secret: str, relation: Relation) -> Secret: - """Create a juju secret and grant it to a relation.""" - secret = {CLIENT_SECRET_FIELD: client_secret} - juju_secret = self.model.app.add_secret(secret, label=self._get_secret_label(relation)) + """Create or update a juju secret and grant it to a relation.""" + content = {CLIENT_SECRET_FIELD: client_secret} + label = self._get_secret_label(relation) + try: + juju_secret = self.model.get_secret(label=label) + except SecretNotFoundError: + juju_secret = self.model.app.add_secret(content, label=label) + else: + juju_secret.set_content(content) juju_secret.grant(relation) return juju_secret @@ -756,6 +807,43 @@ def _delete_juju_secret(self, relation: Relation) -> None: def remove_secret(self, relation: Relation) -> None: return self._delete_juju_secret(relation) + def get_client_secret(self, relation: Relation) -> Optional[str]: + """Return the client secret currently shared with the requirer, if there is one. + + Re-registering a client with this value keeps the requirer working: writing a + different secret would cut a new revision that the requirer does not track. + """ + try: + secret = self.model.get_secret(label=self._get_secret_label(relation)) + except SecretNotFoundError: + return None + + return secret.get_content(refresh=True).get(CLIENT_SECRET_FIELD) + + def get_client_config(self, relation: Relation) -> Optional[ClientConfig]: + """Read the requirer's client configuration from the integration databag. + + Returns None when the requirer has not published its configuration yet. + + Raises: + DataValidationError: if the published data does not match the requirer schema. + """ + if not relation.app: + return None + + data = relation.data[relation.app] + if not data: + return None + + client_data = _load_data(data, OAUTH_REQUIRER_JSON_SCHEMA) + return ClientConfig( + redirect_uri=client_data.get("redirect_uri"), + scope=client_data["scope"], + grant_types=client_data["grant_types"], + audience=client_data["audience"], + token_endpoint_auth_method=client_data["token_endpoint_auth_method"], + ) + def set_provider_info_in_relation_data( self, issuer_url: str, diff --git a/lib/charms/loki_k8s/v1/loki_push_api.py b/lib/charms/loki_k8s/v1/loki_push_api.py index b3ad8223..2b3f09f5 100644 --- a/lib/charms/loki_k8s/v1/loki_push_api.py +++ b/lib/charms/loki_k8s/v1/loki_push_api.py @@ -544,7 +544,7 @@ def __init__(self, ...): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 26 +LIBPATCH = 33 PYDEPS = ["cosl"] @@ -1188,7 +1188,10 @@ def alerts(self) -> dict: # noqa: C901 ) continue - alerts[identifier] = self._tool.apply_label_matchers(alert_rules) # type: ignore + # Topology labels are already injected by _inject_alert_expr_labels using + # alert_expression_dict, which intentionally excludes juju_charm and juju_unit. + # Don't call apply_label_matchers here as it would re-inject juju_charm. + alerts[identifier] = alert_rules _, errmsg = self._tool.validate_alert_rules(cast(OfficialRuleFileFormat, alert_rules)) if errmsg: @@ -1198,11 +1201,51 @@ def alerts(self) -> dict: # noqa: C901 if self._charm.unit.is_leader(): relation.data[self._charm.app]["event"] = json.dumps({"errors": errmsg}) continue + if self._charm.unit.is_leader(): + event_data = json.loads(relation.data[self._charm.app].get("event", "{}")) + event_data.pop("errors", None) + relation.data[self._charm.app]["event"] = json.dumps(event_data) alerts[identifier] = alert_rules return alerts + def has_invalid_alert_rules(self) -> bool: + """Check whether any relation reported invalid alert rules. + + Validation errors, written to relation app data by the :attr:`alerts` + property, are read back to determine whether the relation currently + carries invalid alert rules. Non-leader units never write the app data + that holds these errors, so they always report no errors. + + Returns: + True if any related consumer reported alert rule validation + errors, False otherwise. + """ + if not self._charm.unit.is_leader(): + return False + + for relation in self._charm.model.relations.get(self._relation_name, []): + app_data = relation.data.get(self._charm.app) + if not app_data: + continue + + event_raw = app_data.get("event", "{}") + try: + event_data = json.loads(event_raw) + except (json.JSONDecodeError, TypeError): + continue + + if error_msg := event_data.get("errors"): + logger.error( + "Alert rule validation error on relation %s: %s", + relation.id, + error_msg, + ) + return True + + return False + def _get_identifier_by_alert_rules( self, rules: dict ) -> Tuple[Union[str, None], Union[JujuTopology, None]]: @@ -1279,10 +1322,13 @@ def _inject_alert_expr_labels(self, rules: Dict[str, Any]) -> Dict[str, Any]: charm_name=labels.get("juju_charm", ""), ) - # Inject topology and put it back in the list + # Inject topology and put it back in the list. + # Use alert_expression_dict (excludes juju_charm) instead of + # label_matcher_dict because subordinate charms (e.g. otelcol) + # label logs with their own charm name, not the principal's. rule["expr"] = self._tool.inject_label_matchers( re.sub(r"%%juju_topology%%,?", "", rule["expr"]), - topology.label_matcher_dict, + topology.alert_expression_dict, ) except KeyError: # Some required JujuTopology key is missing. Just move on. @@ -1378,7 +1424,9 @@ def loki_endpoints(self) -> List[dict]: seen_urls = set() for relation in self._charm.model.relations[self._relation_name]: - for unit in relation.units: + # Sort the units so the endpoints list order is stable across runs, + # otherwise the generated promtail config flaps. + for unit in sorted(relation.units, key=lambda u: u.name): if unit.app == self._charm.app: continue @@ -1407,7 +1455,6 @@ def loki_endpoints(self) -> List[dict]: return endpoints - class LokiPushApiConsumer(ConsumerBase): """Loki Consumer class.""" @@ -1700,7 +1747,7 @@ def __init__( self._promtails_ports = self._generate_promtails_ports(logs_scheme) # architecture used for promtail binary - arch = platform.processor() + arch = platform.machine() if arch in ["x86_64", "amd64"]: self._arch = "amd64" elif arch in ["aarch64", "arm64", "armv8b", "armv8l"]: @@ -1742,22 +1789,13 @@ def _on_relation_changed(self, event: RelationEvent) -> None: self._handle_alert_rules(event.relation) if self._charm.unit.is_leader(): - ev = json.loads(event.relation.data[event.app].get("event", "{}")) - - if ev: - valid = bool(ev.get("valid", True)) - errors = ev.get("errors", "") - - if valid and not errors: - self.on.alert_rule_status_changed.emit(valid=valid) - else: - self.on.alert_rule_status_changed.emit(valid=valid, errors=errors) + self._handle_alert_rule_status_changed(event) for container in self._containers.values(): if not container.can_connect(): continue if self.model.relations[self._relation_name]: - if "promtail" not in container.get_plan().services: + if not self._is_promtail_set_up(container): self._setup_promtail(container) continue @@ -1770,11 +1808,24 @@ def _on_relation_changed(self, event: RelationEvent) -> None: # Loki may send endpoints late. Don't necessarily start, there may be # no clients if new_config["clients"]: - container.restart(WORKLOAD_SERVICE_NAME) - self.on.log_proxy_endpoint_joined.emit() + if self._restart_promtail(container): + self.on.log_proxy_endpoint_joined.emit() else: self.on.promtail_digest_error.emit("No promtail client endpoints available!") + def _handle_alert_rule_status_changed(self, event: RelationEvent) -> None: + """Relay the alert rule validation status reported by the Loki provider.""" + ev = json.loads(event.relation.data[event.app].get("event", "{}")) + + if ev: + valid = bool(ev.get("valid", True)) + errors = ev.get("errors", "") + + if valid and not errors: + self.on.alert_rule_status_changed.emit(valid=valid) + else: + self.on.alert_rule_status_changed.emit(valid=valid, errors=errors) + def _on_relation_departed(self, _: RelationEvent) -> None: """Event handler for `relation_departed`. @@ -1793,11 +1844,28 @@ def _on_relation_departed(self, _: RelationEvent) -> None: container.push(WORKLOAD_CONFIG_PATH, yaml.safe_dump(new_config), make_dirs=True) if new_config["clients"]: - container.restart(WORKLOAD_SERVICE_NAME) + self._restart_promtail(container) else: container.stop(WORKLOAD_SERVICE_NAME) self.on.log_proxy_endpoint_departed.emit() + def _restart_promtail(self, container: Container) -> bool: + """Restart promtail, surfacing a Pebble failure as a digest error. + + Args: + container: the workload container running the promtail service. + + Returns: + True on success, False if the restart failed. + """ + try: + container.restart(WORKLOAD_SERVICE_NAME) + except ChangeError as e: + logger.warning("Failed to restart promtail: %s", e) + self.on.promtail_digest_error.emit(str(e)) + return False + return True + def _add_pebble_layer(self, workload_binary_path: str, container: Container) -> None: """Adds Pebble layer that manages Promtail service in Workload container. @@ -2157,6 +2225,35 @@ def _generate_static_configs(self, config: dict, container_name: str) -> list: return static_configs + def _promtail_binary_spec(self) -> dict: + """The promtail binary metadata advertised on the log-proxy relation.""" + relations = self._charm.model.relations[self._relation_name] + if not relations: + return {} + relation = relations[0] + return json.loads(relation.data[relation.app].get("promtail_binary_zip_url", "{}")) + + def _is_promtail_set_up(self, container: Container) -> bool: + """Whether promtail is fully usable in this container. + + Unlike ``_is_promtail_installed`` (binary only), this also requires the + pebble service to be registered. + + The pebble plan alone is not proof: if the workload container lost its + ephemeral filesystem (e.g. after pod churn) the layer may still be in + the plan while the runtime-pushed binary is gone. Trusting the plan and + restarting unconditionally wedges the unit + (https://github.com/canonical/loki-k8s-operator/issues/659). + """ + if "promtail" not in container.get_plan().services: + return False + promtail_info = self._promtail_binary_spec().get(self._arch) + if not promtail_info: + # No promtail binary advertised for this architecture (or nothing + # published on the relation yet), so promtail cannot be running here. + return False + return self._is_promtail_installed(promtail_info, container) + def _setup_promtail(self, container: Container) -> None: # Use the first relations = self._charm.model.relations[self._relation_name] @@ -2171,10 +2268,22 @@ def _setup_promtail(self, container: Container) -> None: relation.data[relation.app].get("promtail_binary_zip_url", "{}") ) if not promtail_binaries: + # The Loki charm hasn't published the binary metadata yet; a later + # relation-changed will bring us back here. + return + + if self._arch not in promtail_binaries: + msg = f"No promtail binary available for architecture {self._arch}" + logger.warning(msg) + self.on.promtail_digest_error.emit(msg) return self._create_directories(container) - self._ensure_promtail_binary(promtail_binaries, container) + if not self._ensure_promtail_binary(promtail_binaries, container): + # Do not add the pebble layer: a service whose command points to a + # missing binary would wedge the unit on every restart attempt + # (https://github.com/canonical/loki-k8s-operator/issues/659). + return container.push( WORKLOAD_CONFIG_PATH, @@ -2197,9 +2306,18 @@ def _setup_promtail(self, container: Container) -> None: else: self.on.promtail_digest_error.emit("No promtail client endpoints available!") - def _ensure_promtail_binary(self, promtail_binaries: dict, container: Container): + def _ensure_promtail_binary(self, promtail_binaries: dict, container: Container) -> bool: + """Ensure the promtail binary is present in the workload container. + + Args: + promtail_binaries: dictionary of promtail binaries per architecture. + container: container in which promtail must be installed. + + Returns: + True if the binary is available, False if it could not be obtained. + """ if self._is_promtail_installed(promtail_binaries[self._arch], container): - return + return True try: self._obtain_promtail(promtail_binaries[self._arch], container) @@ -2207,6 +2325,8 @@ def _ensure_promtail_binary(self, promtail_binaries: dict, container: Container) msg = f"Promtail binary couldn't be downloaded - {str(e)}" logger.warning(msg) self.on.promtail_digest_error.emit(msg) + return False + return True def _is_promtail_installed(self, promtail_info: dict, container: Container) -> bool: """Determine if promtail has already been installed to the container. @@ -2294,6 +2414,7 @@ def _build_log_target( "juju_model_uuid": topology._model_uuid, "juju_application": topology._application, "juju_unit": topology._unit, + "job": f"juju_{topology.identifier}", }, } ) diff --git a/lib/charms/prometheus_k8s/v0/prometheus_scrape.py b/lib/charms/prometheus_k8s/v0/prometheus_scrape.py index 6e102a6a..6f94cded 100644 --- a/lib/charms/prometheus_k8s/v0/prometheus_scrape.py +++ b/lib/charms/prometheus_k8s/v0/prometheus_scrape.py @@ -335,12 +335,13 @@ def _on_scrape_targets_changed(self, event): import tempfile from collections import defaultdict from pathlib import Path -from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union +from typing import Callable, Dict, List, Literal, Optional, Tuple, Union from urllib.parse import urlparse import yaml -from cosl import JujuTopology +from cosl import CosTool, JujuTopology from cosl.rules import AlertRules, generic_alert_groups +from cosl.types import OfficialRuleFileFormat from ops.charm import CharmBase, RelationRole from ops.framework import ( BoundEvent, @@ -361,7 +362,7 @@ def _on_scrape_targets_changed(self, event): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 60 +LIBPATCH = 66 # Version 0.0.53 needed for cosl.rules.generic_alert_groups PYDEPS = ["cosl>=0.0.53"] @@ -843,7 +844,7 @@ def _type_convert_stored(obj): if isinstance(obj, StoredList): return list(map(_type_convert_stored, obj)) if isinstance(obj, StoredDict): - rdict = {} # type: Dict[Any, Any] + rdict = {} for k in obj.keys(): rdict[k] = _type_convert_stored(obj[k]) return rdict @@ -993,7 +994,7 @@ def __init__( self._charm = charm self._relation_name = relation_name self._fallback_scrape_protocol = fallback_scrape_protocol - self._tool = CosTool(self._charm) + self._tool = CosTool("promql") events = self._charm.on[relation_name] self.framework.observe(events.relation_changed, self._on_metrics_provider_relation_changed) self.framework.observe( @@ -1052,13 +1053,18 @@ def jobs(self) -> list: # Therefore we need to dedupe here and after all jobs are collected. static_scrape_jobs = _dedupe_job_names(static_scrape_jobs) try: - self._tool.validate_scrape_jobs(static_scrape_jobs) + _validate_scrape_jobs(static_scrape_jobs) except subprocess.CalledProcessError as e: + logger.error(f"Invalid scrape job file: {e}") if self._charm.unit.is_leader(): data = json.loads(relation.data[self._charm.app].get("event", "{}")) data["scrape_job_errors"] = str(e) relation.data[self._charm.app]["event"] = json.dumps(data) else: + if self._charm.unit.is_leader(): + data = json.loads(relation.data[self._charm.app].get("event", "{}")) + data.pop("scrape_job_errors", None) + relation.data[self._charm.app]["event"] = json.dumps(data) scrape_jobs.extend(static_scrape_jobs) scrape_jobs = _dedupe_job_names(scrape_jobs) @@ -1107,7 +1113,7 @@ def alerts(self) -> dict: A dictionary mapping the Juju topology identifier of the source charm to its list of alert rule groups. """ - alerts = {} # type: Dict[str, dict] # mapping b/w juju identifiers and alert rule files + alerts: Dict[str, OfficialRuleFileFormat] = {} for relation in self._charm.model.relations[self._relation_name]: if not relation.units or not relation.app: continue @@ -1161,7 +1167,7 @@ def alerts(self) -> dict: return alerts def _get_identifier_by_alert_rules( - self, rules: dict + self, rules: OfficialRuleFileFormat ) -> Tuple[Union[str, None], Union[JujuTopology, None]]: """Determine an appropriate dict key for alert rules. @@ -1181,7 +1187,9 @@ def _get_identifier_by_alert_rules( # Construct an ID based on what's in the alert rules if they have labels for group in rules["groups"]: try: - labels = group["rules"][0]["labels"] + labels = group["rules"][0].get("labels") + if not labels: + continue topology = JujuTopology( # Don't try to safely get required constructor fields. There's already # a handler for KeyErrors @@ -1208,7 +1216,7 @@ def _get_identifier_by_alert_rules( return None, None - def _inject_alert_expr_labels(self, rules: Dict[str, Any]) -> Dict[str, Any]: + def _inject_alert_expr_labels(self, rules: OfficialRuleFileFormat) -> OfficialRuleFileFormat: """Iterate through alert rules and inject topology into expressions. Args: @@ -1355,6 +1363,102 @@ def _target_parts(self, target) -> list: return parts + def has_invalid_scrape_jobs(self) -> bool: + """Check whether any relation reported invalid scrape jobs. + + Validation errors, written to relation app data by this consumer + (see :meth:`jobs`), are read back to determine whether the relationship + currently carries an invalid scrape job. + + Returns: + True if any related metrics provider reported scrape job validation + errors, False otherwise. + """ + return self._has_relation_error("scrape_job_errors", "Scrape job validation error") + + def has_invalid_alert_rules(self) -> bool: + """Check whether any relation reported invalid alert rules. + + Validation errors, written to relation app data by this consumer + (see :attr:`alerts`), are read back to determine whether the relationship + currently carries invalid alert rules. + + Returns: + True if any related metrics provider reported alert rule validation + errors, False otherwise. + """ + return self._has_relation_error("errors", "Alert rule validation error") + + def _has_relation_error(self, error_key: str, error_label: str) -> bool: + """Check whether any relation reported the given validation error. + + Args: + error_key: the relation app data key that holds the validation error, + i.e. "scrape_job_errors" or "errors". + error_label: a human readable description of the validation error + type, used for logging, e.g. "Scrape job validation error". + + Returns: + True if any related metrics provider reported the validation error, + False otherwise. + """ + if not self._charm.unit.is_leader(): + return False + + for relation in self._charm.model.relations.get(self._relation_name, []): + app_data = relation.data.get(self._charm.app) + if not app_data: + continue + + event_raw = app_data.get("event", "{}") + try: + event_data = json.loads(event_raw) + except (json.JSONDecodeError, TypeError): + continue + + if error_msg := event_data.get(error_key): + logger.error("%s on relation %s: %s", error_label, relation.id, error_msg) + return True + + return False + + +def _validate_scrape_jobs(jobs: list) -> bool: + """Validate scrape jobs using cos-tool. + + Args: + jobs: A list of Prometheus scrape job dicts to validate. + + Returns: + True if validation passed or cos-tool is unavailable. + + Raises: + subprocess.CalledProcessError: if cos-tool rejects the scrape jobs. + """ + arch = platform.machine() + arch = "amd64" if arch == "x86_64" else arch + cos_tool_path = Path("cos-tool-{}".format(arch)) + try: + cos_tool_path = cos_tool_path.resolve(strict=True) + except (FileNotFoundError, OSError): + logger.debug("cos-tool unavailable. Not validating scrape jobs.") + return True + + conf = {"scrape_configs": jobs} + with tempfile.NamedTemporaryFile(suffix=".yaml", mode="w", delete=False) as tmpfile: + tmpfile.write(yaml.safe_dump(conf)) + tmpfile_name = tmpfile.name + try: + subprocess.run( + [str(cos_tool_path), "validate-config", tmpfile_name], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + finally: + Path(tmpfile_name).unlink(missing_ok=True) + return True + def _dedupe_job_names(jobs: List[dict]): """Deduplicate a list of dicts by appending a hash to the value of the 'job_name' key. @@ -1627,6 +1731,9 @@ def __init__( if not isinstance(refresh_event, list): refresh_event = [refresh_event] + # If there is no leader during relation_joined we will still need to set alert rules. + self.framework.observe(self._charm.on.leader_elected, self.set_scrape_job_spec) + self.framework.observe(events.relation_joined, self.set_scrape_job_spec) for ev in refresh_event: self.framework.observe(ev, self.set_scrape_job_spec) @@ -1822,6 +1929,7 @@ def __init__( events.relation_changed, self._charm.on.leader_elected, self._charm.on.upgrade_charm, + self._charm.on.config_changed, ] for event_source in event_sources: @@ -1846,123 +1954,3 @@ def _update_relation_data(self, _): alert_rules_as_dict, sort_keys=True, # sort, to prevent unnecessary relation_changed events ) - -class CosTool: - """Uses cos-tool to inject label matchers into alert rule expressions and validate rules.""" - - _path = None - _disabled = False - - def __init__(self, charm): - self._charm = charm - - @property - def path(self): - """Lazy lookup of the path of cos-tool.""" - if self._disabled: - return None - if not self._path: - self._path = self._get_tool_path() - if not self._path: - logger.debug("Skipping injection of juju topology as label matchers") - self._disabled = True - return self._path - - def apply_label_matchers(self, rules) -> dict: - """Will apply label matchers to the expression of all alerts in all supplied groups.""" - if not self.path: - return rules - for group in rules["groups"]: - rules_in_group = group.get("rules", []) - for rule in rules_in_group: - topology = {} - # if the user for some reason has provided juju_unit, we'll need to honor it - # in most cases, however, this will be empty - for label in [ - "juju_model", - "juju_model_uuid", - "juju_application", - "juju_charm", - "juju_unit", - ]: - if label in rule["labels"]: - topology[label] = rule["labels"][label] - - rule["expr"] = self.inject_label_matchers(rule["expr"], topology) - return rules - - def validate_alert_rules(self, rules: dict) -> Tuple[bool, str]: - """Will validate correctness of alert rules, returning a boolean and any errors.""" - if not self.path: - logger.debug("`cos-tool` unavailable. Not validating alert correctness.") - return True, "" - - with tempfile.TemporaryDirectory() as tmpdir: - rule_path = Path(tmpdir + "/validate_rule.yaml") - rule_path.write_text(yaml.dump(rules)) - - args = [str(self.path), "validate", str(rule_path)] - # noinspection PyBroadException - try: - self._exec(args) - return True, "" - except subprocess.CalledProcessError as e: - logger.debug("Validating the rules failed: %s", e.output.decode("utf8")) - return False, ", ".join( - [ - line - for line in e.output.decode("utf8").splitlines() - if "error validating" in line - ] - ) - - def validate_scrape_jobs(self, jobs: list) -> bool: - """Validate scrape jobs using cos-tool.""" - if not self.path: - logger.debug("`cos-tool` unavailable. Not validating scrape jobs.") - return True - conf = {"scrape_configs": jobs} - with tempfile.NamedTemporaryFile() as tmpfile: - with open(tmpfile.name, "w") as f: - f.write(yaml.safe_dump(conf)) - try: - self._exec([str(self.path), "validate-config", tmpfile.name]) - except subprocess.CalledProcessError as e: - logger.error("Validating scrape jobs failed: {}".format(e.output)) - raise - return True - - def inject_label_matchers(self, expression, topology) -> str: - """Add label matchers to an expression.""" - if not topology: - return expression - if not self.path: - logger.debug("`cos-tool` unavailable. Leaving expression unchanged: %s", expression) - return expression - args = [str(self.path), "transform"] - args.extend( - ["--label-matcher={}={}".format(key, value) for key, value in topology.items()] - ) - - args.extend(["{}".format(expression)]) - # noinspection PyBroadException - try: - return self._exec(args) - except subprocess.CalledProcessError as e: - logger.debug('Applying the expression failed: "%s", falling back to the original', e) - return expression - - def _get_tool_path(self) -> Optional[Path]: - arch = platform.machine() - arch = "amd64" if arch == "x86_64" else arch - res = "cos-tool-{}".format(arch) - try: - path = Path(res).resolve(strict=True) - return path - except (FileNotFoundError, OSError): - logger.debug('Could not locate cos-tool at: "{}"'.format(res)) - return None - - def _exec(self, cmd) -> str: - result = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - return result.stdout.decode("utf-8").strip()