diff --git a/datadog_checks_dev/changelog.d/24645.added b/datadog_checks_dev/changelog.d/24645.added new file mode 100644 index 0000000000000..da197d7c5d4d8 --- /dev/null +++ b/datadog_checks_dev/changelog.d/24645.added @@ -0,0 +1,2 @@ +Support string-array inputs for local generated configuration discovery strategies. +Add Kubernetes discovery candidate-stability checks for E2E tests. diff --git a/datadog_checks_dev/datadog_checks/dev/kubernetes.py b/datadog_checks_dev/datadog_checks/dev/kubernetes.py new file mode 100644 index 0000000000000..5f7c71051674f --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/kubernetes.py @@ -0,0 +1,254 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import json +import logging +import os +import re +from collections.abc import Callable, Mapping, Sequence +from types import SimpleNamespace +from typing import Any + +from .docker import CONTAINER_STABILITY_LOG_PATTERNS +from .subprocess import run_command + + +def assert_all_discovery_candidates_stable_kubernetes( + dd_agent_check: Callable[..., Any], + check_cls: type[Any], + kubeconfig: str | os.PathLike[str], + *, + namespace: str, + pod_name: str | None = None, + pod_selector: str | None = None, + container_name: str | None = None, + service_id: str | None = None, + dd_agent_check_kwargs: Mapping[str, Any] | None = None, + log_patterns: Sequence[str] = CONTAINER_STABILITY_LOG_PATTERNS, +) -> None: + """Run generated discovery candidates and assert that the target Kubernetes pod stays stable.""" + if bool(pod_name) == bool(pod_selector): + raise TypeError('Exactly one of `pod_name` or `pod_selector` must be provided') + + if pod_selector: + initial_state = _get_selected_pod(kubeconfig, namespace, pod_selector) + pod_name = initial_state['metadata']['name'] + else: + assert pod_name is not None + initial_state = _get_pod(kubeconfig, namespace, pod_name) + + previous_logs = _get_pod_logs(kubeconfig, namespace, initial_state) + service = _build_service_from_pod(initial_state, service_id, container_name=container_name) + candidates = tuple(check_cls.generate_configs(service)) + if not candidates: + raise AssertionError(f'No discovery candidates generated for service {service.id!r}') + + check_kwargs = {'check_rate': True} + if dd_agent_check_kwargs: + check_kwargs.update(dd_agent_check_kwargs) + + for index, candidate in enumerate(candidates, 1): + logging.debug('Probing candidate #%d: %r', index, candidate) + try: + dd_agent_check(candidate, **check_kwargs) + except Exception: + # A failed check may still have contacted the workload. As with the Docker helper, + # candidate errors do not skip the workload stability assertions. + logging.debug('Error probing candidate #%d: %r', index, candidate, exc_info=True) + + if pod_selector: + current_state = _get_selected_pod(kubeconfig, namespace, pod_selector) + else: + current_state = _get_pod(kubeconfig, namespace, pod_name) + _assert_pod_stable(initial_state, current_state, index) + + current_logs = _get_pod_logs(kubeconfig, namespace, current_state) + _assert_no_new_log_patterns(previous_logs, current_logs, log_patterns, index) + previous_logs = current_logs + + +def _run_kubectl(kubeconfig: str | os.PathLike[str], args: Sequence[str], **kwargs: Any) -> Any: + return run_command(['kubectl', '--kubeconfig', os.fspath(kubeconfig), *args], **kwargs) + + +def _get_selected_pod(kubeconfig: str | os.PathLike[str], namespace: str, pod_selector: str) -> dict[str, Any]: + result = _run_kubectl( + kubeconfig, + ['get', 'pods', '--namespace', namespace, '--selector', pod_selector, '--output', 'json'], + capture='out', + check=True, + ) + pods = [pod for pod in json.loads(result.stdout)['items'] if not pod.get('metadata', {}).get('deletionTimestamp')] + if len(pods) != 1: + raise AssertionError( + f'Expected exactly one active pod in namespace {namespace!r} matching selector {pod_selector!r}, ' + f'found {len(pods)}' + ) + return pods[0] + + +def _get_pod(kubeconfig: str | os.PathLike[str], namespace: str, pod_name: str) -> dict[str, Any]: + result = _run_kubectl( + kubeconfig, + ['get', 'pod', pod_name, '--namespace', namespace, '--output', 'json'], + capture='out', + check=True, + ) + return json.loads(result.stdout) + + +def _get_pod_logs( + kubeconfig: str | os.PathLike[str], namespace: str, pod: Mapping[str, Any] +) -> dict[str, tuple[str, str]]: + pod_name = pod['metadata']['name'] + containers = [ + *pod.get('spec', {}).get('initContainers', []), + *pod.get('spec', {}).get('containers', []), + *pod.get('spec', {}).get('ephemeralContainers', []), + ] + logs = {} + for container in containers: + container_name = container['name'] + result = _run_kubectl( + kubeconfig, + ['logs', pod_name, '--namespace', namespace, '--container', container_name], + capture=True, + check=False, + ) + if result.code != 0: + raise AssertionError( + f'Could not read logs for container {container_name!r} in pod {pod_name!r}: ' + f'{result.stdout}{result.stderr}' + ) + logs[container_name] = (result.stdout, result.stderr) + return logs + + +def _build_service_from_pod( + pod: Mapping[str, Any], service_id: str | None, *, container_name: str | None = None +) -> SimpleNamespace: + host = pod.get('status', {}).get('podIP') + if not host: + raise AssertionError(f'Pod {pod.get("metadata", {}).get("name", "")!r} has no IP address') + + containers = pod.get('spec', {}).get('containers', []) + if container_name is None: + if len(containers) != 1: + raise AssertionError('`container_name` is required when the pod has more than one regular container') + container = containers[0] + else: + container = next((candidate for candidate in containers if candidate['name'] == container_name), None) + if container is None: + raise AssertionError(f'Pod has no regular container named {container_name!r}') + + ports = tuple( + sorted( + ( + SimpleNamespace(number=port_spec['containerPort'], name=port_spec.get('name', '')) + for port_spec in container.get('ports', []) + ), + key=lambda port: port.number, + ) + ) + + if service_id is None: + container_status = next( + ( + status + for status in pod.get('status', {}).get('containerStatuses', []) + if status['name'] == container['name'] + ), + None, + ) + service_id = container_status.get('containerID') if container_status else None + if not service_id: + raise AssertionError(f'Container {container["name"]!r} has no runtime ID') + + return SimpleNamespace(id=service_id, host=host, ports=ports) + + +def _assert_pod_stable(initial: Mapping[str, Any], current: Mapping[str, Any], candidate_index: int) -> None: + initial_uid = initial['metadata']['uid'] + current_uid = current['metadata']['uid'] + if current_uid != initial_uid: + raise AssertionError(f'Pod changed while probing candidate #{candidate_index}: {initial_uid} -> {current_uid}') + if current.get('metadata', {}).get('deletionTimestamp'): + raise AssertionError(f'Pod is terminating after probing candidate #{candidate_index}') + + phase = current.get('status', {}).get('phase') + if phase != 'Running': + raise AssertionError(f'Pod phase is {phase!r} after probing candidate #{candidate_index}') + + ready = next( + ( + condition.get('status') == 'True' + for condition in current.get('status', {}).get('conditions', []) + if condition.get('type') == 'Ready' + ), + False, + ) + if not ready: + raise AssertionError(f'Pod is not ready after probing candidate #{candidate_index}') + + for status_key in ('initContainerStatuses', 'containerStatuses', 'ephemeralContainerStatuses'): + initial_statuses = {status['name']: status for status in initial.get('status', {}).get(status_key, [])} + current_statuses = {status['name']: status for status in current.get('status', {}).get(status_key, [])} + if current_statuses.keys() != initial_statuses.keys(): + raise AssertionError(f'Pod containers changed while probing candidate #{candidate_index}') + + for name, current_status in current_statuses.items(): + initial_status = initial_statuses[name] + initial_container_id = initial_status.get('containerID') + current_container_id = current_status.get('containerID') + if current_container_id != initial_container_id: + raise AssertionError( + f'Container {name!r} changed while probing candidate #{candidate_index}: ' + f'{initial_container_id} -> {current_container_id}' + ) + + initial_restarts = initial_status.get('restartCount', 0) + current_restarts = current_status.get('restartCount', 0) + if current_restarts != initial_restarts: + raise AssertionError( + f'Container {name!r} restart count changed while probing candidate #{candidate_index}: ' + f'{initial_restarts} -> {current_restarts}' + ) + if status_key == 'containerStatuses' and not current_status.get('ready', False): + raise AssertionError(f'Container {name!r} is not ready after probing candidate #{candidate_index}') + + for state_key in ('state', 'lastState'): + initial_terminated = initial_status.get(state_key, {}).get('terminated') + current_terminated = current_status.get(state_key, {}).get('terminated') + if current_terminated is not None and initial_terminated is None: + reason = current_terminated.get('reason') or '' + raise AssertionError( + f'Container {name!r} terminated with reason {reason!r} ' + f'after probing candidate #{candidate_index}' + ) + + +def _assert_no_new_log_patterns( + previous: Mapping[str, tuple[str, str]], + current: Mapping[str, tuple[str, str]], + patterns: Sequence[str], + candidate_index: int, +) -> None: + if current.keys() != previous.keys(): + raise AssertionError(f'Pod log streams changed while probing candidate #{candidate_index}') + + for container_name, (current_stdout, current_stderr) in current.items(): + previous_stdout, previous_stderr = previous[container_name] + new_logs = _diff_logs(previous_stdout, current_stdout) + _diff_logs(previous_stderr, current_stderr) + for line in new_logs.splitlines(): + logging.debug('New log line from container %s: %s', container_name, line) + for pattern in patterns: + match = re.search(pattern, new_logs, re.IGNORECASE) + if match: + raise AssertionError( + f'Pod logs for container {container_name!r} matched {pattern!r} after probing ' + f'candidate #{candidate_index}: {match.group(0)!r}' + ) + + +def _diff_logs(previous: str, current: str) -> str: + return current[len(previous) :] if current.startswith(previous) else current diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/registry.py b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/registry.py index 789bc9b8269b3..f622bca71b4d2 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/registry.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/registry.py @@ -20,7 +20,7 @@ class Input: """A declared strategy input, validated against the spec stanza.""" - type: str # "array[int]" | "string" | "integer" | "boolean" + type: str # "array[int]" | "array[string]" | "string" | "integer" | "boolean" required: bool = True diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/spec.py b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/spec.py index 53c77c4a3b364..d01285ee99569 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/spec.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/spec.py @@ -236,6 +236,9 @@ def _validate_strategy_input(stanza: dict, name: str, input_def: Any, loader: An if input_def.type == 'array[int]': if not isinstance(value, list) or not all(isinstance(v, int) and not isinstance(v, bool) for v in value): loader.errors.append(f'{location}: Attribute `{name}` must be an array of integers') + elif input_def.type == 'array[string]': + if not isinstance(value, list) or not all(isinstance(v, str) for v in value): + loader.errors.append(f'{location}: Attribute `{name}` must be an array of strings') elif input_def.type == 'integer': if not isinstance(value, int) or isinstance(value, bool): loader.errors.append(f'{location}: Attribute `{name}` must be an integer') diff --git a/datadog_checks_dev/tests/test_kubernetes.py b/datadog_checks_dev/tests/test_kubernetes.py new file mode 100644 index 0000000000000..2c0d7417ecec5 --- /dev/null +++ b/datadog_checks_dev/tests/test_kubernetes.py @@ -0,0 +1,209 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import copy +import json +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from datadog_checks.dev import kubernetes + + +def _pod( + *, + uid='pod-uid', + container_id='containerd://velero-id', + restart_count=0, + ready=True, + phase='Running', + last_reason=None, +): + return { + 'metadata': {'name': 'velero-123', 'uid': uid}, + 'spec': { + 'containers': [ + { + 'name': 'velero', + 'ports': [ + {'containerPort': 8085, 'name': 'http-monitoring'}, + {'containerPort': 9999, 'name': 'admin'}, + ], + } + ] + }, + 'status': { + 'phase': phase, + 'podIP': '10.0.0.1', + 'conditions': [{'type': 'Ready', 'status': 'True' if ready else 'False'}], + 'containerStatuses': [ + { + 'name': 'velero', + 'containerID': container_id, + 'ready': ready, + 'restartCount': restart_count, + 'lastState': {'terminated': {'reason': last_reason}} if last_reason else {}, + } + ], + }, + } + + +def _result(data='', *, stderr='', code=0): + return SimpleNamespace(stdout=data, stderr=stderr, code=code) + + +def _mock_kubectl(monkeypatch, *, initial=None, current=None, initial_logs='old\n', current_logs='old\nnew\n'): + pod_states = iter((initial or _pod(), current or _pod())) + logs = iter((initial_logs, current_logs)) + + def run_command(command, **kwargs): + assert command[:3] == ['kubectl', '--kubeconfig', '/tmp/kubeconfig'] + if command[3:5] == ['get', 'pods']: + return _result(json.dumps({'items': [next(pod_states)]})) + if command[3:5] == ['get', 'pod']: + return _result(json.dumps(next(pod_states))) + if command[3] == 'logs': + return _result(next(logs)) + raise AssertionError(f'Unexpected command: {command}') + + monkeypatch.setattr(kubernetes, 'run_command', run_command) + + +class _Check: + services = [] + + @classmethod + def generate_configs(cls, service): + cls.services.append(service) + return [ + { + 'init_config': {}, + 'instances': [{'openmetrics_endpoint': f'http://{service.host}:{port.number}/metrics'}], + } + for port in service.ports + if port.name == 'http-monitoring' + ] + + +def test_assert_all_discovery_candidates_stable_kubernetes(monkeypatch): + _mock_kubectl(monkeypatch) + dd_agent_check = Mock() + _Check.services = [] + + kubernetes.assert_all_discovery_candidates_stable_kubernetes( + dd_agent_check, + _Check, + '/tmp/kubeconfig', + namespace='velero', + pod_selector='name=velero', + ) + + service = _Check.services[0] + assert service.id == 'containerd://velero-id' + assert service.host == '10.0.0.1' + assert [(port.number, port.name) for port in service.ports] == [ + (8085, 'http-monitoring'), + (9999, 'admin'), + ] + dd_agent_check.assert_called_once_with( + { + 'init_config': {}, + 'instances': [{'openmetrics_endpoint': 'http://10.0.0.1:8085/metrics'}], + }, + check_rate=True, + ) + + +def test_candidate_error_still_checks_workload_stability(monkeypatch): + _mock_kubectl(monkeypatch, current=_pod(restart_count=1)) + dd_agent_check = Mock(side_effect=RuntimeError('check failed')) + + with pytest.raises(AssertionError, match='restart count changed'): + kubernetes.assert_all_discovery_candidates_stable_kubernetes( + dd_agent_check, + _Check, + '/tmp/kubeconfig', + namespace='velero', + pod_name='velero-123', + ) + + +def test_new_dangerous_workload_logs_fail(monkeypatch): + _mock_kubectl(monkeypatch, current_logs='old\npanic: crashed\n') + + with pytest.raises(AssertionError, match="Pod logs for container 'velero' matched 'panic'"): + kubernetes.assert_all_discovery_candidates_stable_kubernetes( + Mock(), + _Check, + '/tmp/kubeconfig', + namespace='velero', + pod_name='velero-123', + ) + + +def test_selector_detects_replacement_pod(monkeypatch): + _mock_kubectl(monkeypatch, current=_pod(uid='replacement-uid')) + + with pytest.raises(AssertionError, match='Pod changed'): + kubernetes.assert_all_discovery_candidates_stable_kubernetes( + Mock(), + _Check, + '/tmp/kubeconfig', + namespace='velero', + pod_selector='name=velero', + ) + + +def test_service_uses_only_target_container_and_preserves_named_ports(): + pod = _pod() + pod['spec']['containers'][0]['ports'].append({'containerPort': 8085, 'name': 'metrics'}) + pod['spec']['containers'].append({'name': 'sidecar', 'ports': [{'containerPort': 9000, 'name': 'sidecar-metrics'}]}) + + with pytest.raises(AssertionError, match='container_name'): + kubernetes._build_service_from_pod(pod, 'svc') + + service = kubernetes._build_service_from_pod(pod, 'svc', container_name='velero') + assert [(port.number, port.name) for port in service.ports] == [ + (8085, 'http-monitoring'), + (8085, 'metrics'), + (9999, 'admin'), + ] + + +def test_termination_without_reason_is_detected(): + initial = _pod() + current = copy.deepcopy(initial) + current['status']['containerStatuses'][0]['state'] = {'terminated': {'exitCode': 1}} + + with pytest.raises(AssertionError, match="terminated with reason ''"): + kubernetes._assert_pod_stable(initial, current, 1) + + +def test_log_streams_are_diffed_independently(): + kubernetes._assert_no_new_log_patterns( + {'velero': ('old error\n', 'old warning\n')}, + {'velero': ('old error\n', 'old warning\nnew healthy line\n')}, + ('error',), + 1, + ) + + +@pytest.mark.parametrize( + ('pod_name', 'pod_selector'), + [ + (None, None), + ('velero-123', 'name=velero'), + ], +) +def test_exactly_one_pod_identifier_is_required(pod_name, pod_selector): + with pytest.raises(TypeError, match='Exactly one'): + kubernetes.assert_all_discovery_candidates_stable_kubernetes( + Mock(), + _Check, + '/tmp/kubeconfig', + namespace='velero', + pod_name=pod_name, + pod_selector=pod_selector, + ) diff --git a/datadog_checks_dev/tests/tooling/configuration/test_load.py b/datadog_checks_dev/tests/tooling/configuration/test_load.py index cd0380f85027d..ce21ea9f19764 100644 --- a/datadog_checks_dev/tests/tooling/configuration/test_load.py +++ b/datadog_checks_dev/tests/tooling/configuration/test_load.py @@ -361,6 +361,60 @@ def test_discovery_rejects_boolean_port_hints(): assert 'test, test.yaml, discovery, strategy #1: Attribute `port_hints` must be an array of integers' in spec.errors +def test_discovery_named_ports_valid(): + spec = get_spec( + """ + version: 0.0.0 + files: + - name: test.yaml + example_name: test.yaml.example + discovery: + strategies: + - strategy: local:from_named_ports + provides: [port] + inputs: {port_names: 'array[string]'} + port_names: + - metrics + - http-monitoring + candidates: + - openmetrics_endpoint: http://{service.host}:{port.number}/metrics + options: + - template: init_config + - template: instances + """ + ) + spec.load() + + assert not spec.errors + + +def test_discovery_rejects_non_string_port_names(): + spec = get_spec( + """ + version: 0.0.0 + files: + - name: test.yaml + example_name: test.yaml.example + discovery: + strategies: + - strategy: local:from_named_ports + provides: [port] + inputs: {port_names: 'array[string]'} + port_names: + - metrics + - 8080 + candidates: + - openmetrics_endpoint: http://{service.host}:{port.number}/metrics + options: + - template: init_config + - template: instances + """ + ) + spec.load() + + assert 'test, test.yaml, discovery, strategy #1: Attribute `port_names` must be an array of strings' in spec.errors + + def test_discovery_unsupported_strategy(): spec = get_spec( """ diff --git a/velero/assets/configuration/spec.yaml b/velero/assets/configuration/spec.yaml index 8427942921816..139f5273f47a5 100644 --- a/velero/assets/configuration/spec.yaml +++ b/velero/assets/configuration/spec.yaml @@ -2,6 +2,16 @@ name: Velero fleet_configurable: true files: - name: velero.yaml + discovery: + strategies: + - strategy: local:from_named_ports + provides: [port] + inputs: {port_names: 'array[string]'} + port_names: + - http-monitoring + - metrics + candidates: + - openmetrics_endpoint: http://{service.host}:{port.number}/metrics options: - template: init_config options: @@ -14,3 +24,10 @@ files: - type: docker source: velero service: +- name: auto_conf.yaml + options: + - template: ad_identifiers + overrides: + value.example: + - velero + - template: auto_conf/discovery diff --git a/velero/changelog.d/24645.added b/velero/changelog.d/24645.added new file mode 100644 index 0000000000000..1455667b0b73f --- /dev/null +++ b/velero/changelog.d/24645.added @@ -0,0 +1 @@ +Add container-based config discovery support. diff --git a/velero/datadog_checks/velero/config_models/discovery.py b/velero/datadog_checks/velero/config_models/discovery.py new file mode 100644 index 0000000000000..213cadfb6d037 --- /dev/null +++ b/velero/datadog_checks/velero/config_models/discovery.py @@ -0,0 +1,42 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Any + +from datadog_checks.base.utils.discovery import Service +from datadog_checks.velero.config_models import discovery_overrides +from datadog_checks.velero.config_models.discovery_strategies import from_named_ports +from datadog_checks.velero.config_models.instance import InstanceConfig +from datadog_checks.velero.config_models.shared import SharedConfig + + +def _generated_candidates(service: Service) -> Iterator[dict[str, Any]]: + shared = SharedConfig.model_validate({}, context={'configured_fields': frozenset()}).model_dump( + by_alias=True, mode='json', exclude_none=True + ) + # discovery[0]: local:from_named_ports + for ctx in from_named_ports(service, port_names=['http-monitoring', 'metrics']): + instance_data = { + 'openmetrics_endpoint': 'http://{service.host}:{port.number}/metrics'.format(service=service, **ctx), + } + instance = InstanceConfig.model_validate( + instance_data, context={'configured_fields': frozenset(instance_data)} + ).model_dump(by_alias=True, mode='json', exclude_none=True) + yield {'init_config': shared, 'instances': [instance]} + + +def candidates(service: Service) -> Iterator[dict[str, Any]]: + override = getattr(discovery_overrides, 'candidates', None) + if override is None: + yield from _generated_candidates(service) + else: + yield from override(service, default=_generated_candidates) diff --git a/velero/datadog_checks/velero/config_models/discovery_overrides.py b/velero/datadog_checks/velero/config_models/discovery_overrides.py new file mode 100644 index 0000000000000..66af68809dd4c --- /dev/null +++ b/velero/datadog_checks/velero/config_models/discovery_overrides.py @@ -0,0 +1,12 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# Override the generated discovery candidates() for this integration. +# +# Define a candidates(service, default) function to wrap or replace the generated +# candidate generation. `default` is the generated generator; call it to reuse +# the spec-driven candidates, or ignore it to replace them entirely. +# +# def candidates(service, default): +# yield from default(service) diff --git a/velero/datadog_checks/velero/config_models/discovery_strategies.py b/velero/datadog_checks/velero/config_models/discovery_strategies.py new file mode 100644 index 0000000000000..1ad671db44e24 --- /dev/null +++ b/velero/datadog_checks/velero/config_models/discovery_strategies.py @@ -0,0 +1,24 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +from __future__ import annotations + +from collections.abc import Iterable, Iterator +from typing import Any + +from datadog_checks.base.utils.discovery import Service, discovery_strategy + + +@discovery_strategy(provides=('port',)) +def from_named_ports(service: Service, port_names: Iterable[str]) -> Iterator[dict[str, Any]]: + """Yield named service ports in requested order without duplicate port numbers.""" + seen: set[int] = set() + + for name in dict.fromkeys(port_names): + if not name: + continue + for port in service.ports: + if port.name == name and port.number not in seen: + seen.add(port.number) + yield {'port': port} diff --git a/velero/datadog_checks/velero/data/auto_conf.yaml b/velero/datadog_checks/velero/data/auto_conf.yaml new file mode 100644 index 0000000000000..5c7281d343166 --- /dev/null +++ b/velero/datadog_checks/velero/data/auto_conf.yaml @@ -0,0 +1,19 @@ +## @param ad_identifiers - list of strings - required +## A list of container identifiers that are used by Autodiscovery to identify +## which container the check should be run against. For more information, see: +## https://docs.datadoghq.com/agent/guide/ad_identifiers/ +# +ad_identifiers: + - velero + +## Enables configuration discovery +# +discovery: {} + +## Unused init configuration +# +init_config: + +## Unused instance configuration +# +instances: [] diff --git a/velero/pyproject.toml b/velero/pyproject.toml index ad4519b9589a6..ec0dfe1974b24 100644 --- a/velero/pyproject.toml +++ b/velero/pyproject.toml @@ -29,7 +29,7 @@ classifiers = [ "Topic :: System :: Monitoring", ] dependencies = [ - "datadog-checks-base>=37.33.0", + "datadog-checks-base>=37.41.0", ] dynamic = [ "version", diff --git a/velero/tests/conftest.py b/velero/tests/conftest.py index 9d6aaf6bd3a64..67c49444281dd 100644 --- a/velero/tests/conftest.py +++ b/velero/tests/conftest.py @@ -1,20 +1,25 @@ # (C) Datadog, Inc. 2025-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import json import os -from contextlib import ExitStack, contextmanager +from contextlib import contextmanager import pytest from datadog_checks.dev import TempDir, run_command +from datadog_checks.dev._env import get_state, save_state from datadog_checks.dev.fs import path_join from datadog_checks.dev.kind import KindLoad, kind_run -from datadog_checks.dev.kube_port_forward import port_forward from .common import MOCKED_INSTANCE, PORT HERE = os.path.dirname(os.path.abspath(__file__)) +CHECK_ROOT = os.path.dirname(HERE) KIND_DIR = os.path.join(HERE, 'kind') +KUBECONFIG_STATE = 'velero_kubeconfig' +NODE_AGENT_IP_STATE = 'velero_node_agent_ip' +NODE_AGENT_NAME_STATE = 'velero_node_agent_name' @contextmanager @@ -56,17 +61,39 @@ def setup_velero(): ], check=True, ) + node_agent = get_node_agent() + save_state(NODE_AGENT_IP_STATE, node_agent['status']['podIP']) + save_state(NODE_AGENT_NAME_STATE, node_agent['metadata']['name']) -def get_instances(velero_host, velero_port, node_agent_host, node_agent_port): +def get_node_agent(): + result = run_command( + ['kubectl', 'get', 'pods', '--namespace', 'velero', '--output', 'json'], + capture='out', + check=True, + ) + node_agent_pods = [ + pod + for pod in json.loads(result.stdout)['items'] + if any( + owner.get('kind') == 'DaemonSet' and owner.get('name') == 'node-agent' + for owner in pod['metadata'].get('ownerReferences', []) + ) + ] + if len(node_agent_pods) != 1 or not node_agent_pods[0].get('status', {}).get('podIP'): + raise RuntimeError(f'Expected one ready Velero node-agent pod, found {len(node_agent_pods)}') + return node_agent_pods[0] + + +def get_instances(node_agent_ip): return { 'instances': [ { - 'openmetrics_endpoint': f"http://{velero_host}:{velero_port}/metrics", + 'openmetrics_endpoint': f"http://velero.velero.svc.cluster.local:{PORT}/metrics", 'tags': ['test:tag'], }, { - 'openmetrics_endpoint': f"http://{node_agent_host}:{node_agent_port}/metrics", + 'openmetrics_endpoint': f"http://{node_agent_ip}:{PORT}/metrics", 'tags': ['test:tag'], }, ] @@ -75,28 +102,38 @@ def get_instances(velero_host, velero_port, node_agent_host, node_agent_port): @pytest.fixture(scope='session') def dd_environment(): - kind_config = os.path.join(KIND_DIR, 'kind-config.yaml') custom_kubectl_image_tag = "custom-kubectl:latest" with TempDir('helm_dir') as helm_dir: with kind_run( wrappers=[build_and_load_kubectl_image(custom_kubectl_image_tag)], conditions=[KindLoad(custom_kubectl_image_tag), setup_velero], - kind_config=kind_config, env_vars={ "HELM_CACHE_HOME": path_join(helm_dir, 'Caches'), "HELM_CONFIG_HOME": path_join(helm_dir, 'Preferences'), }, ) as kubeconfig: - with ExitStack() as stack: - ip_ports = [ - stack.enter_context(port_forward(kubeconfig, 'velero', PORT, ressource, name)) - for ressource, name in [('service', 'velero'), ('daemonset', 'node-agent')] - ] + save_state(KUBECONFIG_STATE, kubeconfig) + instances = get_instances(get_state(NODE_AGENT_IP_STATE)) + metadata = { + 'agent_type': 'kubernetes', + 'kubernetes': { + 'kubeconfig': kubeconfig, + 'auto_conf': os.path.join(CHECK_ROOT, 'datadog_checks', 'velero', 'data', 'auto_conf.yaml'), + }, + } - instances = get_instances(ip_ports[0][0], ip_ports[0][1], ip_ports[1][0], ip_ports[1][1]) + yield instances, metadata - yield instances + +@pytest.fixture(scope='session') +def velero_kubeconfig(): + return get_state(KUBECONFIG_STATE) + + +@pytest.fixture(scope='session') +def velero_node_agent_name(): + return get_state(NODE_AGENT_NAME_STATE) @pytest.fixture diff --git a/velero/tests/kind/kind-config.yaml b/velero/tests/kind/kind-config.yaml deleted file mode 100644 index 93ad325fe45a3..0000000000000 --- a/velero/tests/kind/kind-config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: -- role: control-plane - extraPortMappings: - - containerPort: 8085 - hostPort: 8085 \ No newline at end of file diff --git a/velero/tests/test_discovery.py b/velero/tests/test_discovery.py new file mode 100644 index 0000000000000..b0db30795eeaf --- /dev/null +++ b/velero/tests/test_discovery.py @@ -0,0 +1,24 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from datadog_checks.base.utils.discovery import Port, Service +from datadog_checks.velero import VeleroCheck + + +def test_generated_discovery_uses_named_metrics_ports(): + service = Service( + id='velero', + host='10.0.0.1', + ports=( + Port(number=8085, name='metrics'), + Port(number=9999, name='admin'), + Port(number=8086, name='http-monitoring'), + ), + ) + + configs = list(VeleroCheck.generate_configs(service)) + + assert [config['instances'][0]['openmetrics_endpoint'] for config in configs] == [ + 'http://10.0.0.1:8086/metrics', + 'http://10.0.0.1:8085/metrics', + ] diff --git a/velero/tests/test_e2e.py b/velero/tests/test_e2e.py index 8067bb6f0e82a..cad44bd633c8a 100644 --- a/velero/tests/test_e2e.py +++ b/velero/tests/test_e2e.py @@ -4,14 +4,27 @@ import pytest from datadog_checks.base.constants import ServiceCheck +from datadog_checks.dev.docker import CONTAINER_STABILITY_LOG_PATTERNS +from datadog_checks.dev.kubernetes import assert_all_discovery_candidates_stable_kubernetes +from datadog_checks.velero import VeleroCheck from .common import OPTIONAL_METRICS, TEST_METRICS +BENIGN_DISCOVERY_ERROR_LOG_PATTERNS = ( + r'BackupStorageLocation is in unavailable state, skip syncing backup from it', + r'Current BackupStorageLocations available/unavailable/unknown:', +) -@pytest.mark.e2e -def test_check_velero_e2e(dd_agent_check): - aggregator = dd_agent_check(rate=True) +# Velero can log backup-storage-location state transitions at error level while its controllers settle. +DISCOVERY_STABILITY_LOG_PATTERNS = tuple( + r'(?m)^(?![^\n]*(?:{}))[^\n]*error'.format('|'.join(BENIGN_DISCOVERY_ERROR_LOG_PATTERNS)) + if pattern == 'error' + else pattern + for pattern in CONTAINER_STABILITY_LOG_PATTERNS +) + +def assert_metrics(aggregator): for metric, _ in TEST_METRICS.items(): if metric in OPTIONAL_METRICS: aggregator.assert_metric(name=metric, at_least=0) @@ -19,3 +32,32 @@ def test_check_velero_e2e(dd_agent_check): aggregator.assert_metric(name=metric, at_least=1) aggregator.assert_service_check('velero.openmetrics.health', ServiceCheck.OK) + + +@pytest.mark.e2e +def test_check_velero_e2e(dd_agent_check): + assert_metrics(dd_agent_check(rate=True)) + + +@pytest.mark.e2e +def test_e2e_discovery(dd_agent_check_discovery): + # The Velero server and node-agent pods produce one discovered instance each. + aggregator = dd_agent_check_discovery(check_rate=True, discovery_min_instances=2) + assert_metrics(aggregator) + + +@pytest.mark.e2e +def test_e2e_discovery_all_candidates(dd_agent_check, velero_kubeconfig, velero_node_agent_name): + targets = ( + {'pod_selector': 'name=velero'}, + {'pod_name': velero_node_agent_name}, + ) + for target in targets: + assert_all_discovery_candidates_stable_kubernetes( + dd_agent_check, + VeleroCheck, + velero_kubeconfig, + namespace='velero', + log_patterns=DISCOVERY_STABILITY_LOG_PATTERNS, + **target, + )