From 6b6177153b1f58210a602117b7349b16205314a0 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Wed, 22 Jul 2026 07:28:10 +0000 Subject: [PATCH 01/50] Add Kubernetes Agent E2E backend --- datadog_checks_dev/changelog.d/24507.added | 1 + .../datadog_checks/dev/plugin/pytest.py | 5 +- .../discovery/core_strategies.py | 22 + .../configuration/discovery/registry.py | 2 +- .../dev/tooling/configuration/spec.py | 3 + .../openmetrics_from_named_ports.yaml | 5 + .../consumers/model/test_discovery.py | 82 +++ .../tests/tooling/configuration/test_load.py | 50 ++ ddev/changelog.d/24507.added | 1 + ddev/src/ddev/cli/env/__init__.py | 2 + ddev/src/ddev/cli/env/agent.py | 2 + ddev/src/ddev/cli/env/logs.py | 37 ++ ddev/src/ddev/cli/env/start.py | 17 +- ddev/src/ddev/cli/env/stop.py | 6 +- ddev/src/ddev/e2e/agent/__init__.py | 5 + ddev/src/ddev/e2e/agent/docker.py | 3 + ddev/src/ddev/e2e/agent/interface.py | 7 + ddev/src/ddev/e2e/agent/kubernetes.py | 545 ++++++++++++++++++ ddev/tests/cli/env/test_agent.py | 56 ++ ddev/tests/cli/env/test_logs.py | 52 ++ ddev/tests/cli/env/test_start.py | 25 + ddev/tests/cli/env/test_stop.py | 20 + ddev/tests/e2e/agent/test_kubernetes.py | 504 ++++++++++++++++ docs/developer/ddev/plugins.md | 7 +- docs/developer/ddev/test.md | 36 ++ velero/assets/configuration/spec.yaml | 14 + velero/changelog.d/24507.added | 1 + .../velero/config_models/discovery.py | 52 ++ .../config_models/discovery_overrides.py | 12 + .../config_models/discovery_strategies.py | 18 + .../datadog_checks/velero/data/auto_conf.yaml | 19 + velero/pyproject.toml | 2 +- velero/tests/conftest.py | 52 +- velero/tests/test_e2e.py | 16 +- 34 files changed, 1652 insertions(+), 29 deletions(-) create mode 100644 datadog_checks_dev/changelog.d/24507.added create mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/discovery/openmetrics_from_named_ports.yaml create mode 100644 ddev/changelog.d/24507.added create mode 100644 ddev/src/ddev/cli/env/logs.py create mode 100644 ddev/src/ddev/e2e/agent/kubernetes.py create mode 100644 ddev/tests/cli/env/test_logs.py create mode 100644 ddev/tests/e2e/agent/test_kubernetes.py create mode 100644 velero/changelog.d/24507.added create mode 100644 velero/datadog_checks/velero/config_models/discovery.py create mode 100644 velero/datadog_checks/velero/config_models/discovery_overrides.py create mode 100644 velero/datadog_checks/velero/config_models/discovery_strategies.py create mode 100644 velero/datadog_checks/velero/data/auto_conf.yaml diff --git a/datadog_checks_dev/changelog.d/24507.added b/datadog_checks_dev/changelog.d/24507.added new file mode 100644 index 0000000000000..30b10aaabda4b --- /dev/null +++ b/datadog_checks_dev/changelog.d/24507.added @@ -0,0 +1 @@ +Add a Kubernetes Agent backend for E2E tests and a generated configuration strategy for Kubernetes named ports. diff --git a/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py b/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py index 5ad3e146c9234..9980634edc1d2 100644 --- a/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py +++ b/datadog_checks_dev/datadog_checks/dev/plugin/pytest.py @@ -219,7 +219,10 @@ def run_check(config=None, **kwargs): if not matches: message_parts = [] - debug_result = run_command(['docker', 'logs', 'dd_{}_{}'.format(check, env)], capture=True) + debug_result = run_command( + [python_path, '-m', 'ddev', 'env', 'logs', check, env], + capture=True, + ) if not debug_result.code: message_parts.append(debug_result.stdout + debug_result.stderr) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/core_strategies.py b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/core_strategies.py index 7ebd65174c528..c7b2cc24d9ecf 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/core_strategies.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/core_strategies.py @@ -21,3 +21,25 @@ def from_ports(stanza: dict[str, Any]) -> list[str]: f' for port in candidate_ports(service, {port_hints!r}):', " ctx = {'port': port}", ] + + +@strategy( + 'from_named_ports', + provides=('port',), + inputs={'port_names': Input('array[string]')}, +) +def from_named_ports(stanza: dict[str, Any]) -> list[str]: + """Open a loop over explicitly named service ports and bind a `ctx` exposing the current `port`.""" + port_names = stanza['port_names'] + # Keep generated integrations compatible with the base version that first + # introduced discovery while preserving the requested name order and + # deduplicating ports by number. + return [ + ' seen_ports = set()', + f' for port in (port for port_name in dict.fromkeys({port_names!r}) if port_name ' + 'for port in service.ports if port.name == port_name):', + ' if port.number in seen_ports:', + ' continue', + ' seen_ports.add(port.number)', + " ctx = {'port': port}", + ] diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/registry.py b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/registry.py index 789bc9b8269b3..f622bca71b4d2 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/registry.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/registry.py @@ -20,7 +20,7 @@ class Input: """A declared strategy input, validated against the spec stanza.""" - type: str # "array[int]" | "string" | "integer" | "boolean" + type: str # "array[int]" | "array[string]" | "string" | "integer" | "boolean" required: bool = True diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/spec.py b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/spec.py index d92af56b06abc..f02a690a3d1ee 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/spec.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/spec.py @@ -236,6 +236,9 @@ def _validate_strategy_input(stanza: dict, name: str, input_def: Any, loader: An if input_def.type == 'array[int]': if not isinstance(value, list) or not all(isinstance(v, int) and not isinstance(v, bool) for v in value): loader.errors.append(f'{location}: Attribute `{name}` must be an array of integers') + elif input_def.type == 'array[string]': + if not isinstance(value, list) or not all(isinstance(v, str) for v in value): + loader.errors.append(f'{location}: Attribute `{name}` must be an array of strings') elif input_def.type == 'integer': if not isinstance(value, int) or isinstance(value, bool): loader.errors.append(f'{location}: Attribute `{name}` must be an integer') diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/discovery/openmetrics_from_named_ports.yaml b/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/discovery/openmetrics_from_named_ports.yaml new file mode 100644 index 0000000000000..17ce4376de88c --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/discovery/openmetrics_from_named_ports.yaml @@ -0,0 +1,5 @@ +strategy: from_named_ports +port_names: +- metrics +candidates: +- openmetrics_endpoint: "http://{service.host}:{port.number}/metrics" diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py index f75b4fb19eda7..c659a3010631b 100644 --- a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py @@ -152,6 +152,88 @@ def candidates(service: Service) -> Iterator[dict[str, Any]]: ) +def test_from_named_ports(): + consumer = get_model_consumer( + """ + name: test + version: 0.0.0 + files: + - name: test.yaml + discovery: + strategies: + - strategy: from_named_ports + port_names: + - metrics + - http-monitoring + candidates: + - endpoint: http://{service.host}:{port.number}/m + options: + - template: init_config + options: [] + - template: instances + options: + - name: endpoint + description: words + required: true + value: + type: string + """ + ) + + model_definitions = consumer.render() + files = model_definitions['test.yaml'] + + discovery_contents, discovery_errors = files['discovery.py'] + assert not discovery_errors + assert discovery_contents == normalize_yaml( + """ + from __future__ import annotations + + from collections.abc import Iterator + from typing import Any + + from datadog_checks.base.utils.discovery import Service + from datadog_checks.test.config_models import discovery_overrides + from datadog_checks.test.config_models.instance import InstanceConfig + from datadog_checks.test.config_models.shared import SharedConfig + + + def _generated_candidates(service: Service) -> Iterator[dict[str, Any]]: + shared = SharedConfig.model_validate({}, context={'configured_fields': frozenset()}).model_dump( + by_alias=True, mode='json', exclude_none=True + ) + # discovery[0]: from_named_ports + seen_ports = set() + for port in ( + port + for port_name in dict.fromkeys(['metrics', 'http-monitoring']) + if port_name + for port in service.ports + if port.name == port_name + ): + if port.number in seen_ports: + continue + seen_ports.add(port.number) + ctx = {'port': port} + instance_data = { + 'endpoint': 'http://{service.host}:{port.number}/m'.format(service=service, **ctx), + } + instance = InstanceConfig.model_validate( + instance_data, context={'configured_fields': frozenset(instance_data)} + ).model_dump(by_alias=True, mode='json', exclude_none=True) + yield {'init_config': shared, 'instances': [instance]} + + + def candidates(service: Service) -> Iterator[dict[str, Any]]: + override = getattr(discovery_overrides, 'candidates', None) + if override is None: + yield from _generated_candidates(service) + else: + yield from override(service, default=_generated_candidates) + """ + ) + + def test_local_strategy(): consumer = get_model_consumer( """ diff --git a/datadog_checks_dev/tests/tooling/configuration/test_load.py b/datadog_checks_dev/tests/tooling/configuration/test_load.py index f00072d36bc87..d355de42ad416 100644 --- a/datadog_checks_dev/tests/tooling/configuration/test_load.py +++ b/datadog_checks_dev/tests/tooling/configuration/test_load.py @@ -361,6 +361,56 @@ def test_discovery_rejects_boolean_port_hints(): assert 'test, test.yaml, discovery, strategy #1: Attribute `port_hints` must be an array of integers' in spec.errors +def test_discovery_named_ports_valid(): + spec = get_spec( + """ + version: 0.0.0 + files: + - name: test.yaml + example_name: test.yaml.example + discovery: + strategies: + - strategy: from_named_ports + port_names: + - metrics + - http-monitoring + candidates: + - openmetrics_endpoint: http://{service.host}:{port.number}/metrics + options: + - template: init_config + - template: instances + """ + ) + spec.load() + + assert not spec.errors + + +def test_discovery_rejects_non_string_port_names(): + spec = get_spec( + """ + version: 0.0.0 + files: + - name: test.yaml + example_name: test.yaml.example + discovery: + strategies: + - strategy: from_named_ports + port_names: + - metrics + - 8080 + candidates: + - openmetrics_endpoint: http://{service.host}:{port.number}/metrics + options: + - template: init_config + - template: instances + """ + ) + spec.load() + + assert 'test, test.yaml, discovery, strategy #1: Attribute `port_names` must be an array of strings' in spec.errors + + def test_discovery_unsupported_strategy(): spec = get_spec( """ diff --git a/ddev/changelog.d/24507.added b/ddev/changelog.d/24507.added new file mode 100644 index 0000000000000..41944a4c74b1b --- /dev/null +++ b/ddev/changelog.d/24507.added @@ -0,0 +1 @@ +Add a Kubernetes Agent backend for running integration E2E checks and Autodiscovery tests inside a cluster. diff --git a/ddev/src/ddev/cli/env/__init__.py b/ddev/src/ddev/cli/env/__init__.py index 6f38d7bf9730c..769a4079ef916 100644 --- a/ddev/src/ddev/cli/env/__init__.py +++ b/ddev/src/ddev/cli/env/__init__.py @@ -6,6 +6,7 @@ from ddev.cli.env.agent import agent from ddev.cli.env.check import check from ddev.cli.env.config import config +from ddev.cli.env.logs import logs from ddev.cli.env.reload import reload_command from ddev.cli.env.shell import shell from ddev.cli.env.show import show @@ -24,6 +25,7 @@ def env(): env.add_command(agent) env.add_command(check) env.add_command(config) +env.add_command(logs) env.add_command(reload_command) env.add_command(shell) env.add_command(show) diff --git a/ddev/src/ddev/cli/env/agent.py b/ddev/src/ddev/cli/env/agent.py index d67192f00e03e..159bc2579990b 100644 --- a/ddev/src/ddev/cli/env/agent.py +++ b/ddev/src/ddev/cli/env/agent.py @@ -131,6 +131,7 @@ def agent( app.abort(str(e)) finally: env_data.config_file.unlink() + agent.sync_config() else: temp_config_file = env_data.config_file.parent / f'{env_data.config_file.name}.bak.example' env_data.config_file.replace(temp_config_file) @@ -141,3 +142,4 @@ def agent( app.abort(str(e)) finally: temp_config_file.replace(env_data.config_file) + agent.sync_config() diff --git a/ddev/src/ddev/cli/env/logs.py b/ddev/src/ddev/cli/env/logs.py new file mode 100644 index 0000000000000..052d37ef8be23 --- /dev/null +++ b/ddev/src/ddev/cli/env/logs.py @@ -0,0 +1,37 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +from typing import TYPE_CHECKING + +import click + +if TYPE_CHECKING: + from ddev.cli.application import Application + + +@click.command('logs', short_help='Show logs for the Agent') +@click.argument('intg_name', metavar='INTEGRATION') +@click.argument('environment') +@click.pass_obj +def logs(app: Application, *, intg_name: str, environment: str): + """Show backend-specific diagnostics for the Agent.""" + from ddev.e2e.agent import get_agent_interface + from ddev.e2e.config import EnvDataStorage + from ddev.e2e.constants import DEFAULT_AGENT_TYPE, E2EMetadata + + integration = app.repo.integrations.get(intg_name) + env_data = EnvDataStorage(app.data_dir).get(integration.name, environment) + + if not env_data.exists(): + app.abort(f'Environment `{environment}` for integration `{integration.name}` is not running') + + metadata = env_data.read_metadata() + agent_type = metadata.get(E2EMetadata.AGENT_TYPE, DEFAULT_AGENT_TYPE) + agent = get_agent_interface(agent_type)(app, integration, environment, metadata, env_data.config_file) + + try: + agent.show_logs() + except Exception as e: + app.abort(str(e)) diff --git a/ddev/src/ddev/cli/env/start.py b/ddev/src/ddev/cli/env/start.py index fd0c1577c8cfb..f86ff42537daa 100644 --- a/ddev/src/ddev/cli/env/start.py +++ b/ddev/src/ddev/cli/env/start.py @@ -132,6 +132,13 @@ def start( result = json.loads(result_file.read_text()) metadata = result['metadata'] + agent_type = metadata.get(E2EMetadata.AGENT_TYPE, DEFAULT_AGENT_TYPE) + if agent_type == 'kubernetes': + from secrets import token_hex + + # Persist a unique owner before backend startup so error cleanup cannot + # delete resources belonging to another environment with the same name. + metadata['_kubernetes_owner_id'] = token_hex(16) # TODO Remove once we have migrated the `docker_run` function if serialized_volumes := metadata.get(E2EMetadata.ENV_VARS, {}).get(E2EEnvVars.DOCKER_VOLUMES): @@ -144,17 +151,19 @@ def start( config = result['config'] env_data.write_config(config) - agent_type = metadata.get(E2EMetadata.AGENT_TYPE, DEFAULT_AGENT_TYPE) - if agent_type == "vagrant" and running_on_ci(): app.abort(text="Vagrant is not supported on CI", code=0) agent = get_agent_interface(agent_type)(app, integration, environment, metadata, env_data.config_file) if not agent_build: + configured_agent_build = app.config.agent.config.get(agent_type) + if configured_agent_build is None and agent_type == 'kubernetes': + # Kubernetes uses the same container image as the Docker backend. + configured_agent_build = app.config.agent.config.get('docker') agent_build = ( os.getenv(E2EEnvVars.AGENT_BUILD_PY2 if agent.python_version[0] == 2 else E2EEnvVars.AGENT_BUILD) - or app.config.agent.config.get(agent_type) + or configured_agent_build or '' ) @@ -162,6 +171,8 @@ def start( try: agent.start(agent_build=agent_build, local_packages=local_packages, env_vars=agent_env_vars) + # Backends may add runtime metadata needed by later ddev processes. + env_data.write_metadata(metadata) except Exception as e: from ddev.cli.env.stop import stop diff --git a/ddev/src/ddev/cli/env/stop.py b/ddev/src/ddev/cli/env/stop.py index 62e8e3a19266f..e20b33f74f88d 100644 --- a/ddev/src/ddev/cli/env/stop.py +++ b/ddev/src/ddev/cli/env/stop.py @@ -69,9 +69,11 @@ def stop(app: Application, *, intg_name: str, environment: str, ignore_state: bo try: agent.stop() finally: - env_data.remove() - with integration.path.as_cwd(env_vars=env_vars), runner.stop() as command: process = app.platform.run_command(command) if process.returncode: app.abort(code=process.returncode) + + # Keep the owner metadata when either cleanup stage fails so a + # later `ddev env stop` can safely retry resource deletion. + env_data.remove() diff --git a/ddev/src/ddev/e2e/agent/__init__.py b/ddev/src/ddev/e2e/agent/__init__.py index f0dcfe5a29556..a7a04fd0b7f09 100644 --- a/ddev/src/ddev/e2e/agent/__init__.py +++ b/ddev/src/ddev/e2e/agent/__init__.py @@ -20,4 +20,9 @@ def get_agent_interface(agent_type: str) -> type[AgentInterface]: return VagrantAgent + if agent_type == "kubernetes": + from ddev.e2e.agent.kubernetes import KubernetesAgent + + return KubernetesAgent + raise NotImplementedError(f"Unsupported Agent type: {agent_type}") diff --git a/ddev/src/ddev/e2e/agent/docker.py b/ddev/src/ddev/e2e/agent/docker.py index a362b0f92f19d..37a90fc7dacde 100644 --- a/ddev/src/ddev/e2e/agent/docker.py +++ b/ddev/src/ddev/e2e/agent/docker.py @@ -176,6 +176,9 @@ def _execute_lifecycle_command( def _show_logs(self) -> None: self._run_command(['docker', 'logs', self._container_name]) + def show_logs(self) -> None: + self._run_command(['docker', 'logs', self._container_name], check=True) + def get_id(self) -> str: return self._container_name diff --git a/ddev/src/ddev/e2e/agent/interface.py b/ddev/src/ddev/e2e/agent/interface.py index 135d832a5e6d6..c0592bbc1443d 100644 --- a/ddev/src/ddev/e2e/agent/interface.py +++ b/ddev/src/ddev/e2e/agent/interface.py @@ -77,3 +77,10 @@ def invoke(self, args: list[str], *, env_vars: dict[str, str] | None = None) -> @abstractmethod def enter_shell(self) -> None: ... + + def sync_config(self) -> None: + """Synchronize the persisted host configuration with the Agent.""" + + def show_logs(self) -> None: + """Show backend-specific diagnostics for the running Agent.""" + self.invoke(['status']) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py new file mode 100644 index 0000000000000..ea171e1433e45 --- /dev/null +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -0,0 +1,545 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +import hashlib +import json +import re +import time +from functools import cached_property +from typing import TYPE_CHECKING, Any + +from ddev.e2e.agent.docker import _normalize_agent_image_name +from ddev.e2e.agent.interface import AgentInterface + +if TYPE_CHECKING: + import subprocess + + from ddev.utils.fs import Path + +_POD_NAME = 'ddev-agent' +_CONTAINER_NAME = 'agent' +_DEFAULT_NAMESPACE_PREFIX = 'ddev-agent' +_DEFAULT_WAIT_TIMEOUT = 120 +_LOCAL_PACKAGES_METADATA = 'local_packages' +_OWNER_ID_METADATA = '_kubernetes_owner_id' +_OWNER_LABEL = 'ddev.datadoghq.com/environment' + + +class KubernetesAgent(AgentInterface): + """Run the E2E Agent inside a Kubernetes test cluster. + + The initial implementation intentionally supports one schedulable node and + one Agent pod. Cluster creation and image loading belong to the environment + provisioner (for example, ``kind_run``), not this backend. + """ + + @cached_property + def _kubernetes_metadata(self) -> dict[str, Any]: + metadata = self.metadata.get('kubernetes') + if not isinstance(metadata, dict): + raise ValueError('Kubernetes Agent metadata must contain a `kubernetes` mapping') + return metadata + + @cached_property + def _kubeconfig(self) -> str: + kubeconfig = self._kubernetes_metadata.get('kubeconfig') + if not isinstance(kubeconfig, str) or not kubeconfig: + raise ValueError('Kubernetes Agent metadata must define a non-empty `kubeconfig` path') + return kubeconfig + + @cached_property + def _namespace(self) -> str: + namespace = self._kubernetes_metadata.get('namespace') + if namespace is not None: + if not isinstance(namespace, str) or not re.fullmatch(r'[a-z0-9]([-a-z0-9]*[a-z0-9])?', namespace): + raise ValueError(f'Invalid Kubernetes Agent namespace: {namespace!r}') + if len(namespace) > 63: + raise ValueError('Kubernetes Agent namespace must contain at most 63 characters') + return namespace + + raw_id = super().get_id().lower() + normalized_id = re.sub(r'[^a-z0-9]+', '-', raw_id).strip('-') + digest = hashlib.sha256(raw_id.encode()).hexdigest()[:8] + # Leave room for the prefix, separators, and digest. + normalized_id = normalized_id[: 63 - len(_DEFAULT_NAMESPACE_PREFIX) - len(digest) - 2].rstrip('-') + return f'{_DEFAULT_NAMESPACE_PREFIX}-{normalized_id}-{digest}' + + @cached_property + def _cluster_resource_name(self) -> str: + return self._namespace + + @cached_property + def _owner_id(self) -> str: + owner_id = self.metadata.get(_OWNER_ID_METADATA) + if not isinstance(owner_id, str) or not re.fullmatch(r'[a-z0-9]([-a-z0-9]*[a-z0-9])?', owner_id): + raise ValueError('Kubernetes Agent metadata is missing a valid internal owner ID') + if len(owner_id) > 63: + raise ValueError('Kubernetes Agent internal owner ID must contain at most 63 characters') + return owner_id + + @cached_property + def _resource_labels(self) -> dict[str, str]: + return { + 'app.kubernetes.io/managed-by': 'ddev', + _OWNER_LABEL: self._owner_id, + } + + @cached_property + def _config_dir(self) -> str: + return f'/etc/datadog-agent/conf.d/{self.integration.name}.d' + + @cached_property + def _python_path(self) -> str: + return f'/opt/datadog-agent/embedded/bin/python{self.python_version[0]}' + + @cached_property + def _kubectl_prefix(self) -> list[str]: + return ['kubectl', '--kubeconfig', self._kubeconfig] + + @cached_property + def _wait_timeout(self) -> int: + timeout = self._kubernetes_metadata.get('wait_timeout', _DEFAULT_WAIT_TIMEOUT) + if isinstance(timeout, bool) or not isinstance(timeout, int) or timeout <= 0: + raise ValueError('Kubernetes Agent `wait_timeout` must be a positive integer') + return timeout + + def _kubectl(self, args: list[str], **kwargs) -> subprocess.CompletedProcess: + return self.platform.run_command([*self._kubectl_prefix, *args], **kwargs) + + def _captured_kubectl(self, args: list[str], **kwargs) -> subprocess.CompletedProcess: + return self._kubectl( + args, + stdout=self.platform.modules.subprocess.PIPE, + stderr=self.platform.modules.subprocess.STDOUT, + **kwargs, + ) + + @staticmethod + def _process_output(process: subprocess.CompletedProcess) -> str: + output = process.stdout or b'' + if isinstance(output, bytes): + return output.decode('utf-8', errors='replace') + return output + + def _exec( + self, + command: list[str], + *, + env_vars: dict[str, str] | None = None, + check: bool = True, + capture: bool = False, + ) -> subprocess.CompletedProcess: + args = ['exec', '--namespace', self._namespace, f'pod/{_POD_NAME}', '--container', _CONTAINER_NAME, '--'] + if env_vars: + args.append('env') + args.extend(f'{key}={value}' for key, value in sorted(env_vars.items())) + args.extend(command) + if capture: + return self._captured_kubectl(args, check=check) + return self._kubectl(args, check=check) + + def _validate_resource_ownership(self) -> None: + resources = ( + ('namespace', self._namespace), + ('clusterrole', self._cluster_resource_name), + ('clusterrolebinding', self._cluster_resource_name), + ) + existing_resources = [] + for resource, name in resources: + process = self._captured_kubectl(['get', resource, name, '--ignore-not-found=true', '-o', 'name']) + if process.returncode: + raise RuntimeError( + f'Unable to inspect Kubernetes Agent {resource} `{name}`: {self._process_output(process)}' + ) + if self._process_output(process).strip(): + existing_resources.append(f'{resource}/{name}') + + if existing_resources: + raise RuntimeError( + 'Refusing to overwrite Kubernetes resources not owned by this environment: ' + + ', '.join(existing_resources) + ) + + def _validate_topology(self) -> None: + process = self._captured_kubectl(['get', 'nodes', '-o', 'json']) + if process.returncode: + raise RuntimeError(f'Unable to inspect Kubernetes nodes: {self._process_output(process)}') + + try: + node_data = json.loads(self._process_output(process)) + schedulable_nodes = [node for node in node_data['items'] if not node.get('spec', {}).get('unschedulable')] + except (KeyError, TypeError, json.JSONDecodeError) as e: + raise RuntimeError(f'Unable to parse Kubernetes node data: {e}') from e + + if len(schedulable_nodes) != 1: + raise NotImplementedError( + 'KubernetesAgent currently requires exactly one schedulable node; ' + f'found {len(schedulable_nodes)}. Multi-node execution needs an explicit Agent targeting policy.' + ) + + def _manifest(self, agent_build: str, env_vars: dict[str, str]) -> dict[str, Any]: + env_vars = env_vars.copy() + env_vars.setdefault('DD_API_KEY', 'a' * 32) + env_vars.setdefault('DD_APM_ENABLED', 'false') + env_vars.setdefault('DD_AUTOCONFIG_FROM_ENVIRONMENT', 'true') + env_vars.setdefault('DD_HOSTNAME', self._namespace) + env_vars.setdefault('DD_KUBELET_TLS_VERIFY', 'false') + + container_env: list[dict[str, Any]] = [{'name': key, 'value': value} for key, value in sorted(env_vars.items())] + container_env.extend( + [ + { + 'name': 'DD_KUBERNETES_KUBELET_HOST', + 'valueFrom': {'fieldRef': {'fieldPath': 'status.hostIP'}}, + }, + { + 'name': 'DD_KUBERNETES_KUBELET_NODENAME', + 'valueFrom': {'fieldRef': {'fieldPath': 'spec.nodeName'}}, + }, + ] + ) + + extra_labels = self._kubernetes_metadata.get('pod_labels', {}) + if not isinstance(extra_labels, dict) or not all( + isinstance(key, str) and isinstance(value, str) for key, value in extra_labels.items() + ): + raise ValueError('Kubernetes Agent `pod_labels` must be a mapping of strings to strings') + labels = {**extra_labels, **self._resource_labels, 'app.kubernetes.io/name': _POD_NAME} + + image_pull_policy = self._kubernetes_metadata.get('image_pull_policy', 'Always') + if image_pull_policy not in {'Always', 'IfNotPresent', 'Never'}: + raise ValueError('Kubernetes Agent `image_pull_policy` must be Always, IfNotPresent, or Never') + + role_rules = [ + {'apiGroups': [''], 'resources': ['nodes'], 'verbs': ['get', 'list', 'watch']}, + { + 'apiGroups': [''], + 'resources': ['nodes/metrics', 'nodes/spec', 'nodes/stats', 'nodes/proxy'], + 'verbs': ['get'], + }, + { + 'apiGroups': [''], + 'resources': ['pods', 'endpoints', 'services'], + 'verbs': ['get', 'list', 'watch'], + }, + ] + + return { + 'apiVersion': 'v1', + 'kind': 'List', + 'items': [ + { + 'apiVersion': 'v1', + 'kind': 'Namespace', + 'metadata': {'name': self._namespace, 'labels': self._resource_labels}, + }, + { + 'apiVersion': 'v1', + 'kind': 'ServiceAccount', + 'metadata': {'name': _POD_NAME, 'namespace': self._namespace, 'labels': self._resource_labels}, + }, + { + 'apiVersion': 'rbac.authorization.k8s.io/v1', + 'kind': 'ClusterRole', + 'metadata': {'name': self._cluster_resource_name, 'labels': self._resource_labels}, + 'rules': role_rules, + }, + { + 'apiVersion': 'rbac.authorization.k8s.io/v1', + 'kind': 'ClusterRoleBinding', + 'metadata': {'name': self._cluster_resource_name, 'labels': self._resource_labels}, + 'roleRef': { + 'apiGroup': 'rbac.authorization.k8s.io', + 'kind': 'ClusterRole', + 'name': self._cluster_resource_name, + }, + 'subjects': [ + {'kind': 'ServiceAccount', 'name': _POD_NAME, 'namespace': self._namespace}, + ], + }, + { + 'apiVersion': 'v1', + 'kind': 'Pod', + 'metadata': {'name': _POD_NAME, 'namespace': self._namespace, 'labels': labels}, + 'spec': { + 'serviceAccountName': _POD_NAME, + 'restartPolicy': 'Always', + 'terminationGracePeriodSeconds': 0, + 'tolerations': [{'operator': 'Exists'}], + 'containers': [ + { + 'name': _CONTAINER_NAME, + 'image': agent_build, + 'imagePullPolicy': image_pull_policy, + 'env': container_env, + } + ], + }, + }, + ], + } + + def _create_payload(self, payload: dict[str, Any]) -> None: + process = self._captured_kubectl(['create', '-f', '-'], input=json.dumps(payload).encode()) + if process.returncode: + raise RuntimeError(f'Unable to create Kubernetes Agent resources: {self._process_output(process)}') + + def _create_manifest(self, manifest: dict[str, Any]) -> None: + # Acquire the namespace as an atomic ownership lock before creating any + # other resource. `create`, unlike `apply`, fails rather than overwriting + # a namespace that appears after the ownership preflight. + namespace, *resources = manifest['items'] + self._create_payload(namespace) + self._create_payload({'apiVersion': 'v1', 'kind': 'List', 'items': resources}) + + def _wait_for_pod(self) -> None: + process = self._captured_kubectl( + [ + 'wait', + '--namespace', + self._namespace, + '--for=condition=Ready', + f'pod/{_POD_NAME}', + f'--timeout={self._wait_timeout}s', + ] + ) + if process.returncode: + self._show_logs() + raise RuntimeError(f'Kubernetes Agent pod did not become ready: {self._process_output(process)}') + + def _wait_for_agent(self) -> None: + deadline = time.monotonic() + self._wait_timeout + last_output = '' + while time.monotonic() < deadline: + process = self._exec(['agent', 'status'], check=False, capture=True) + if process.returncode == 0: + return + last_output = self._process_output(process) + time.sleep(1) + + self._show_logs() + raise RuntimeError(f'Kubernetes Agent did not become ready: {last_output}') + + def _copy_file(self, source: str, destination: str) -> None: + from ddev.utils.fs import Path + + source_path = Path(source).resolve() + self._kubectl( + [ + 'cp', + '--container', + _CONTAINER_NAME, + source_path.name, + f'{self._namespace}/{_POD_NAME}:{destination}', + ], + check=True, + cwd=source_path.parent, + ) + + def _sync_config(self) -> None: + self._exec(['mkdir', '-p', self._config_dir]) + destination = f'{self._config_dir}/conf.yaml' + if self.config_file.is_file(): + self._copy_file(str(self.config_file), destination) + else: + self._exec(['rm', '-f', destination]) + + def _sync_auto_conf(self) -> None: + auto_conf = self._kubernetes_metadata.get('auto_conf') + if auto_conf is None: + return + if not isinstance(auto_conf, str) or not auto_conf: + raise ValueError('Kubernetes Agent `auto_conf` must be a non-empty path') + self._exec(['mkdir', '-p', self._config_dir]) + self._copy_file(auto_conf, f'{self._config_dir}/auto_conf.yaml') + + def _remember_local_packages(self, local_packages: dict[Path, str]) -> None: + self._kubernetes_metadata[_LOCAL_PACKAGES_METADATA] = [ + {'path': str(local_package), 'name': local_package.name, 'features': features} + for local_package, features in local_packages.items() + ] + + def _local_package_specs(self) -> list[dict[str, str]]: + specs = self._kubernetes_metadata.get(_LOCAL_PACKAGES_METADATA, []) + if not isinstance(specs, list): + raise ValueError('Kubernetes Agent `local_packages` must be a list') + + for spec in specs: + if not isinstance(spec, dict) or not all( + isinstance(spec.get(key), str) for key in ('path', 'name', 'features') + ): + raise ValueError('Kubernetes Agent local package entries must define path, name, and features strings') + if not re.fullmatch(r'[A-Za-z0-9_.-]+', spec['name']) or spec['name'] in {'.', '..'}: + raise ValueError(f'Invalid Kubernetes Agent local package name: {spec["name"]!r}') + return specs + + def _sync_local_packages(self, *, install: bool = False) -> None: + for spec in self._local_package_specs(): + destination = f'/home/{spec["name"]}' + self._exec(['rm', '-rf', destination]) + self._copy_file(spec['path'], destination) + if install: + self._exec( + [ + self._python_path, + '-m', + 'pip', + 'install', + '--disable-pip-version-check', + '-e', + f'{destination}{spec["features"]}', + ] + ) + + def _run_metadata_commands(self, key: str) -> None: + commands = self.metadata.get(key, []) + if not isinstance(commands, list) or not all(isinstance(command, str) for command in commands): + raise ValueError(f'Kubernetes Agent `{key}` must be a list of commands') + for command in commands: + self._exec(self.platform.modules.shlex.split(command)) + + def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env_vars: dict[str, str]) -> None: + agent_build = _normalize_agent_image_name( + agent_build, self.python_version[0], self.metadata.get('use_jmx', False) + ) + # Validate values needed by teardown before creating any resources. + _ = self._owner_id, self._wait_timeout + self._validate_topology() + self._validate_resource_ownership() + self._create_manifest(self._manifest(agent_build, env_vars)) + self._wait_for_pod() + self._run_metadata_commands('start_commands') + self._remember_local_packages(local_packages) + self._sync_local_packages(install=True) + self._sync_config() + self._sync_auto_conf() + self._run_metadata_commands('post_install_commands') + self._restart_agent_process() + + def _resource_is_owned(self, resource: str, name: str) -> bool: + process = self._captured_kubectl(['get', resource, name, '--ignore-not-found=true', '-o', 'json']) + if process.returncode: + output = self._process_output(process) + raise RuntimeError(f'Unable to inspect Kubernetes Agent {resource} `{name}`: {output}') + if not self._process_output(process).strip(): + return False + try: + data = json.loads(self._process_output(process)) + return data.get('metadata', {}).get('labels', {}).get(_OWNER_LABEL) == self._owner_id + except (AttributeError, json.JSONDecodeError) as e: + raise RuntimeError(f'Unable to parse Kubernetes Agent {resource} `{name}`: {e}') from e + + def stop(self) -> None: + errors: list[Exception] = [] + + # Do not execute commands in a namespace that merely happens to have + # the configured name. Startup may have failed because it was already + # owned by the cluster's caller. + try: + namespace_owned = self._resource_is_owned('namespace', self._namespace) + except Exception as e: + errors.append(e) + namespace_owned = False + + if namespace_owned: + pod = self._captured_kubectl( + ['get', 'pod', _POD_NAME, '--namespace', self._namespace, '--ignore-not-found=true', '-o', 'name'] + ) + if pod.returncode == 0 and self._process_output(pod).strip(): + try: + self._run_metadata_commands('stop_commands') + except Exception as e: + errors.append(e) + + selector = f'{_OWNER_LABEL}={self._owner_id}' + for args in ( + [ + 'delete', + 'namespace', + '--selector', + selector, + '--ignore-not-found=true', + '--wait=true', + f'--timeout={self._wait_timeout}s', + ], + [ + 'delete', + 'clusterrole,clusterrolebinding', + '--selector', + selector, + '--ignore-not-found=true', + ], + ): + process = self._captured_kubectl(args) + if process.returncode: + errors.append( + RuntimeError(f'Unable to remove Kubernetes Agent resources: {self._process_output(process)}') + ) + + if errors: + details = '; '.join(str(error) for error in errors) + raise RuntimeError(f'Errors while stopping Kubernetes Agent: {details}') from errors[0] + + def _restart_agent_process(self) -> None: + # The Agent image's s6 finish handler normally shuts down the whole + # service tree when the main Agent exits. Remove it before killing the + # process so s6 starts a fresh Agent in the same container, preserving + # copied configuration and installed packages. + restart_command = ( + 'old_pid=$(pidof agent) || exit 1; ' + 'set -- $old_pid; old_pid=$1; ' + 'rm -f /var/run/s6/services/agent/finish; ' + 'kill "$old_pid" || exit 1; ' + 'elapsed=0; ' + 'while kill -0 "$old_pid" 2>/dev/null; do ' + f'[ "$elapsed" -ge {self._wait_timeout} ] && exit 1; ' + 'sleep 1; elapsed=$((elapsed + 1)); ' + 'done' + ) + self._exec(['sh', '-c', restart_command]) + self._wait_for_agent() + + def restart(self) -> None: + self._sync_local_packages() + self._sync_config() + self._sync_auto_conf() + self._restart_agent_process() + + def sync_config(self) -> None: + self._sync_config() + + def invoke(self, args: list[str], *, env_vars: dict[str, str] | None = None) -> None: + self._sync_local_packages() + self._sync_config() + self._sync_auto_conf() + self._exec(['agent', *args], env_vars=env_vars) + + def enter_shell(self) -> None: + self._kubectl( + [ + 'exec', + '-it', + '--namespace', + self._namespace, + f'pod/{_POD_NAME}', + '--container', + _CONTAINER_NAME, + '--', + 'bash', + ], + check=True, + ) + + def _show_logs(self) -> None: + self._kubectl( + ['logs', '--namespace', self._namespace, f'pod/{_POD_NAME}', '--container', _CONTAINER_NAME], + check=False, + ) + + def show_logs(self) -> None: + self._kubectl( + ['logs', '--namespace', self._namespace, f'pod/{_POD_NAME}', '--container', _CONTAINER_NAME], + check=True, + ) diff --git a/ddev/tests/cli/env/test_agent.py b/ddev/tests/cli/env/test_agent.py index 73d911e4c994b..f00baab58477a 100644 --- a/ddev/tests/cli/env/test_agent.py +++ b/ddev/tests/cli/env/test_agent.py @@ -171,3 +171,59 @@ def test_trigger_run_inject_integration(ddev, data_dir, mocker): assert not result.output invoke.assert_called_once_with(['check', integration, '-l', 'debug'], env_vars=None) + + +def test_temporary_config_is_restored_and_synchronized(ddev, data_dir, temp_dir, mocker): + invoke = mocker.patch('ddev.e2e.agent.docker.DockerAgent.invoke') + sync_config = mocker.patch('ddev.e2e.agent.docker.DockerAgent.sync_config') + + integration = 'postgres' + environment = 'py3.12' + env_data = EnvDataStorage(data_dir).get(integration, environment) + env_data.write_metadata({}) + original_config = {'instances': [{'host': 'original'}]} + env_data.write_config(original_config) + temporary_config = temp_dir / 'temporary.json' + temporary_config.write_text('{"instances": []}') + + result = ddev( + 'env', + 'agent', + integration, + environment, + '--config-file', + str(temporary_config), + 'check', + ) + + assert result.exit_code == 0, result.output + assert env_data.read_config() == original_config + invoke.assert_called_once_with(['check', integration], env_vars=None) + sync_config.assert_called_once_with() + + +def test_temporary_config_absence_is_restored_and_synchronized(ddev, data_dir, temp_dir, mocker): + invoke = mocker.patch('ddev.e2e.agent.docker.DockerAgent.invoke') + sync_config = mocker.patch('ddev.e2e.agent.docker.DockerAgent.sync_config') + + integration = 'postgres' + environment = 'py3.12' + env_data = EnvDataStorage(data_dir).get(integration, environment) + env_data.write_metadata({}) + temporary_config = temp_dir / 'temporary.json' + temporary_config.write_text('{"instances": []}') + + result = ddev( + 'env', + 'agent', + integration, + environment, + '--config-file', + str(temporary_config), + 'check', + ) + + assert result.exit_code == 0, result.output + assert not env_data.config_file.exists() + invoke.assert_called_once_with(['check', integration], env_vars=None) + sync_config.assert_called_once_with() diff --git a/ddev/tests/cli/env/test_logs.py b/ddev/tests/cli/env/test_logs.py new file mode 100644 index 0000000000000..b0bae1a361d59 --- /dev/null +++ b/ddev/tests/cli/env/test_logs.py @@ -0,0 +1,52 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from ddev.e2e.config import EnvDataStorage + + +def test_nonexistent(ddev, helpers, mocker): + show_logs = mocker.patch('ddev.e2e.agent.docker.DockerAgent.show_logs') + + integration = 'postgres' + environment = 'py3.12' + + result = ddev('env', 'logs', integration, environment) + + assert result.exit_code == 1, result.output + assert result.output == helpers.dedent( + f""" + Environment `{environment}` for integration `{integration}` is not running + """ + ) + show_logs.assert_not_called() + + +def test_basic(ddev, data_dir, mocker): + show_logs = mocker.patch('ddev.e2e.agent.docker.DockerAgent.show_logs') + + integration = 'postgres' + environment = 'py3.12' + env_data = EnvDataStorage(data_dir).get(integration, environment) + env_data.write_metadata({}) + + result = ddev('env', 'logs', integration, environment) + + assert result.exit_code == 0, result.output + assert not result.output + show_logs.assert_called_once_with() + + +def test_log_failure_is_propagated(ddev, data_dir, mocker): + show_logs = mocker.patch( + 'ddev.e2e.agent.docker.DockerAgent.show_logs', side_effect=RuntimeError('Agent logs are unavailable') + ) + + integration = 'postgres' + environment = 'py3.12' + EnvDataStorage(data_dir).get(integration, environment).write_metadata({}) + + result = ddev('env', 'logs', integration, environment) + + assert result.exit_code == 1 + assert 'Agent logs are unavailable' in result.output + show_logs.assert_called_once_with() diff --git a/ddev/tests/cli/env/test_start.py b/ddev/tests/cli/env/test_start.py index 1ecab0a0256ad..cc4a5576c82d4 100644 --- a/ddev/tests/cli/env/test_start.py +++ b/ddev/tests/cli/env/test_start.py @@ -175,6 +175,31 @@ def test_agent_build_config(ddev, config_file, helpers, data_dir, write_result_f ) +def test_kubernetes_agent_build_uses_configured_docker_image(ddev, config_file, data_dir, write_result_file, mocker): + config_file.model.agent = '7' + config_file.save() + + metadata = {'agent_type': 'kubernetes', 'kubernetes': {'kubeconfig': '/tmp/kubeconfig'}} + config = {} + mocker.patch('secrets.token_hex', return_value='test-owner') + mocker.patch('subprocess.run', side_effect=write_result_file({'metadata': metadata, 'config': config})) + start = mocker.patch('ddev.e2e.agent.kubernetes.KubernetesAgent.start') + + integration = 'postgres' + environment = 'py3.12' + env_data = EnvDataStorage(data_dir).get(integration, environment) + + result = ddev('env', 'start', integration, environment) + + assert result.exit_code == 0, result.output + assert env_data.read_metadata() == {**metadata, '_kubernetes_owner_id': 'test-owner'} + start.assert_called_once_with( + agent_build='registry.datadoghq.com/agent:7', + local_packages={}, + env_vars={'DD_DD_URL': 'https://app.datadoghq.com', 'DD_SITE': 'datadoghq.com'}, + ) + + def test_agent_build_env_var(ddev, config_file, helpers, data_dir, write_result_file, mocker): config_file.model.agent = '7' config_file.save() diff --git a/ddev/tests/cli/env/test_stop.py b/ddev/tests/cli/env/test_stop.py index 3e132c822a9e4..3990bb2385a22 100644 --- a/ddev/tests/cli/env/test_stop.py +++ b/ddev/tests/cli/env/test_stop.py @@ -1,6 +1,8 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + from ddev.e2e.config import EnvDataStorage @@ -46,6 +48,24 @@ def test_basic(ddev, helpers, data_dir, mocker): stop.assert_called_once() +def test_failed_agent_cleanup_preserves_environment_state(ddev, data_dir, mocker): + teardown = mocker.patch('subprocess.run', return_value=mocker.MagicMock(returncode=0)) + stop = mocker.patch('ddev.e2e.agent.docker.DockerAgent.stop', side_effect=RuntimeError('cleanup failed')) + + integration = 'postgres' + environment = 'py3.12' + env_data = EnvDataStorage(data_dir).get(integration, environment) + metadata = {'owner': 'retry-me'} + env_data.write_metadata(metadata) + + with pytest.raises(RuntimeError, match='cleanup failed'): + ddev('env', 'stop', integration, environment) + + assert env_data.read_metadata() == metadata + stop.assert_called_once_with() + teardown.assert_called_once() + + def test_stop_all(ddev, helpers, data_dir, mocker): mocker.patch('subprocess.run', return_value=mocker.MagicMock(returncode=0)) stop = mocker.patch('ddev.e2e.agent.docker.DockerAgent.stop') diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py new file mode 100644 index 0000000000000..ba57ae9b9684f --- /dev/null +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -0,0 +1,504 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import json +import subprocess + +import pytest + +from ddev.e2e.agent.kubernetes import KubernetesAgent +from ddev.integration.core import Integration +from ddev.repo.config import RepositoryConfig + +RESTART_COMMAND = ( + 'old_pid=$(pidof agent) || exit 1; ' + 'set -- $old_pid; old_pid=$1; ' + 'rm -f /var/run/s6/services/agent/finish; ' + 'kill "$old_pid" || exit 1; ' + 'elapsed=0; ' + 'while kill -0 "$old_pid" 2>/dev/null; do ' + '[ "$elapsed" -ge 120 ] && exit 1; ' + 'sleep 1; elapsed=$((elapsed + 1)); ' + 'done' +) + + +@pytest.fixture(scope='module') +def get_integration(local_repo): + def _get_integration(name): + return Integration(local_repo / name, local_repo, RepositoryConfig(local_repo / '.ddev' / 'config.toml')) + + return _get_integration + + +@pytest.fixture +def config_file(temp_dir): + path = temp_dir / 'config' / 'velero.yaml' + path.parent.ensure_dir_exists() + path.write_text('instances: []\n') + return path + + +@pytest.fixture +def auto_conf(temp_dir): + path = temp_dir / 'auto_conf.yaml' + path.write_text('ad_identifiers:\n - velero\n') + return path + + +@pytest.fixture +def metadata(auto_conf): + return { + '_kubernetes_owner_id': 'test-owner', + 'kubernetes': { + 'kubeconfig': '/tmp/kubeconfig', + 'auto_conf': str(auto_conf), + }, + } + + +@pytest.fixture +def agent(app, get_integration, metadata, config_file): + return KubernetesAgent(app, get_integration('velero'), 'py3.12', metadata, config_file) + + +def successful_process(command, *, stdout=b''): + return subprocess.CompletedProcess(command, 0, stdout=stdout) + + +@pytest.fixture +def run_command(app, mocker): + def run(command, **kwargs): + if command[-4:] == ['get', 'nodes', '-o', 'json']: + nodes = {'items': [{'metadata': {'name': 'kind-control-plane'}, 'spec': {}}]} + return successful_process(command, stdout=json.dumps(nodes).encode()) + return successful_process(command) + + return mocker.patch.object(app.platform, 'run_command', side_effect=run) + + +def command_calls(run_command): + return [call.args[0] for call in run_command.call_args_list] + + +def test_start_uses_selected_image_rbac_config_and_local_packages( + agent, metadata, config_file, auto_conf, temp_dir, run_command +): + local_base = temp_dir / 'datadog_checks_base' + local_base.ensure_dir_exists() + integration = temp_dir / 'velero' + integration.ensure_dir_exists() + + agent.start( + agent_build='registry.example.com/datadog-agent:test', + local_packages={local_base: '[kube]', integration: '[deps]'}, + env_vars={'DD_SITE': 'datadoghq.com'}, + ) + + calls = command_calls(run_command) + prefix = ['kubectl', '--kubeconfig', '/tmp/kubeconfig'] + assert calls[0] == [*prefix, 'get', 'nodes', '-o', 'json'] + + create_calls = [call for call in run_command.call_args_list if call.args[0] == [*prefix, 'create', '-f', '-']] + namespace = json.loads(create_calls[0].kwargs['input']) + manifest = json.loads(create_calls[1].kwargs['input']) + resources = {item['kind']: item for item in [namespace, *manifest['items']]} + assert namespace['kind'] == 'Namespace' + assert resources['Pod']['spec']['containers'][0]['image'] == 'registry.example.com/datadog-agent:test' + assert resources['Pod']['spec']['containers'][0]['imagePullPolicy'] == 'Always' + assert resources['Pod']['spec']['serviceAccountName'] == 'ddev-agent' + for kind in ('Namespace', 'ServiceAccount', 'ClusterRole', 'ClusterRoleBinding', 'Pod'): + assert resources[kind]['metadata']['labels']['ddev.datadoghq.com/environment'] == 'test-owner' + assert resources['ClusterRole']['rules'] == [ + {'apiGroups': [''], 'resources': ['nodes'], 'verbs': ['get', 'list', 'watch']}, + { + 'apiGroups': [''], + 'resources': ['nodes/metrics', 'nodes/spec', 'nodes/stats', 'nodes/proxy'], + 'verbs': ['get'], + }, + { + 'apiGroups': [''], + 'resources': ['pods', 'endpoints', 'services'], + 'verbs': ['get', 'list', 'watch'], + }, + ] + env = {item['name']: item.get('value') for item in resources['Pod']['spec']['containers'][0]['env']} + assert env['DD_API_KEY'] == 'a' * 32 + assert env['DD_SITE'] == 'datadoghq.com' + assert env['DD_AUTOCONFIG_FROM_ENVIRONMENT'] == 'true' + assert 'DD_KUBERNETES_KUBELET_HOST' in env + assert 'DD_KUBERNETES_KUBELET_NODENAME' in env + + assert [ + *prefix, + 'cp', + '--container', + 'agent', + local_base.name, + f'{agent._namespace}/ddev-agent:/home/datadog_checks_base', + ] in calls + assert [ + *prefix, + 'exec', + '--namespace', + agent._namespace, + 'pod/ddev-agent', + '--container', + 'agent', + '--', + '/opt/datadog-agent/embedded/bin/python3', + '-m', + 'pip', + 'install', + '--disable-pip-version-check', + '-e', + '/home/datadog_checks_base[kube]', + ] in calls + assert [ + *prefix, + 'cp', + '--container', + 'agent', + config_file.name, + f'{agent._namespace}/ddev-agent:/etc/datadog-agent/conf.d/velero.d/conf.yaml', + ] in calls + assert [ + *prefix, + 'cp', + '--container', + 'agent', + auto_conf.name, + f'{agent._namespace}/ddev-agent:/etc/datadog-agent/conf.d/velero.d/auto_conf.yaml', + ] in calls + assert [ + *prefix, + 'exec', + '--namespace', + agent._namespace, + 'pod/ddev-agent', + '--container', + 'agent', + '--', + 'sh', + '-c', + RESTART_COMMAND, + ] in calls + assert metadata['kubernetes']['local_packages'] == [ + {'path': str(local_base), 'name': 'datadog_checks_base', 'features': '[kube]'}, + {'path': str(integration), 'name': 'velero', 'features': '[deps]'}, + ] + local_base_copy = next(call for call in run_command.call_args_list if call.args[0][-2] == local_base.name) + assert local_base_copy.kwargs['cwd'] == local_base.resolve().parent + + +@pytest.mark.parametrize('wait_timeout', [0, True]) +def test_rejects_invalid_wait_timeout_before_creating_resources(agent, metadata, run_command, wait_timeout): + metadata['kubernetes']['wait_timeout'] = wait_timeout + + with pytest.raises(ValueError, match='wait_timeout'): + agent.start(agent_build='', local_packages={}, env_vars={}) + + run_command.assert_not_called() + + +def test_rejects_multi_node_clusters(agent, app, mocker): + nodes = {'items': [{'metadata': {'name': 'one'}, 'spec': {}}, {'metadata': {'name': 'two'}, 'spec': {}}]} + run_command = mocker.patch.object( + app.platform, + 'run_command', + return_value=successful_process([], stdout=json.dumps(nodes).encode()), + ) + + with pytest.raises(NotImplementedError, match='exactly one schedulable node'): + agent.start(agent_build='', local_packages={}, env_vars={}) + + run_command.assert_called_once() + + +def test_rejects_preexisting_owned_resources(agent, app, mocker): + nodes = {'items': [{'metadata': {'name': 'kind-control-plane'}, 'spec': {}}]} + + def run(command, **kwargs): + if command[-4:] == ['get', 'nodes', '-o', 'json']: + return successful_process(command, stdout=json.dumps(nodes).encode()) + if 'namespace' in command and '--ignore-not-found=true' in command: + return successful_process(command, stdout=f'namespace/{agent._namespace}\n'.encode()) + return successful_process(command) + + run_command = mocker.patch.object(app.platform, 'run_command', side_effect=run) + + with pytest.raises(RuntimeError, match='Refusing to overwrite Kubernetes resources'): + agent.start(agent_build='', local_packages={}, env_vars={}) + + assert not any(call.args[0][-3:] == ['create', '-f', '-'] for call in run_command.call_args_list) + + +def test_creation_fails_if_a_resource_appears_after_preflight(agent, app, mocker): + nodes = {'items': [{'metadata': {'name': 'kind-control-plane'}, 'spec': {}}]} + + def run(command, **kwargs): + if command[-4:] == ['get', 'nodes', '-o', 'json']: + return successful_process(command, stdout=json.dumps(nodes).encode()) + if command[-3:] == ['create', '-f', '-']: + return subprocess.CompletedProcess(command, 1, stdout=b'', stderr=b'namespace already exists') + return successful_process(command) + + run_command = mocker.patch.object(app.platform, 'run_command', side_effect=run) + + with pytest.raises(RuntimeError, match='Unable to create Kubernetes Agent resources'): + agent.start(agent_build='', local_packages={}, env_vars={}) + + calls = command_calls(run_command) + assert sum(command[-3:] == ['create', '-f', '-'] for command in calls) == 1 + assert not any('apply' in command or 'wait' in command for command in calls) + + +def test_invoke_synchronizes_config_and_environment(agent, metadata, config_file, auto_conf, run_command): + config_file.write_text('instances:\n - openmetrics_endpoint: http://velero:8085/metrics\n') + + agent.invoke(['check', 'velero', '--json'], env_vars={'ZED': 'last', 'ALPHA': 'first'}) + + calls = command_calls(run_command) + prefix = ['kubectl', '--kubeconfig', '/tmp/kubeconfig'] + assert calls == [ + [ + *prefix, + 'exec', + '--namespace', + agent._namespace, + 'pod/ddev-agent', + '--container', + 'agent', + '--', + 'mkdir', + '-p', + '/etc/datadog-agent/conf.d/velero.d', + ], + [ + *prefix, + 'cp', + '--container', + 'agent', + config_file.name, + f'{agent._namespace}/ddev-agent:/etc/datadog-agent/conf.d/velero.d/conf.yaml', + ], + [ + *prefix, + 'exec', + '--namespace', + agent._namespace, + 'pod/ddev-agent', + '--container', + 'agent', + '--', + 'mkdir', + '-p', + '/etc/datadog-agent/conf.d/velero.d', + ], + [ + *prefix, + 'cp', + '--container', + 'agent', + auto_conf.name, + f'{agent._namespace}/ddev-agent:/etc/datadog-agent/conf.d/velero.d/auto_conf.yaml', + ], + [ + *prefix, + 'exec', + '--namespace', + agent._namespace, + 'pod/ddev-agent', + '--container', + 'agent', + '--', + 'env', + 'ALPHA=first', + 'ZED=last', + 'agent', + 'check', + 'velero', + '--json', + ], + ] + + +def test_invoke_removes_pod_config_when_host_config_is_absent(agent, config_file, run_command): + config_file.remove() + + agent.invoke(['status']) + + calls = command_calls(run_command) + assert [ + 'kubectl', + '--kubeconfig', + '/tmp/kubeconfig', + 'exec', + '--namespace', + agent._namespace, + 'pod/ddev-agent', + '--container', + 'agent', + '--', + 'rm', + '-f', + '/etc/datadog-agent/conf.d/velero.d/conf.yaml', + ] in calls + + +def test_stop_is_idempotent_when_pod_is_absent(agent, run_command): + agent.stop() + + calls = command_calls(run_command) + prefix = ['kubectl', '--kubeconfig', '/tmp/kubeconfig'] + selector = 'ddev.datadoghq.com/environment=test-owner' + assert calls == [ + [*prefix, 'get', 'namespace', agent._namespace, '--ignore-not-found=true', '-o', 'json'], + [ + *prefix, + 'delete', + 'namespace', + '--selector', + selector, + '--ignore-not-found=true', + '--wait=true', + '--timeout=120s', + ], + [ + *prefix, + 'delete', + 'clusterrole,clusterrolebinding', + '--selector', + selector, + '--ignore-not-found=true', + ], + ] + + +def test_stop_does_not_enter_or_delete_another_environment_namespace(agent, app, mocker): + namespace = {'metadata': {'labels': {'ddev.datadoghq.com/environment': 'another-owner'}}} + + def run(command, **kwargs): + if command[-1:] == ['json'] and 'namespace' in command: + return successful_process(command, stdout=json.dumps(namespace).encode()) + return successful_process(command) + + run_command = mocker.patch.object(app.platform, 'run_command', side_effect=run) + + agent.stop() + + calls = command_calls(run_command) + assert not any('pod' in command for command in calls) + delete_calls = [command for command in calls if 'delete' in command] + assert len(delete_calls) == 2 + assert all('ddev.datadoghq.com/environment=test-owner' in command for command in delete_calls) + assert all('another-owner' not in command for command in delete_calls) + + +def test_restart_resynchronizes_config_auto_conf_and_editable_sources( + agent, metadata, config_file, temp_dir, run_command +): + local_package = temp_dir / 'velero-source' + local_package.ensure_dir_exists() + metadata['kubernetes']['local_packages'] = [ + {'path': str(local_package), 'name': 'velero-source', 'features': '[deps]'} + ] + + agent.restart() + + calls = command_calls(run_command) + prefix = ['kubectl', '--kubeconfig', '/tmp/kubeconfig'] + package_copy = [ + *prefix, + 'cp', + '--container', + 'agent', + local_package.name, + f'{agent._namespace}/ddev-agent:/home/velero-source', + ] + config_copy = [ + *prefix, + 'cp', + '--container', + 'agent', + config_file.name, + f'{agent._namespace}/ddev-agent:/etc/datadog-agent/conf.d/velero.d/conf.yaml', + ] + restart = [ + *prefix, + 'exec', + '--namespace', + agent._namespace, + 'pod/ddev-agent', + '--container', + 'agent', + '--', + 'sh', + '-c', + RESTART_COMMAND, + ] + assert package_copy in calls + assert config_copy in calls + assert calls.index(package_copy) < calls.index(config_copy) < calls.index(restart) + assert not any('pip' in command for command in calls) + + +def test_stop_cleans_resources_after_stop_command_failure(agent, app, mocker): + agent.metadata['stop_commands'] = ['false'] + + def run(command, **kwargs): + if command[-1:] == ['json'] and 'namespace' in command: + namespace = {'metadata': {'labels': {'ddev.datadoghq.com/environment': 'test-owner'}}} + return successful_process(command, stdout=json.dumps(namespace).encode()) + if command[-1:] == ['name'] and 'pod' in command: + return successful_process(command, stdout=b'pod/ddev-agent\n') + if command[-1:] == ['false']: + raise subprocess.CalledProcessError(1, command) + return successful_process(command) + + run_command = mocker.patch.object(app.platform, 'run_command', side_effect=run) + + with pytest.raises(RuntimeError, match='Errors while stopping Kubernetes Agent'): + agent.stop() + + calls = command_calls(run_command) + assert any('namespace' in command and 'delete' in command for command in calls) + assert any('clusterrole,clusterrolebinding' in command and 'delete' in command for command in calls) + + +def test_shell_and_logs_use_backend_commands(agent, run_command): + agent.enter_shell() + agent.show_logs() + + calls = command_calls(run_command) + prefix = ['kubectl', '--kubeconfig', '/tmp/kubeconfig'] + assert calls == [ + [ + *prefix, + 'exec', + '-it', + '--namespace', + agent._namespace, + 'pod/ddev-agent', + '--container', + 'agent', + '--', + 'bash', + ], + [*prefix, 'logs', '--namespace', agent._namespace, 'pod/ddev-agent', '--container', 'agent'], + ] + assert run_command.call_args_list[-1].kwargs['check'] is True + + +@pytest.mark.parametrize( + 'metadata, match', + [ + ({}, 'must contain a `kubernetes` mapping'), + ({'kubernetes': {}}, 'non-empty `kubeconfig`'), + ({'kubernetes': {'kubeconfig': '/tmp/config', 'namespace': 'INVALID'}}, 'Invalid Kubernetes Agent namespace'), + ], +) +def test_metadata_validation(app, get_integration, config_file, metadata, match): + agent = KubernetesAgent(app, get_integration('velero'), 'py3.12', metadata, config_file) + with pytest.raises(ValueError, match=match): + _ = agent._namespace if 'namespace' in metadata.get('kubernetes', {}) else agent._kubeconfig diff --git a/docs/developer/ddev/plugins.md b/docs/developer/ddev/plugins.md index 470dc3ad8ec51..fbf1c4599fa1d 100644 --- a/docs/developer/ddev/plugins.md +++ b/docs/developer/ddev/plugins.md @@ -137,8 +137,9 @@ is desired. This fixture is responsible for starting and stopping environments a #### Metadata -- `env_type` - This is the type of interface that will be used to interact with the Agent. Currently, we support `docker` (default), `vagrant` and `local`. +- `agent_type` - The interface used to interact with the Agent. Supported values are `docker` (default), `vagrant`, and `kubernetes`. - `env_vars` - A `dict` of environment variables and their values that will be present when starting the Agent. -- `docker_volumes` - A `list` of `str` representing [Docker volume mounts][docker-volume-docs] if `env_type` is `docker` e.g. `/local/path:/agent/container/path:ro`. -- `docker_platform` - The container architecture to use if `env_type` is `docker`. Currently, we support `linux` (default) and `windows`. +- `docker_volumes` - A `list` of `str` representing [Docker volume mounts][docker-volume-docs] if `agent_type` is `docker` e.g. `/local/path:/agent/container/path:ro`. +- `docker_platform` - The container architecture to use if `agent_type` is `docker`. Currently, we support `linux` (default) and `windows`. +- `kubernetes` - Configuration used when `agent_type` is `kubernetes`. It requires `kubeconfig` and optionally accepts `namespace`, `auto_conf`, `pod_labels`, `image_pull_policy`, and `wait_timeout`. The backend owns the namespace, which must not already exist. - `logs_config` - A `list` of configs that will be used by the Logs Agent. You will never need to use this directly, but rather via [higher level abstractions](test.md#logs). diff --git a/docs/developer/ddev/test.md b/docs/developer/ddev/test.md index 34895ef89ad78..1e6ed2b49f1a9 100644 --- a/docs/developer/ddev/test.md +++ b/docs/developer/ddev/test.md @@ -91,6 +91,42 @@ You can use the `%HOST%` template variable in your configuration instead of hard Note: Vagrant environments are not supported in CI environments due to virtualization constraints. +### Kubernetes Agent + +The Kubernetes Agent backend runs the Datadog Agent inside an existing Kubernetes test cluster. The cluster lifecycle +remains the responsibility of an environment helper such as `kind_run`; the backend only requires a kubeconfig and uses +standard `kubectl` operations, so it is not tied to Kind. + +```python +@pytest.fixture(scope='session') +def dd_environment(): + with kind_run(conditions=[setup_workload]) as kubeconfig: + yield { + 'instances': [ + # Use endpoints reachable from inside the cluster. + {'openmetrics_endpoint': 'http://my-service.default.svc.cluster.local:8080/metrics'}, + ], + }, { + 'agent_type': 'kubernetes', + 'kubernetes': { + 'kubeconfig': kubeconfig, + # Optional: install an Autodiscovery template for this integration. + 'auto_conf': os.path.join(CHECK_ROOT, 'datadog_checks', CHECK_NAME, 'data', 'auto_conf.yaml'), + }, + } +``` + +The backend uses the Agent image selected by `ddev env start --agent` or `DDEV_E2E_AGENT`, installs and synchronizes +local packages requested by `--dev` or `--base`, and implements Agent commands through `kubectl exec`. Static and +discovery E2E tests therefore continue to use `dd_agent_check` and `dd_agent_check_discovery`. + +Agent images default to the `Always` pull policy so mutable release and development tags are refreshed. Environments +that import a local image into the cluster can set `image_pull_policy` to `IfNotPresent` or `Never`. A custom `namespace` +must not already exist; the backend owns and deletes the namespace and its cluster-scoped RBAC resources. + +The initial implementation supports exactly one schedulable Kubernetes node. It rejects multi-node clusters until +Agent targeting or fan-out semantics are defined. + ### Terraform The `terraform_run` utility makes it easy to create services from a directory of [Terraform][terraform-home] files. diff --git a/velero/assets/configuration/spec.yaml b/velero/assets/configuration/spec.yaml index 8427942921816..3e179ce3dd5ed 100644 --- a/velero/assets/configuration/spec.yaml +++ b/velero/assets/configuration/spec.yaml @@ -2,6 +2,13 @@ name: Velero fleet_configurable: true files: - name: velero.yaml + discovery: + strategies: + - template: discovery/openmetrics_from_named_ports + overrides: + port_names: + - http-monitoring + - metrics options: - template: init_config options: @@ -14,3 +21,10 @@ files: - type: docker source: velero service: +- name: auto_conf.yaml + options: + - template: ad_identifiers + overrides: + value.example: + - velero + - template: auto_conf/discovery diff --git a/velero/changelog.d/24507.added b/velero/changelog.d/24507.added new file mode 100644 index 0000000000000..1455667b0b73f --- /dev/null +++ b/velero/changelog.d/24507.added @@ -0,0 +1 @@ +Add container-based config discovery support. diff --git a/velero/datadog_checks/velero/config_models/discovery.py b/velero/datadog_checks/velero/config_models/discovery.py new file mode 100644 index 0000000000000..c05ad4ed3fd2d --- /dev/null +++ b/velero/datadog_checks/velero/config_models/discovery.py @@ -0,0 +1,52 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Any + +from datadog_checks.base.utils.discovery import Service +from datadog_checks.velero.config_models import discovery_overrides +from datadog_checks.velero.config_models.instance import InstanceConfig +from datadog_checks.velero.config_models.shared import SharedConfig + + +def _generated_candidates(service: Service) -> Iterator[dict[str, Any]]: + shared = SharedConfig.model_validate({}, context={'configured_fields': frozenset()}).model_dump( + by_alias=True, mode='json', exclude_none=True + ) + # discovery[0]: from_named_ports + seen_ports = set() + for port in ( + port + for port_name in dict.fromkeys(['http-monitoring', 'metrics']) + if port_name + for port in service.ports + if port.name == port_name + ): + if port.number in seen_ports: + continue + seen_ports.add(port.number) + ctx = {'port': port} + instance_data = { + 'openmetrics_endpoint': 'http://{service.host}:{port.number}/metrics'.format(service=service, **ctx), + } + instance = InstanceConfig.model_validate( + instance_data, context={'configured_fields': frozenset(instance_data)} + ).model_dump(by_alias=True, mode='json', exclude_none=True) + yield {'init_config': shared, 'instances': [instance]} + + +def candidates(service: Service) -> Iterator[dict[str, Any]]: + override = getattr(discovery_overrides, 'candidates', None) + if override is None: + yield from _generated_candidates(service) + else: + yield from override(service, default=_generated_candidates) diff --git a/velero/datadog_checks/velero/config_models/discovery_overrides.py b/velero/datadog_checks/velero/config_models/discovery_overrides.py new file mode 100644 index 0000000000000..66af68809dd4c --- /dev/null +++ b/velero/datadog_checks/velero/config_models/discovery_overrides.py @@ -0,0 +1,12 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# Override the generated discovery candidates() for this integration. +# +# Define a candidates(service, default) function to wrap or replace the generated +# candidate generation. `default` is the generated generator; call it to reuse +# the spec-driven candidates, or ignore it to replace them entirely. +# +# def candidates(service, default): +# yield from default(service) diff --git a/velero/datadog_checks/velero/config_models/discovery_strategies.py b/velero/datadog_checks/velero/config_models/discovery_strategies.py new file mode 100644 index 0000000000000..5ac036ddb4684 --- /dev/null +++ b/velero/datadog_checks/velero/config_models/discovery_strategies.py @@ -0,0 +1,18 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# Here you can define custom (local:) discovery strategies for this integration. +# +# Decorate a generator with @discovery_strategy (imported from +# datadog_checks.base.utils.discovery) and reference it from the spec discovery +# stanza as `strategy: local:`. The function receives the +# discovered Service plus the inputs declared in the spec and yields one context +# (ctx) mapping per candidate, exposing the keys listed in `provides`. +# +# from datadog_checks.base.utils.discovery import discovery_strategy +# +# @discovery_strategy(provides=('svc',)) +# def from_some_config(service, config_path): +# ... +# yield {'svc': ...} diff --git a/velero/datadog_checks/velero/data/auto_conf.yaml b/velero/datadog_checks/velero/data/auto_conf.yaml new file mode 100644 index 0000000000000..5c7281d343166 --- /dev/null +++ b/velero/datadog_checks/velero/data/auto_conf.yaml @@ -0,0 +1,19 @@ +## @param ad_identifiers - list of strings - required +## A list of container identifiers that are used by Autodiscovery to identify +## which container the check should be run against. For more information, see: +## https://docs.datadoghq.com/agent/guide/ad_identifiers/ +# +ad_identifiers: + - velero + +## Enables configuration discovery +# +discovery: {} + +## Unused init configuration +# +init_config: + +## Unused instance configuration +# +instances: [] diff --git a/velero/pyproject.toml b/velero/pyproject.toml index ad4519b9589a6..ec0dfe1974b24 100644 --- a/velero/pyproject.toml +++ b/velero/pyproject.toml @@ -29,7 +29,7 @@ classifiers = [ "Topic :: System :: Monitoring", ] dependencies = [ - "datadog-checks-base>=37.33.0", + "datadog-checks-base>=37.41.0", ] dynamic = [ "version", diff --git a/velero/tests/conftest.py b/velero/tests/conftest.py index 9d6aaf6bd3a64..bcaefe0ad6b4e 100644 --- a/velero/tests/conftest.py +++ b/velero/tests/conftest.py @@ -1,20 +1,23 @@ # (C) Datadog, Inc. 2025-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import json import os -from contextlib import ExitStack, contextmanager +from contextlib import contextmanager import pytest from datadog_checks.dev import TempDir, run_command +from datadog_checks.dev._env import get_state, save_state from datadog_checks.dev.fs import path_join from datadog_checks.dev.kind import KindLoad, kind_run -from datadog_checks.dev.kube_port_forward import port_forward from .common import MOCKED_INSTANCE, PORT HERE = os.path.dirname(os.path.abspath(__file__)) +CHECK_ROOT = os.path.dirname(HERE) KIND_DIR = os.path.join(HERE, 'kind') +NODE_AGENT_IP_STATE = 'velero_node_agent_ip' @contextmanager @@ -56,17 +59,37 @@ def setup_velero(): ], check=True, ) + save_state(NODE_AGENT_IP_STATE, get_node_agent_ip()) -def get_instances(velero_host, velero_port, node_agent_host, node_agent_port): +def get_node_agent_ip(): + result = run_command( + ['kubectl', 'get', 'pods', '--namespace', 'velero', '--output', 'json'], + capture='out', + check=True, + ) + node_agent_pods = [ + pod + for pod in json.loads(result.stdout)['items'] + if any( + owner.get('kind') == 'DaemonSet' and owner.get('name') == 'node-agent' + for owner in pod['metadata'].get('ownerReferences', []) + ) + ] + if len(node_agent_pods) != 1 or not node_agent_pods[0].get('status', {}).get('podIP'): + raise RuntimeError(f'Expected one ready Velero node-agent pod, found {len(node_agent_pods)}') + return node_agent_pods[0]['status']['podIP'] + + +def get_instances(node_agent_ip): return { 'instances': [ { - 'openmetrics_endpoint': f"http://{velero_host}:{velero_port}/metrics", + 'openmetrics_endpoint': f"http://velero.velero.svc.cluster.local:{PORT}/metrics", 'tags': ['test:tag'], }, { - 'openmetrics_endpoint': f"http://{node_agent_host}:{node_agent_port}/metrics", + 'openmetrics_endpoint': f"http://{node_agent_ip}:{PORT}/metrics", 'tags': ['test:tag'], }, ] @@ -88,15 +111,16 @@ def dd_environment(): "HELM_CONFIG_HOME": path_join(helm_dir, 'Preferences'), }, ) as kubeconfig: - with ExitStack() as stack: - ip_ports = [ - stack.enter_context(port_forward(kubeconfig, 'velero', PORT, ressource, name)) - for ressource, name in [('service', 'velero'), ('daemonset', 'node-agent')] - ] - - instances = get_instances(ip_ports[0][0], ip_ports[0][1], ip_ports[1][0], ip_ports[1][1]) - - yield instances + instances = get_instances(get_state(NODE_AGENT_IP_STATE)) + metadata = { + 'agent_type': 'kubernetes', + 'kubernetes': { + 'kubeconfig': kubeconfig, + 'auto_conf': os.path.join(CHECK_ROOT, 'datadog_checks', 'velero', 'data', 'auto_conf.yaml'), + }, + } + + yield instances, metadata @pytest.fixture diff --git a/velero/tests/test_e2e.py b/velero/tests/test_e2e.py index 8067bb6f0e82a..39181929b8d0f 100644 --- a/velero/tests/test_e2e.py +++ b/velero/tests/test_e2e.py @@ -8,10 +8,7 @@ from .common import OPTIONAL_METRICS, TEST_METRICS -@pytest.mark.e2e -def test_check_velero_e2e(dd_agent_check): - aggregator = dd_agent_check(rate=True) - +def assert_metrics(aggregator): for metric, _ in TEST_METRICS.items(): if metric in OPTIONAL_METRICS: aggregator.assert_metric(name=metric, at_least=0) @@ -19,3 +16,14 @@ def test_check_velero_e2e(dd_agent_check): aggregator.assert_metric(name=metric, at_least=1) aggregator.assert_service_check('velero.openmetrics.health', ServiceCheck.OK) + + +@pytest.mark.e2e +def test_check_velero_e2e(dd_agent_check): + assert_metrics(dd_agent_check(rate=True)) + + +@pytest.mark.e2e +def test_check_velero_discovery_e2e(dd_agent_check_discovery): + aggregator = dd_agent_check_discovery(check_rate=True, discovery_min_instances=2) + assert_metrics(aggregator) From 496b3c25e12f54ed09ab1ad3c99f869edf534ece Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Wed, 22 Jul 2026 07:30:19 +0000 Subject: [PATCH 02/50] Use draft PR number for changelogs --- datadog_checks_dev/changelog.d/{24507.added => 24639.added} | 0 ddev/changelog.d/{24507.added => 24639.added} | 0 velero/changelog.d/{24507.added => 24639.added} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename datadog_checks_dev/changelog.d/{24507.added => 24639.added} (100%) rename ddev/changelog.d/{24507.added => 24639.added} (100%) rename velero/changelog.d/{24507.added => 24639.added} (100%) diff --git a/datadog_checks_dev/changelog.d/24507.added b/datadog_checks_dev/changelog.d/24639.added similarity index 100% rename from datadog_checks_dev/changelog.d/24507.added rename to datadog_checks_dev/changelog.d/24639.added diff --git a/ddev/changelog.d/24507.added b/ddev/changelog.d/24639.added similarity index 100% rename from ddev/changelog.d/24507.added rename to ddev/changelog.d/24639.added diff --git a/velero/changelog.d/24507.added b/velero/changelog.d/24639.added similarity index 100% rename from velero/changelog.d/24507.added rename to velero/changelog.d/24639.added From d4e2ba3d84970ae00f824cc4b9e071a342f990b6 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Wed, 22 Jul 2026 08:33:01 +0000 Subject: [PATCH 03/50] Remove unused Velero Kind port mapping --- velero/tests/conftest.py | 2 -- velero/tests/kind/kind-config.yaml | 7 ------- 2 files changed, 9 deletions(-) delete mode 100644 velero/tests/kind/kind-config.yaml diff --git a/velero/tests/conftest.py b/velero/tests/conftest.py index bcaefe0ad6b4e..1f6b339348b47 100644 --- a/velero/tests/conftest.py +++ b/velero/tests/conftest.py @@ -98,14 +98,12 @@ def get_instances(node_agent_ip): @pytest.fixture(scope='session') def dd_environment(): - kind_config = os.path.join(KIND_DIR, 'kind-config.yaml') custom_kubectl_image_tag = "custom-kubectl:latest" with TempDir('helm_dir') as helm_dir: with kind_run( wrappers=[build_and_load_kubectl_image(custom_kubectl_image_tag)], conditions=[KindLoad(custom_kubectl_image_tag), setup_velero], - kind_config=kind_config, env_vars={ "HELM_CACHE_HOME": path_join(helm_dir, 'Caches'), "HELM_CONFIG_HOME": path_join(helm_dir, 'Preferences'), diff --git a/velero/tests/kind/kind-config.yaml b/velero/tests/kind/kind-config.yaml deleted file mode 100644 index 93ad325fe45a3..0000000000000 --- a/velero/tests/kind/kind-config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: -- role: control-plane - extraPortMappings: - - containerPort: 8085 - hostPort: 8085 \ No newline at end of file From 0ae34764c5eeaba5d14337a4490b6f526a616bc1 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Wed, 22 Jul 2026 09:10:32 +0000 Subject: [PATCH 04/50] Use shared named-port discovery helper --- .../discovery/core_strategies.py | 11 +--- .../consumers/model/test_discovery.py | 54 ++----------------- .../velero/config_models/discovery.py | 14 +---- velero/pyproject.toml | 2 +- velero/tests/test_discovery.py | 24 +++++++++ 5 files changed, 32 insertions(+), 73 deletions(-) create mode 100644 velero/tests/test_discovery.py diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/core_strategies.py b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/core_strategies.py index c7b2cc24d9ecf..d0c7627f57e71 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/core_strategies.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/core_strategies.py @@ -27,19 +27,12 @@ def from_ports(stanza: dict[str, Any]) -> list[str]: 'from_named_ports', provides=('port',), inputs={'port_names': Input('array[string]')}, + runtime_imports=('from datadog_checks.base.utils.discovery import candidate_ports_by_name',), ) def from_named_ports(stanza: dict[str, Any]) -> list[str]: """Open a loop over explicitly named service ports and bind a `ctx` exposing the current `port`.""" port_names = stanza['port_names'] - # Keep generated integrations compatible with the base version that first - # introduced discovery while preserving the requested name order and - # deduplicating ports by number. return [ - ' seen_ports = set()', - f' for port in (port for port_name in dict.fromkeys({port_names!r}) if port_name ' - 'for port in service.ports if port.name == port_name):', - ' if port.number in seen_ports:', - ' continue', - ' seen_ports.add(port.number)', + f' for port in candidate_ports_by_name(service, {port_names!r}):', " ctx = {'port': port}", ] diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py index c659a3010631b..3df232338524e 100644 --- a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py @@ -180,58 +180,10 @@ def test_from_named_ports(): """ ) - model_definitions = consumer.render() - files = model_definitions['test.yaml'] - - discovery_contents, discovery_errors = files['discovery.py'] + discovery_contents, discovery_errors = consumer.render()['test.yaml']['discovery.py'] assert not discovery_errors - assert discovery_contents == normalize_yaml( - """ - from __future__ import annotations - - from collections.abc import Iterator - from typing import Any - - from datadog_checks.base.utils.discovery import Service - from datadog_checks.test.config_models import discovery_overrides - from datadog_checks.test.config_models.instance import InstanceConfig - from datadog_checks.test.config_models.shared import SharedConfig - - - def _generated_candidates(service: Service) -> Iterator[dict[str, Any]]: - shared = SharedConfig.model_validate({}, context={'configured_fields': frozenset()}).model_dump( - by_alias=True, mode='json', exclude_none=True - ) - # discovery[0]: from_named_ports - seen_ports = set() - for port in ( - port - for port_name in dict.fromkeys(['metrics', 'http-monitoring']) - if port_name - for port in service.ports - if port.name == port_name - ): - if port.number in seen_ports: - continue - seen_ports.add(port.number) - ctx = {'port': port} - instance_data = { - 'endpoint': 'http://{service.host}:{port.number}/m'.format(service=service, **ctx), - } - instance = InstanceConfig.model_validate( - instance_data, context={'configured_fields': frozenset(instance_data)} - ).model_dump(by_alias=True, mode='json', exclude_none=True) - yield {'init_config': shared, 'instances': [instance]} - - - def candidates(service: Service) -> Iterator[dict[str, Any]]: - override = getattr(discovery_overrides, 'candidates', None) - if override is None: - yield from _generated_candidates(service) - else: - yield from override(service, default=_generated_candidates) - """ - ) + assert 'from datadog_checks.base.utils.discovery import Service, candidate_ports_by_name' in discovery_contents + assert "for port in candidate_ports_by_name(service, ['metrics', 'http-monitoring']):" in discovery_contents def test_local_strategy(): diff --git a/velero/datadog_checks/velero/config_models/discovery.py b/velero/datadog_checks/velero/config_models/discovery.py index c05ad4ed3fd2d..7ed1478dc356e 100644 --- a/velero/datadog_checks/velero/config_models/discovery.py +++ b/velero/datadog_checks/velero/config_models/discovery.py @@ -12,7 +12,7 @@ from collections.abc import Iterator from typing import Any -from datadog_checks.base.utils.discovery import Service +from datadog_checks.base.utils.discovery import Service, candidate_ports_by_name from datadog_checks.velero.config_models import discovery_overrides from datadog_checks.velero.config_models.instance import InstanceConfig from datadog_checks.velero.config_models.shared import SharedConfig @@ -23,17 +23,7 @@ def _generated_candidates(service: Service) -> Iterator[dict[str, Any]]: by_alias=True, mode='json', exclude_none=True ) # discovery[0]: from_named_ports - seen_ports = set() - for port in ( - port - for port_name in dict.fromkeys(['http-monitoring', 'metrics']) - if port_name - for port in service.ports - if port.name == port_name - ): - if port.number in seen_ports: - continue - seen_ports.add(port.number) + for port in candidate_ports_by_name(service, ['http-monitoring', 'metrics']): ctx = {'port': port} instance_data = { 'openmetrics_endpoint': 'http://{service.host}:{port.number}/metrics'.format(service=service, **ctx), diff --git a/velero/pyproject.toml b/velero/pyproject.toml index ec0dfe1974b24..96ea6757c2346 100644 --- a/velero/pyproject.toml +++ b/velero/pyproject.toml @@ -29,7 +29,7 @@ classifiers = [ "Topic :: System :: Monitoring", ] dependencies = [ - "datadog-checks-base>=37.41.0", + "datadog-checks-base>=37.43.0", ] dynamic = [ "version", diff --git a/velero/tests/test_discovery.py b/velero/tests/test_discovery.py new file mode 100644 index 0000000000000..b0db30795eeaf --- /dev/null +++ b/velero/tests/test_discovery.py @@ -0,0 +1,24 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from datadog_checks.base.utils.discovery import Port, Service +from datadog_checks.velero import VeleroCheck + + +def test_generated_discovery_uses_named_metrics_ports(): + service = Service( + id='velero', + host='10.0.0.1', + ports=( + Port(number=8085, name='metrics'), + Port(number=9999, name='admin'), + Port(number=8086, name='http-monitoring'), + ), + ) + + configs = list(VeleroCheck.generate_configs(service)) + + assert [config['instances'][0]['openmetrics_endpoint'] for config in configs] == [ + 'http://10.0.0.1:8086/metrics', + 'http://10.0.0.1:8085/metrics', + ] From 9711d666ab0cfc376fbb813c9ac8a786823a4297 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Wed, 22 Jul 2026 09:54:45 +0000 Subject: [PATCH 05/50] Test generated discovery behavior --- .../consumers/model/test_discovery.py | 177 +++++++++--------- 1 file changed, 92 insertions(+), 85 deletions(-) diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py index 3df232338524e..8d95fe95318cb 100644 --- a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py @@ -1,10 +1,60 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from ...utils import get_model_consumer, get_spec, normalize_yaml - - -def test(): +import sys +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock + +from ...utils import get_model_consumer, get_spec + + +class _Model: + def __init__(self, data): + self.data = data + + @classmethod + def model_validate(cls, data, *, context): + return cls(data) + + def model_dump(self, **kwargs): + return self.data + + +def _load_discovery(contents, monkeypatch, **strategy_helpers): + discovery_utils = ModuleType('datadog_checks.base.utils.discovery') + discovery_utils.Service = object + for name, helper in strategy_helpers.items(): + setattr(discovery_utils, name, helper) + + integration_package = ModuleType('datadog_checks.test') + integration_package.__path__ = [] + config_models = ModuleType('datadog_checks.test.config_models') + config_models.__path__ = [] + discovery_overrides = ModuleType('datadog_checks.test.config_models.discovery_overrides') + instance = ModuleType('datadog_checks.test.config_models.instance') + instance.InstanceConfig = _Model + shared = ModuleType('datadog_checks.test.config_models.shared') + shared.SharedConfig = _Model + + integration_package.config_models = config_models + config_models.discovery_overrides = discovery_overrides + modules = { + discovery_utils.__name__: discovery_utils, + integration_package.__name__: integration_package, + config_models.__name__: config_models, + discovery_overrides.__name__: discovery_overrides, + instance.__name__: instance, + shared.__name__: shared, + } + for name, module in modules.items(): + monkeypatch.setitem(sys.modules, name, module) + + namespace = {} + exec(compile(contents, 'discovery.py', 'exec'), namespace) + return namespace + + +def test_generated_candidates(monkeypatch): consumer = get_model_consumer( """ name: test @@ -31,51 +81,20 @@ def test(): """ ) - model_definitions = consumer.render() - files = model_definitions['test.yaml'] - - discovery_contents, discovery_errors = files['discovery.py'] + discovery_contents, discovery_errors = consumer.render()['test.yaml']['discovery.py'] assert not discovery_errors - assert discovery_contents == normalize_yaml( - """ - from __future__ import annotations - - from collections.abc import Iterator - from typing import Any - - from datadog_checks.base.utils.discovery import Service, candidate_ports - from datadog_checks.test.config_models import discovery_overrides - from datadog_checks.test.config_models.instance import InstanceConfig - from datadog_checks.test.config_models.shared import SharedConfig - - - def _generated_candidates(service: Service) -> Iterator[dict[str, Any]]: - shared = SharedConfig.model_validate({}, context={'configured_fields': frozenset()}).model_dump( - by_alias=True, mode='json', exclude_none=True - ) - # discovery[0]: from_ports - for port in candidate_ports(service, [9090]): - ctx = {'port': port} - instance_data = { - 'endpoint': 'http://{service.host}:{port.number}/m'.format(service=service, **ctx), - } - instance = InstanceConfig.model_validate( - instance_data, context={'configured_fields': frozenset(instance_data)} - ).model_dump(by_alias=True, mode='json', exclude_none=True) - yield {'init_config': shared, 'instances': [instance]} - - - def candidates(service: Service) -> Iterator[dict[str, Any]]: - override = getattr(discovery_overrides, 'candidates', None) - if override is None: - yield from _generated_candidates(service) - else: - yield from override(service, default=_generated_candidates) - """ - ) + candidate_ports = Mock(return_value=[SimpleNamespace(number=9090)]) + candidates = _load_discovery(discovery_contents, monkeypatch, candidate_ports=candidate_ports)['candidates'] + service = SimpleNamespace(host='example.com') + + assert list(candidates(service)) == [ + {'init_config': {}, 'instances': [{'endpoint': 'http://example.com:9090/m'}]}, + ] + candidate_ports.assert_called_once_with(service, [9090]) -def test_literal_candidate_values(): + +def test_literal_candidate_values(monkeypatch): consumer = get_model_consumer( """ name: test @@ -112,47 +131,26 @@ def test_literal_candidate_values(): discovery_contents, discovery_errors = consumer.render()['test.yaml']['discovery.py'] assert not discovery_errors - assert discovery_contents == normalize_yaml( - """ - from __future__ import annotations - - from collections.abc import Iterator - from typing import Any - - from datadog_checks.base.utils.discovery import Service, candidate_ports - from datadog_checks.test.config_models import discovery_overrides - from datadog_checks.test.config_models.instance import InstanceConfig - from datadog_checks.test.config_models.shared import SharedConfig - - - def _generated_candidates(service: Service) -> Iterator[dict[str, Any]]: - shared = SharedConfig.model_validate({}, context={'configured_fields': frozenset()}).model_dump( - by_alias=True, mode='json', exclude_none=True - ) - # discovery[0]: from_ports - for port in candidate_ports(service, [9090]): - ctx = {'port': port} - instance_data = { - 'endpoint': 'http://{service.host}:{port.number}/m'.format(service=service, **ctx), + + candidate_ports = Mock(return_value=[SimpleNamespace(number=9090)]) + candidates = _load_discovery(discovery_contents, monkeypatch, candidate_ports=candidate_ports)['candidates'] + service = SimpleNamespace(host='example.com') + + assert list(candidates(service)) == [ + { + 'init_config': {}, + 'instances': [ + { + 'endpoint': 'http://example.com:9090/m', 'metric_patterns': {'include': ['test.metric.{2}']}, } - instance = InstanceConfig.model_validate( - instance_data, context={'configured_fields': frozenset(instance_data)} - ).model_dump(by_alias=True, mode='json', exclude_none=True) - yield {'init_config': shared, 'instances': [instance]} - - - def candidates(service: Service) -> Iterator[dict[str, Any]]: - override = getattr(discovery_overrides, 'candidates', None) - if override is None: - yield from _generated_candidates(service) - else: - yield from override(service, default=_generated_candidates) - """ - ) + ], + } + ] + candidate_ports.assert_called_once_with(service, [9090]) -def test_from_named_ports(): +def test_from_named_ports(monkeypatch): consumer = get_model_consumer( """ name: test @@ -182,8 +180,17 @@ def test_from_named_ports(): discovery_contents, discovery_errors = consumer.render()['test.yaml']['discovery.py'] assert not discovery_errors - assert 'from datadog_checks.base.utils.discovery import Service, candidate_ports_by_name' in discovery_contents - assert "for port in candidate_ports_by_name(service, ['metrics', 'http-monitoring']):" in discovery_contents + + candidate_ports_by_name = Mock(return_value=[SimpleNamespace(number=8085)]) + candidates = _load_discovery(discovery_contents, monkeypatch, candidate_ports_by_name=candidate_ports_by_name)[ + 'candidates' + ] + service = SimpleNamespace(host='example.com') + + assert list(candidates(service)) == [ + {'init_config': {}, 'instances': [{'endpoint': 'http://example.com:8085/m'}]}, + ] + candidate_ports_by_name.assert_called_once_with(service, ['metrics', 'http-monitoring']) def test_local_strategy(): From 0065786f4aea8332291724e6d8c6704816c889ca Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Wed, 22 Jul 2026 10:29:08 +0000 Subject: [PATCH 06/50] Test generated discovery through public contract --- .../consumers/model/model_consumer.py | 5 +- .../consumers/model/test_discovery.py | 251 ++++++++---------- 2 files changed, 117 insertions(+), 139 deletions(-) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/consumers/model/model_consumer.py b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/consumers/model/model_consumer.py index fcfab100e66a3..5bd10841c99cc 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/consumers/model/model_consumer.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/consumers/model/model_consumer.py @@ -93,7 +93,10 @@ def render(self) -> Dict[str, Dict[str, str]]: model_files.update(self._build_model_files(model_info, package_info)) if 'discovery' in spec_file: pkg_name = spec_file['name'].removesuffix('.yaml') - has_shared = any(s['name'] == 'init_config' for s in spec_file.get('options', [])) + has_shared = any( + section['name'] == 'init_config' and section.get('options') + for section in spec_file.get('options', []) + ) model_files['discovery.py'] = ( self._build_discovery_file(spec_file['discovery'], pkg_name, has_shared=has_shared), [], diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py index 8d95fe95318cb..99da232bba22b 100644 --- a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py @@ -1,60 +1,51 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import importlib +import re import sys -from types import ModuleType, SimpleNamespace -from unittest.mock import Mock -from ...utils import get_model_consumer, get_spec - - -class _Model: - def __init__(self, data): - self.data = data - - @classmethod - def model_validate(cls, data, *, context): - return cls(data) - - def model_dump(self, **kwargs): - return self.data - - -def _load_discovery(contents, monkeypatch, **strategy_helpers): - discovery_utils = ModuleType('datadog_checks.base.utils.discovery') - discovery_utils.Service = object - for name, helper in strategy_helpers.items(): - setattr(discovery_utils, name, helper) +import datadog_checks +from datadog_checks.base import AgentCheck +from datadog_checks.base.utils.discovery import Port, Service - integration_package = ModuleType('datadog_checks.test') - integration_package.__path__ = [] - config_models = ModuleType('datadog_checks.test.config_models') - config_models.__path__ = [] - discovery_overrides = ModuleType('datadog_checks.test.config_models.discovery_overrides') - instance = ModuleType('datadog_checks.test.config_models.instance') - instance.InstanceConfig = _Model - shared = ModuleType('datadog_checks.test.config_models.shared') - shared.SharedConfig = _Model - - integration_package.config_models = config_models - config_models.discovery_overrides = discovery_overrides - modules = { - discovery_utils.__name__: discovery_utils, - integration_package.__name__: integration_package, - config_models.__name__: config_models, - discovery_overrides.__name__: discovery_overrides, - instance.__name__: instance, - shared.__name__: shared, - } - for name, module in modules.items(): - monkeypatch.setitem(sys.modules, name, module) - - namespace = {} - exec(compile(contents, 'discovery.py', 'exec'), namespace) - return namespace +from ...utils import get_model_consumer, get_spec -def test_generated_candidates(monkeypatch): +def _generate_configs_from_rendered_package(consumer, service, tmp_path, monkeypatch, *, custom_files=None): + package_name = f'generated_{re.sub(r"\W", "_", tmp_path.name)}' + spec_file = consumer.spec['files'][0] + spec_file['name'] = f'{package_name}.yaml' + + rendered_files = consumer.render()[spec_file['name']] + package_root = tmp_path / 'datadog_checks' + package_dir = package_root / package_name + config_models_dir = package_dir / 'config_models' + config_models_dir.mkdir(parents=True) + (package_dir / '__init__.py').write_text('') + + custom_files = custom_files or {} + for file_name, (contents, errors) in rendered_files.items(): + assert not errors + (config_models_dir / file_name).write_text(custom_files.get(file_name, contents)) + + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.setattr(datadog_checks, '__path__', [str(package_root), *datadog_checks.__path__]) + importlib.invalidate_caches() + + module_name = f'datadog_checks.{package_name}' + check = type('GeneratedDiscoveryCheck', (AgentCheck,), {'__module__': module_name}) + try: + return list(check.generate_configs(service)) + finally: + for imported_module in tuple(sys.modules): + if imported_module == module_name or imported_module.startswith(f'{module_name}.'): + sys.modules.pop(imported_module) + if hasattr(datadog_checks, package_name): + delattr(datadog_checks, package_name) + + +def test_generated_candidates_follow_public_discovery_contract(tmp_path, monkeypatch): consumer = get_model_consumer( """ name: test @@ -80,21 +71,15 @@ def test_generated_candidates(monkeypatch): type: string """ ) + service = Service(id='svc', host='example.com', ports=(Port(number=9090),)) + configs = _generate_configs_from_rendered_package(consumer, service, tmp_path, monkeypatch) - discovery_contents, discovery_errors = consumer.render()['test.yaml']['discovery.py'] - assert not discovery_errors - - candidate_ports = Mock(return_value=[SimpleNamespace(number=9090)]) - candidates = _load_discovery(discovery_contents, monkeypatch, candidate_ports=candidate_ports)['candidates'] - service = SimpleNamespace(host='example.com') - - assert list(candidates(service)) == [ + assert configs == [ {'init_config': {}, 'instances': [{'endpoint': 'http://example.com:9090/m'}]}, ] - candidate_ports.assert_called_once_with(service, [9090]) -def test_literal_candidate_values(monkeypatch): +def test_literal_candidate_values_follow_public_discovery_contract(tmp_path, monkeypatch): consumer = get_model_consumer( """ name: test @@ -125,18 +110,17 @@ def test_literal_candidate_values(monkeypatch): description: words value: type: object - additionalProperties: true + properties: + - name: include + type: array + items: + type: string """ ) + service = Service(id='svc', host='example.com', ports=(Port(number=9090),)) + configs = _generate_configs_from_rendered_package(consumer, service, tmp_path, monkeypatch) - discovery_contents, discovery_errors = consumer.render()['test.yaml']['discovery.py'] - assert not discovery_errors - - candidate_ports = Mock(return_value=[SimpleNamespace(number=9090)]) - candidates = _load_discovery(discovery_contents, monkeypatch, candidate_ports=candidate_ports)['candidates'] - service = SimpleNamespace(host='example.com') - - assert list(candidates(service)) == [ + assert configs == [ { 'init_config': {}, 'instances': [ @@ -147,10 +131,9 @@ def test_literal_candidate_values(monkeypatch): ], } ] - candidate_ports.assert_called_once_with(service, [9090]) -def test_from_named_ports(monkeypatch): +def test_from_named_ports_follows_public_discovery_contract(tmp_path, monkeypatch): consumer = get_model_consumer( """ name: test @@ -177,23 +160,24 @@ def test_from_named_ports(monkeypatch): type: string """ ) + service = Service( + id='svc', + host='example.com', + ports=( + Port(number=8086, name='http-monitoring'), + Port(number=9999, name='admin'), + Port(number=8085, name='metrics'), + ), + ) + configs = _generate_configs_from_rendered_package(consumer, service, tmp_path, monkeypatch) - discovery_contents, discovery_errors = consumer.render()['test.yaml']['discovery.py'] - assert not discovery_errors - - candidate_ports_by_name = Mock(return_value=[SimpleNamespace(number=8085)]) - candidates = _load_discovery(discovery_contents, monkeypatch, candidate_ports_by_name=candidate_ports_by_name)[ - 'candidates' - ] - service = SimpleNamespace(host='example.com') - - assert list(candidates(service)) == [ + assert configs == [ {'init_config': {}, 'instances': [{'endpoint': 'http://example.com:8085/m'}]}, + {'init_config': {}, 'instances': [{'endpoint': 'http://example.com:8086/m'}]}, ] - candidate_ports_by_name.assert_called_once_with(service, ['metrics', 'http-monitoring']) -def test_local_strategy(): +def test_local_strategy_follows_public_discovery_contract(tmp_path, monkeypatch): consumer = get_model_consumer( """ name: test @@ -220,16 +204,31 @@ def test_local_strategy(): type: string """ ) + service = Service(id='svc', host='example.com', ports=()) + configs = _generate_configs_from_rendered_package( + consumer, + service, + tmp_path, + monkeypatch, + custom_files={ + 'discovery_strategies.py': """ +from types import SimpleNamespace + + +def from_app_config(service, config_path): + assert service.id == 'svc' + assert config_path == '/etc/app.conf' + yield {'svc': SimpleNamespace(port=8123)} +""", + }, + ) - discovery_contents, discovery_errors = consumer.render()['test.yaml']['discovery.py'] - assert not discovery_errors - assert 'from datadog_checks.test.config_models.discovery_strategies import from_app_config' in discovery_contents - assert "for ctx in from_app_config(service, config_path='/etc/app.conf'):" in discovery_contents - assert 'InstanceConfig.model_validate(' in discovery_contents - assert 'from datadog_checks.test.config_models import discovery_overrides' in discovery_contents + assert configs == [ + {'init_config': {}, 'instances': [{'endpoint': 'http://example.com:8123/m'}]}, + ] -def test_override_seam_wired(): +def test_override_seam_follows_public_discovery_contract(tmp_path, monkeypatch): consumer = get_model_consumer( """ name: test @@ -254,15 +253,25 @@ def test_override_seam_wired(): type: string """ ) + service = Service(id='svc', host='example.com', ports=(Port(number=9090),)) + configs = _generate_configs_from_rendered_package( + consumer, + service, + tmp_path, + monkeypatch, + custom_files={ + 'discovery_overrides.py': """ +def candidates(service, default): + for config in default(service): + config['instances'][0]['endpoint'] += '?source=override' + yield config +""", + }, + ) - discovery_contents, discovery_errors = consumer.render()['test.yaml']['discovery.py'] - assert not discovery_errors - # The override seam is wired even though this integration does not use it: - # the public candidates() delegates to discovery_overrides.candidates when present. - assert 'from datadog_checks.test.config_models import discovery_overrides' in discovery_contents - assert "override = getattr(discovery_overrides, 'candidates', None)" in discovery_contents - assert 'yield from _generated_candidates(service)' in discovery_contents - assert 'yield from override(service, default=_generated_candidates)' in discovery_contents + assert configs == [ + {'init_config': {}, 'instances': [{'endpoint': 'http://example.com:9090/m?source=override'}]}, + ] def test_unknown_strategy_clean_error(): @@ -291,44 +300,10 @@ def test_unknown_strategy_clean_error(): ) spec.load() - # An unknown strategy must surface as a clean spec error from validation, - # never as a traceback or NotImplementedError from the generator. - assert any('Unsupported strategy `from_services`' in e for e in spec.errors) + assert any('Unsupported strategy `from_services`' in error for error in spec.errors) -def test_custom_files_are_stubbed(): - consumer = get_model_consumer( - """ - name: test - version: 0.0.0 - files: - - name: test.yaml - discovery: - strategies: - - strategy: from_ports - port_hints: [9090] - candidates: - - endpoint: http://{service.host}:{port.number}/m - options: - - template: init_config - options: [] - - template: instances - options: - - name: endpoint - description: words - required: true - value: - type: string - """ - ) - - files = consumer.render()['test.yaml'] - assert 'discovery_strategies.py' in files - assert 'discovery_overrides.py' in files - assert 'candidates(service, default)' in files['discovery_overrides.py'][0] - - -def test_no_init_config_section(): +def test_no_init_config_follows_public_discovery_contract(tmp_path, monkeypatch): consumer = get_model_consumer( """ name: test @@ -352,9 +327,9 @@ def test_no_init_config_section(): type: string """ ) + service = Service(id='svc', host='example.com', ports=(Port(number=9090),)) + configs = _generate_configs_from_rendered_package(consumer, service, tmp_path, monkeypatch) - discovery_contents, discovery_errors = consumer.render()['test.yaml']['discovery.py'] - assert not discovery_errors - assert 'SharedConfig' not in discovery_contents - assert 'from datadog_checks.test.config_models.shared' not in discovery_contents - assert ' shared = {}' in discovery_contents + assert configs == [ + {'init_config': {}, 'instances': [{'endpoint': 'http://example.com:9090/m'}]}, + ] From 9f63a549637edafbfced8d42fc86c083dd6aeb22 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Wed, 22 Jul 2026 10:50:38 +0000 Subject: [PATCH 07/50] Extract Velero migration into stacked PR --- datadog_checks_dev/changelog.d/24639.added | 2 +- .../consumers/model/model_consumer.py | 5 +- .../discovery/core_strategies.py | 15 - .../configuration/discovery/registry.py | 2 +- .../dev/tooling/configuration/spec.py | 3 - .../openmetrics_from_named_ports.yaml | 5 - .../consumers/model/test_discovery.py | 284 +++++++++--------- .../tests/tooling/configuration/test_load.py | 50 --- velero/assets/configuration/spec.yaml | 14 - velero/changelog.d/24639.added | 1 - .../velero/config_models/discovery.py | 42 --- .../config_models/discovery_overrides.py | 12 - .../config_models/discovery_strategies.py | 18 -- .../datadog_checks/velero/data/auto_conf.yaml | 19 -- velero/pyproject.toml | 2 +- velero/tests/conftest.py | 54 +--- velero/tests/kind/kind-config.yaml | 7 + velero/tests/test_discovery.py | 24 -- velero/tests/test_e2e.py | 16 +- 19 files changed, 165 insertions(+), 410 deletions(-) delete mode 100644 datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/discovery/openmetrics_from_named_ports.yaml delete mode 100644 velero/changelog.d/24639.added delete mode 100644 velero/datadog_checks/velero/config_models/discovery.py delete mode 100644 velero/datadog_checks/velero/config_models/discovery_overrides.py delete mode 100644 velero/datadog_checks/velero/config_models/discovery_strategies.py delete mode 100644 velero/datadog_checks/velero/data/auto_conf.yaml create mode 100644 velero/tests/kind/kind-config.yaml delete mode 100644 velero/tests/test_discovery.py diff --git a/datadog_checks_dev/changelog.d/24639.added b/datadog_checks_dev/changelog.d/24639.added index 30b10aaabda4b..d4a5b54b8f258 100644 --- a/datadog_checks_dev/changelog.d/24639.added +++ b/datadog_checks_dev/changelog.d/24639.added @@ -1 +1 @@ -Add a Kubernetes Agent backend for E2E tests and a generated configuration strategy for Kubernetes named ports. +Use backend-neutral Agent log retrieval for E2E test diagnostics. diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/consumers/model/model_consumer.py b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/consumers/model/model_consumer.py index 5bd10841c99cc..fcfab100e66a3 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/consumers/model/model_consumer.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/consumers/model/model_consumer.py @@ -93,10 +93,7 @@ def render(self) -> Dict[str, Dict[str, str]]: model_files.update(self._build_model_files(model_info, package_info)) if 'discovery' in spec_file: pkg_name = spec_file['name'].removesuffix('.yaml') - has_shared = any( - section['name'] == 'init_config' and section.get('options') - for section in spec_file.get('options', []) - ) + has_shared = any(s['name'] == 'init_config' for s in spec_file.get('options', [])) model_files['discovery.py'] = ( self._build_discovery_file(spec_file['discovery'], pkg_name, has_shared=has_shared), [], diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/core_strategies.py b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/core_strategies.py index d0c7627f57e71..7ebd65174c528 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/core_strategies.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/core_strategies.py @@ -21,18 +21,3 @@ def from_ports(stanza: dict[str, Any]) -> list[str]: f' for port in candidate_ports(service, {port_hints!r}):', " ctx = {'port': port}", ] - - -@strategy( - 'from_named_ports', - provides=('port',), - inputs={'port_names': Input('array[string]')}, - runtime_imports=('from datadog_checks.base.utils.discovery import candidate_ports_by_name',), -) -def from_named_ports(stanza: dict[str, Any]) -> list[str]: - """Open a loop over explicitly named service ports and bind a `ctx` exposing the current `port`.""" - port_names = stanza['port_names'] - return [ - f' for port in candidate_ports_by_name(service, {port_names!r}):', - " ctx = {'port': port}", - ] diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/registry.py b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/registry.py index f622bca71b4d2..789bc9b8269b3 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/registry.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/discovery/registry.py @@ -20,7 +20,7 @@ class Input: """A declared strategy input, validated against the spec stanza.""" - type: str # "array[int]" | "array[string]" | "string" | "integer" | "boolean" + type: str # "array[int]" | "string" | "integer" | "boolean" required: bool = True diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/spec.py b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/spec.py index f02a690a3d1ee..d92af56b06abc 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/configuration/spec.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/configuration/spec.py @@ -236,9 +236,6 @@ def _validate_strategy_input(stanza: dict, name: str, input_def: Any, loader: An if input_def.type == 'array[int]': if not isinstance(value, list) or not all(isinstance(v, int) and not isinstance(v, bool) for v in value): loader.errors.append(f'{location}: Attribute `{name}` must be an array of integers') - elif input_def.type == 'array[string]': - if not isinstance(value, list) or not all(isinstance(v, str) for v in value): - loader.errors.append(f'{location}: Attribute `{name}` must be an array of strings') elif input_def.type == 'integer': if not isinstance(value, int) or isinstance(value, bool): loader.errors.append(f'{location}: Attribute `{name}` must be an integer') diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/discovery/openmetrics_from_named_ports.yaml b/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/discovery/openmetrics_from_named_ports.yaml deleted file mode 100644 index 17ce4376de88c..0000000000000 --- a/datadog_checks_dev/datadog_checks/dev/tooling/templates/configuration/discovery/openmetrics_from_named_ports.yaml +++ /dev/null @@ -1,5 +0,0 @@ -strategy: from_named_ports -port_names: -- metrics -candidates: -- openmetrics_endpoint: "http://{service.host}:{port.number}/metrics" diff --git a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py index 99da232bba22b..f75b4fb19eda7 100644 --- a/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py +++ b/datadog_checks_dev/tests/tooling/configuration/consumers/model/test_discovery.py @@ -1,51 +1,10 @@ # (C) Datadog, Inc. 2026-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import importlib -import re -import sys - -import datadog_checks -from datadog_checks.base import AgentCheck -from datadog_checks.base.utils.discovery import Port, Service - -from ...utils import get_model_consumer, get_spec - - -def _generate_configs_from_rendered_package(consumer, service, tmp_path, monkeypatch, *, custom_files=None): - package_name = f'generated_{re.sub(r"\W", "_", tmp_path.name)}' - spec_file = consumer.spec['files'][0] - spec_file['name'] = f'{package_name}.yaml' - - rendered_files = consumer.render()[spec_file['name']] - package_root = tmp_path / 'datadog_checks' - package_dir = package_root / package_name - config_models_dir = package_dir / 'config_models' - config_models_dir.mkdir(parents=True) - (package_dir / '__init__.py').write_text('') - - custom_files = custom_files or {} - for file_name, (contents, errors) in rendered_files.items(): - assert not errors - (config_models_dir / file_name).write_text(custom_files.get(file_name, contents)) - - monkeypatch.syspath_prepend(str(tmp_path)) - monkeypatch.setattr(datadog_checks, '__path__', [str(package_root), *datadog_checks.__path__]) - importlib.invalidate_caches() - - module_name = f'datadog_checks.{package_name}' - check = type('GeneratedDiscoveryCheck', (AgentCheck,), {'__module__': module_name}) - try: - return list(check.generate_configs(service)) - finally: - for imported_module in tuple(sys.modules): - if imported_module == module_name or imported_module.startswith(f'{module_name}.'): - sys.modules.pop(imported_module) - if hasattr(datadog_checks, package_name): - delattr(datadog_checks, package_name) - - -def test_generated_candidates_follow_public_discovery_contract(tmp_path, monkeypatch): +from ...utils import get_model_consumer, get_spec, normalize_yaml + + +def test(): consumer = get_model_consumer( """ name: test @@ -71,15 +30,52 @@ def test_generated_candidates_follow_public_discovery_contract(tmp_path, monkeyp type: string """ ) - service = Service(id='svc', host='example.com', ports=(Port(number=9090),)) - configs = _generate_configs_from_rendered_package(consumer, service, tmp_path, monkeypatch) - assert configs == [ - {'init_config': {}, 'instances': [{'endpoint': 'http://example.com:9090/m'}]}, - ] + model_definitions = consumer.render() + files = model_definitions['test.yaml'] + + discovery_contents, discovery_errors = files['discovery.py'] + assert not discovery_errors + assert discovery_contents == normalize_yaml( + """ + from __future__ import annotations + + from collections.abc import Iterator + from typing import Any + + from datadog_checks.base.utils.discovery import Service, candidate_ports + from datadog_checks.test.config_models import discovery_overrides + from datadog_checks.test.config_models.instance import InstanceConfig + from datadog_checks.test.config_models.shared import SharedConfig -def test_literal_candidate_values_follow_public_discovery_contract(tmp_path, monkeypatch): + def _generated_candidates(service: Service) -> Iterator[dict[str, Any]]: + shared = SharedConfig.model_validate({}, context={'configured_fields': frozenset()}).model_dump( + by_alias=True, mode='json', exclude_none=True + ) + # discovery[0]: from_ports + for port in candidate_ports(service, [9090]): + ctx = {'port': port} + instance_data = { + 'endpoint': 'http://{service.host}:{port.number}/m'.format(service=service, **ctx), + } + instance = InstanceConfig.model_validate( + instance_data, context={'configured_fields': frozenset(instance_data)} + ).model_dump(by_alias=True, mode='json', exclude_none=True) + yield {'init_config': shared, 'instances': [instance]} + + + def candidates(service: Service) -> Iterator[dict[str, Any]]: + override = getattr(discovery_overrides, 'candidates', None) + if override is None: + yield from _generated_candidates(service) + else: + yield from override(service, default=_generated_candidates) + """ + ) + + +def test_literal_candidate_values(): consumer = get_model_consumer( """ name: test @@ -110,30 +106,53 @@ def test_literal_candidate_values_follow_public_discovery_contract(tmp_path, mon description: words value: type: object - properties: - - name: include - type: array - items: - type: string + additionalProperties: true """ ) - service = Service(id='svc', host='example.com', ports=(Port(number=9090),)) - configs = _generate_configs_from_rendered_package(consumer, service, tmp_path, monkeypatch) - - assert configs == [ - { - 'init_config': {}, - 'instances': [ - { - 'endpoint': 'http://example.com:9090/m', + + discovery_contents, discovery_errors = consumer.render()['test.yaml']['discovery.py'] + assert not discovery_errors + assert discovery_contents == normalize_yaml( + """ + from __future__ import annotations + + from collections.abc import Iterator + from typing import Any + + from datadog_checks.base.utils.discovery import Service, candidate_ports + from datadog_checks.test.config_models import discovery_overrides + from datadog_checks.test.config_models.instance import InstanceConfig + from datadog_checks.test.config_models.shared import SharedConfig + + + def _generated_candidates(service: Service) -> Iterator[dict[str, Any]]: + shared = SharedConfig.model_validate({}, context={'configured_fields': frozenset()}).model_dump( + by_alias=True, mode='json', exclude_none=True + ) + # discovery[0]: from_ports + for port in candidate_ports(service, [9090]): + ctx = {'port': port} + instance_data = { + 'endpoint': 'http://{service.host}:{port.number}/m'.format(service=service, **ctx), 'metric_patterns': {'include': ['test.metric.{2}']}, } - ], - } - ] + instance = InstanceConfig.model_validate( + instance_data, context={'configured_fields': frozenset(instance_data)} + ).model_dump(by_alias=True, mode='json', exclude_none=True) + yield {'init_config': shared, 'instances': [instance]} + + + def candidates(service: Service) -> Iterator[dict[str, Any]]: + override = getattr(discovery_overrides, 'candidates', None) + if override is None: + yield from _generated_candidates(service) + else: + yield from override(service, default=_generated_candidates) + """ + ) -def test_from_named_ports_follows_public_discovery_contract(tmp_path, monkeypatch): +def test_local_strategy(): consumer = get_model_consumer( """ name: test @@ -142,12 +161,12 @@ def test_from_named_ports_follows_public_discovery_contract(tmp_path, monkeypatc - name: test.yaml discovery: strategies: - - strategy: from_named_ports - port_names: - - metrics - - http-monitoring + - strategy: local:from_app_config + provides: [svc] + inputs: {config_path: string} + config_path: /etc/app.conf candidates: - - endpoint: http://{service.host}:{port.number}/m + - endpoint: http://{service.host}:{svc.port}/m options: - template: init_config options: [] @@ -160,24 +179,16 @@ def test_from_named_ports_follows_public_discovery_contract(tmp_path, monkeypatc type: string """ ) - service = Service( - id='svc', - host='example.com', - ports=( - Port(number=8086, name='http-monitoring'), - Port(number=9999, name='admin'), - Port(number=8085, name='metrics'), - ), - ) - configs = _generate_configs_from_rendered_package(consumer, service, tmp_path, monkeypatch) - assert configs == [ - {'init_config': {}, 'instances': [{'endpoint': 'http://example.com:8085/m'}]}, - {'init_config': {}, 'instances': [{'endpoint': 'http://example.com:8086/m'}]}, - ] + discovery_contents, discovery_errors = consumer.render()['test.yaml']['discovery.py'] + assert not discovery_errors + assert 'from datadog_checks.test.config_models.discovery_strategies import from_app_config' in discovery_contents + assert "for ctx in from_app_config(service, config_path='/etc/app.conf'):" in discovery_contents + assert 'InstanceConfig.model_validate(' in discovery_contents + assert 'from datadog_checks.test.config_models import discovery_overrides' in discovery_contents -def test_local_strategy_follows_public_discovery_contract(tmp_path, monkeypatch): +def test_override_seam_wired(): consumer = get_model_consumer( """ name: test @@ -186,12 +197,10 @@ def test_local_strategy_follows_public_discovery_contract(tmp_path, monkeypatch) - name: test.yaml discovery: strategies: - - strategy: local:from_app_config - provides: [svc] - inputs: {config_path: string} - config_path: /etc/app.conf + - strategy: from_ports + port_hints: [9090] candidates: - - endpoint: http://{service.host}:{svc.port}/m + - endpoint: http://{service.host}:{port.number}/m options: - template: init_config options: [] @@ -204,32 +213,19 @@ def test_local_strategy_follows_public_discovery_contract(tmp_path, monkeypatch) type: string """ ) - service = Service(id='svc', host='example.com', ports=()) - configs = _generate_configs_from_rendered_package( - consumer, - service, - tmp_path, - monkeypatch, - custom_files={ - 'discovery_strategies.py': """ -from types import SimpleNamespace - - -def from_app_config(service, config_path): - assert service.id == 'svc' - assert config_path == '/etc/app.conf' - yield {'svc': SimpleNamespace(port=8123)} -""", - }, - ) - assert configs == [ - {'init_config': {}, 'instances': [{'endpoint': 'http://example.com:8123/m'}]}, - ] + discovery_contents, discovery_errors = consumer.render()['test.yaml']['discovery.py'] + assert not discovery_errors + # The override seam is wired even though this integration does not use it: + # the public candidates() delegates to discovery_overrides.candidates when present. + assert 'from datadog_checks.test.config_models import discovery_overrides' in discovery_contents + assert "override = getattr(discovery_overrides, 'candidates', None)" in discovery_contents + assert 'yield from _generated_candidates(service)' in discovery_contents + assert 'yield from override(service, default=_generated_candidates)' in discovery_contents -def test_override_seam_follows_public_discovery_contract(tmp_path, monkeypatch): - consumer = get_model_consumer( +def test_unknown_strategy_clean_error(): + spec = get_spec( """ name: test version: 0.0.0 @@ -237,10 +233,9 @@ def test_override_seam_follows_public_discovery_contract(tmp_path, monkeypatch): - name: test.yaml discovery: strategies: - - strategy: from_ports - port_hints: [9090] + - strategy: from_services candidates: - - endpoint: http://{service.host}:{port.number}/m + - endpoint: http://{service.host}/m options: - template: init_config options: [] @@ -253,29 +248,15 @@ def test_override_seam_follows_public_discovery_contract(tmp_path, monkeypatch): type: string """ ) - service = Service(id='svc', host='example.com', ports=(Port(number=9090),)) - configs = _generate_configs_from_rendered_package( - consumer, - service, - tmp_path, - monkeypatch, - custom_files={ - 'discovery_overrides.py': """ -def candidates(service, default): - for config in default(service): - config['instances'][0]['endpoint'] += '?source=override' - yield config -""", - }, - ) + spec.load() - assert configs == [ - {'init_config': {}, 'instances': [{'endpoint': 'http://example.com:9090/m?source=override'}]}, - ] + # An unknown strategy must surface as a clean spec error from validation, + # never as a traceback or NotImplementedError from the generator. + assert any('Unsupported strategy `from_services`' in e for e in spec.errors) -def test_unknown_strategy_clean_error(): - spec = get_spec( +def test_custom_files_are_stubbed(): + consumer = get_model_consumer( """ name: test version: 0.0.0 @@ -283,9 +264,10 @@ def test_unknown_strategy_clean_error(): - name: test.yaml discovery: strategies: - - strategy: from_services + - strategy: from_ports + port_hints: [9090] candidates: - - endpoint: http://{service.host}/m + - endpoint: http://{service.host}:{port.number}/m options: - template: init_config options: [] @@ -298,12 +280,14 @@ def test_unknown_strategy_clean_error(): type: string """ ) - spec.load() - assert any('Unsupported strategy `from_services`' in error for error in spec.errors) + files = consumer.render()['test.yaml'] + assert 'discovery_strategies.py' in files + assert 'discovery_overrides.py' in files + assert 'candidates(service, default)' in files['discovery_overrides.py'][0] -def test_no_init_config_follows_public_discovery_contract(tmp_path, monkeypatch): +def test_no_init_config_section(): consumer = get_model_consumer( """ name: test @@ -327,9 +311,9 @@ def test_no_init_config_follows_public_discovery_contract(tmp_path, monkeypatch) type: string """ ) - service = Service(id='svc', host='example.com', ports=(Port(number=9090),)) - configs = _generate_configs_from_rendered_package(consumer, service, tmp_path, monkeypatch) - assert configs == [ - {'init_config': {}, 'instances': [{'endpoint': 'http://example.com:9090/m'}]}, - ] + discovery_contents, discovery_errors = consumer.render()['test.yaml']['discovery.py'] + assert not discovery_errors + assert 'SharedConfig' not in discovery_contents + assert 'from datadog_checks.test.config_models.shared' not in discovery_contents + assert ' shared = {}' in discovery_contents diff --git a/datadog_checks_dev/tests/tooling/configuration/test_load.py b/datadog_checks_dev/tests/tooling/configuration/test_load.py index d355de42ad416..f00072d36bc87 100644 --- a/datadog_checks_dev/tests/tooling/configuration/test_load.py +++ b/datadog_checks_dev/tests/tooling/configuration/test_load.py @@ -361,56 +361,6 @@ def test_discovery_rejects_boolean_port_hints(): assert 'test, test.yaml, discovery, strategy #1: Attribute `port_hints` must be an array of integers' in spec.errors -def test_discovery_named_ports_valid(): - spec = get_spec( - """ - version: 0.0.0 - files: - - name: test.yaml - example_name: test.yaml.example - discovery: - strategies: - - strategy: from_named_ports - port_names: - - metrics - - http-monitoring - candidates: - - openmetrics_endpoint: http://{service.host}:{port.number}/metrics - options: - - template: init_config - - template: instances - """ - ) - spec.load() - - assert not spec.errors - - -def test_discovery_rejects_non_string_port_names(): - spec = get_spec( - """ - version: 0.0.0 - files: - - name: test.yaml - example_name: test.yaml.example - discovery: - strategies: - - strategy: from_named_ports - port_names: - - metrics - - 8080 - candidates: - - openmetrics_endpoint: http://{service.host}:{port.number}/metrics - options: - - template: init_config - - template: instances - """ - ) - spec.load() - - assert 'test, test.yaml, discovery, strategy #1: Attribute `port_names` must be an array of strings' in spec.errors - - def test_discovery_unsupported_strategy(): spec = get_spec( """ diff --git a/velero/assets/configuration/spec.yaml b/velero/assets/configuration/spec.yaml index 3e179ce3dd5ed..8427942921816 100644 --- a/velero/assets/configuration/spec.yaml +++ b/velero/assets/configuration/spec.yaml @@ -2,13 +2,6 @@ name: Velero fleet_configurable: true files: - name: velero.yaml - discovery: - strategies: - - template: discovery/openmetrics_from_named_ports - overrides: - port_names: - - http-monitoring - - metrics options: - template: init_config options: @@ -21,10 +14,3 @@ files: - type: docker source: velero service: -- name: auto_conf.yaml - options: - - template: ad_identifiers - overrides: - value.example: - - velero - - template: auto_conf/discovery diff --git a/velero/changelog.d/24639.added b/velero/changelog.d/24639.added deleted file mode 100644 index 1455667b0b73f..0000000000000 --- a/velero/changelog.d/24639.added +++ /dev/null @@ -1 +0,0 @@ -Add container-based config discovery support. diff --git a/velero/datadog_checks/velero/config_models/discovery.py b/velero/datadog_checks/velero/config_models/discovery.py deleted file mode 100644 index 7ed1478dc356e..0000000000000 --- a/velero/datadog_checks/velero/config_models/discovery.py +++ /dev/null @@ -1,42 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -# This file is autogenerated. -# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: -# ddev -x validate config -s -# ddev -x validate models -s - -from __future__ import annotations - -from collections.abc import Iterator -from typing import Any - -from datadog_checks.base.utils.discovery import Service, candidate_ports_by_name -from datadog_checks.velero.config_models import discovery_overrides -from datadog_checks.velero.config_models.instance import InstanceConfig -from datadog_checks.velero.config_models.shared import SharedConfig - - -def _generated_candidates(service: Service) -> Iterator[dict[str, Any]]: - shared = SharedConfig.model_validate({}, context={'configured_fields': frozenset()}).model_dump( - by_alias=True, mode='json', exclude_none=True - ) - # discovery[0]: from_named_ports - for port in candidate_ports_by_name(service, ['http-monitoring', 'metrics']): - ctx = {'port': port} - instance_data = { - 'openmetrics_endpoint': 'http://{service.host}:{port.number}/metrics'.format(service=service, **ctx), - } - instance = InstanceConfig.model_validate( - instance_data, context={'configured_fields': frozenset(instance_data)} - ).model_dump(by_alias=True, mode='json', exclude_none=True) - yield {'init_config': shared, 'instances': [instance]} - - -def candidates(service: Service) -> Iterator[dict[str, Any]]: - override = getattr(discovery_overrides, 'candidates', None) - if override is None: - yield from _generated_candidates(service) - else: - yield from override(service, default=_generated_candidates) diff --git a/velero/datadog_checks/velero/config_models/discovery_overrides.py b/velero/datadog_checks/velero/config_models/discovery_overrides.py deleted file mode 100644 index 66af68809dd4c..0000000000000 --- a/velero/datadog_checks/velero/config_models/discovery_overrides.py +++ /dev/null @@ -1,12 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -# Override the generated discovery candidates() for this integration. -# -# Define a candidates(service, default) function to wrap or replace the generated -# candidate generation. `default` is the generated generator; call it to reuse -# the spec-driven candidates, or ignore it to replace them entirely. -# -# def candidates(service, default): -# yield from default(service) diff --git a/velero/datadog_checks/velero/config_models/discovery_strategies.py b/velero/datadog_checks/velero/config_models/discovery_strategies.py deleted file mode 100644 index 5ac036ddb4684..0000000000000 --- a/velero/datadog_checks/velero/config_models/discovery_strategies.py +++ /dev/null @@ -1,18 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) - -# Here you can define custom (local:) discovery strategies for this integration. -# -# Decorate a generator with @discovery_strategy (imported from -# datadog_checks.base.utils.discovery) and reference it from the spec discovery -# stanza as `strategy: local:`. The function receives the -# discovered Service plus the inputs declared in the spec and yields one context -# (ctx) mapping per candidate, exposing the keys listed in `provides`. -# -# from datadog_checks.base.utils.discovery import discovery_strategy -# -# @discovery_strategy(provides=('svc',)) -# def from_some_config(service, config_path): -# ... -# yield {'svc': ...} diff --git a/velero/datadog_checks/velero/data/auto_conf.yaml b/velero/datadog_checks/velero/data/auto_conf.yaml deleted file mode 100644 index 5c7281d343166..0000000000000 --- a/velero/datadog_checks/velero/data/auto_conf.yaml +++ /dev/null @@ -1,19 +0,0 @@ -## @param ad_identifiers - list of strings - required -## A list of container identifiers that are used by Autodiscovery to identify -## which container the check should be run against. For more information, see: -## https://docs.datadoghq.com/agent/guide/ad_identifiers/ -# -ad_identifiers: - - velero - -## Enables configuration discovery -# -discovery: {} - -## Unused init configuration -# -init_config: - -## Unused instance configuration -# -instances: [] diff --git a/velero/pyproject.toml b/velero/pyproject.toml index 96ea6757c2346..ad4519b9589a6 100644 --- a/velero/pyproject.toml +++ b/velero/pyproject.toml @@ -29,7 +29,7 @@ classifiers = [ "Topic :: System :: Monitoring", ] dependencies = [ - "datadog-checks-base>=37.43.0", + "datadog-checks-base>=37.33.0", ] dynamic = [ "version", diff --git a/velero/tests/conftest.py b/velero/tests/conftest.py index 1f6b339348b47..9d6aaf6bd3a64 100644 --- a/velero/tests/conftest.py +++ b/velero/tests/conftest.py @@ -1,23 +1,20 @@ # (C) Datadog, Inc. 2025-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import json import os -from contextlib import contextmanager +from contextlib import ExitStack, contextmanager import pytest from datadog_checks.dev import TempDir, run_command -from datadog_checks.dev._env import get_state, save_state from datadog_checks.dev.fs import path_join from datadog_checks.dev.kind import KindLoad, kind_run +from datadog_checks.dev.kube_port_forward import port_forward from .common import MOCKED_INSTANCE, PORT HERE = os.path.dirname(os.path.abspath(__file__)) -CHECK_ROOT = os.path.dirname(HERE) KIND_DIR = os.path.join(HERE, 'kind') -NODE_AGENT_IP_STATE = 'velero_node_agent_ip' @contextmanager @@ -59,37 +56,17 @@ def setup_velero(): ], check=True, ) - save_state(NODE_AGENT_IP_STATE, get_node_agent_ip()) -def get_node_agent_ip(): - result = run_command( - ['kubectl', 'get', 'pods', '--namespace', 'velero', '--output', 'json'], - capture='out', - check=True, - ) - node_agent_pods = [ - pod - for pod in json.loads(result.stdout)['items'] - if any( - owner.get('kind') == 'DaemonSet' and owner.get('name') == 'node-agent' - for owner in pod['metadata'].get('ownerReferences', []) - ) - ] - if len(node_agent_pods) != 1 or not node_agent_pods[0].get('status', {}).get('podIP'): - raise RuntimeError(f'Expected one ready Velero node-agent pod, found {len(node_agent_pods)}') - return node_agent_pods[0]['status']['podIP'] - - -def get_instances(node_agent_ip): +def get_instances(velero_host, velero_port, node_agent_host, node_agent_port): return { 'instances': [ { - 'openmetrics_endpoint': f"http://velero.velero.svc.cluster.local:{PORT}/metrics", + 'openmetrics_endpoint': f"http://{velero_host}:{velero_port}/metrics", 'tags': ['test:tag'], }, { - 'openmetrics_endpoint': f"http://{node_agent_ip}:{PORT}/metrics", + 'openmetrics_endpoint': f"http://{node_agent_host}:{node_agent_port}/metrics", 'tags': ['test:tag'], }, ] @@ -98,27 +75,28 @@ def get_instances(node_agent_ip): @pytest.fixture(scope='session') def dd_environment(): + kind_config = os.path.join(KIND_DIR, 'kind-config.yaml') custom_kubectl_image_tag = "custom-kubectl:latest" with TempDir('helm_dir') as helm_dir: with kind_run( wrappers=[build_and_load_kubectl_image(custom_kubectl_image_tag)], conditions=[KindLoad(custom_kubectl_image_tag), setup_velero], + kind_config=kind_config, env_vars={ "HELM_CACHE_HOME": path_join(helm_dir, 'Caches'), "HELM_CONFIG_HOME": path_join(helm_dir, 'Preferences'), }, ) as kubeconfig: - instances = get_instances(get_state(NODE_AGENT_IP_STATE)) - metadata = { - 'agent_type': 'kubernetes', - 'kubernetes': { - 'kubeconfig': kubeconfig, - 'auto_conf': os.path.join(CHECK_ROOT, 'datadog_checks', 'velero', 'data', 'auto_conf.yaml'), - }, - } - - yield instances, metadata + with ExitStack() as stack: + ip_ports = [ + stack.enter_context(port_forward(kubeconfig, 'velero', PORT, ressource, name)) + for ressource, name in [('service', 'velero'), ('daemonset', 'node-agent')] + ] + + instances = get_instances(ip_ports[0][0], ip_ports[0][1], ip_ports[1][0], ip_ports[1][1]) + + yield instances @pytest.fixture diff --git a/velero/tests/kind/kind-config.yaml b/velero/tests/kind/kind-config.yaml new file mode 100644 index 0000000000000..93ad325fe45a3 --- /dev/null +++ b/velero/tests/kind/kind-config.yaml @@ -0,0 +1,7 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 8085 + hostPort: 8085 \ No newline at end of file diff --git a/velero/tests/test_discovery.py b/velero/tests/test_discovery.py deleted file mode 100644 index b0db30795eeaf..0000000000000 --- a/velero/tests/test_discovery.py +++ /dev/null @@ -1,24 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) -from datadog_checks.base.utils.discovery import Port, Service -from datadog_checks.velero import VeleroCheck - - -def test_generated_discovery_uses_named_metrics_ports(): - service = Service( - id='velero', - host='10.0.0.1', - ports=( - Port(number=8085, name='metrics'), - Port(number=9999, name='admin'), - Port(number=8086, name='http-monitoring'), - ), - ) - - configs = list(VeleroCheck.generate_configs(service)) - - assert [config['instances'][0]['openmetrics_endpoint'] for config in configs] == [ - 'http://10.0.0.1:8086/metrics', - 'http://10.0.0.1:8085/metrics', - ] diff --git a/velero/tests/test_e2e.py b/velero/tests/test_e2e.py index 39181929b8d0f..8067bb6f0e82a 100644 --- a/velero/tests/test_e2e.py +++ b/velero/tests/test_e2e.py @@ -8,7 +8,10 @@ from .common import OPTIONAL_METRICS, TEST_METRICS -def assert_metrics(aggregator): +@pytest.mark.e2e +def test_check_velero_e2e(dd_agent_check): + aggregator = dd_agent_check(rate=True) + for metric, _ in TEST_METRICS.items(): if metric in OPTIONAL_METRICS: aggregator.assert_metric(name=metric, at_least=0) @@ -16,14 +19,3 @@ def assert_metrics(aggregator): aggregator.assert_metric(name=metric, at_least=1) aggregator.assert_service_check('velero.openmetrics.health', ServiceCheck.OK) - - -@pytest.mark.e2e -def test_check_velero_e2e(dd_agent_check): - assert_metrics(dd_agent_check(rate=True)) - - -@pytest.mark.e2e -def test_check_velero_discovery_e2e(dd_agent_check_discovery): - aggregator = dd_agent_check_discovery(check_rate=True, discovery_min_instances=2) - assert_metrics(aggregator) From 23d19476eebb823c64086aa8063bd26b89698e52 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Wed, 22 Jul 2026 13:05:55 +0000 Subject: [PATCH 08/50] Encapsulate Agent backend startup policy --- ddev/src/ddev/cli/env/agent.py | 6 ++---- ddev/src/ddev/cli/env/logs.py | 6 ++---- ddev/src/ddev/cli/env/reload.py | 6 ++---- ddev/src/ddev/cli/env/shell.py | 6 ++---- ddev/src/ddev/cli/env/show.py | 4 ++-- ddev/src/ddev/cli/env/start.py | 22 +++++++--------------- ddev/src/ddev/cli/env/stop.py | 7 +++---- ddev/src/ddev/e2e/agent/__init__.py | 18 +++++++++++++++++- ddev/src/ddev/e2e/agent/docker.py | 2 ++ ddev/src/ddev/e2e/agent/interface.py | 15 +++++++++++++++ ddev/src/ddev/e2e/agent/kubernetes.py | 9 +++++++++ ddev/src/ddev/e2e/agent/vagrant.py | 2 ++ 12 files changed, 65 insertions(+), 38 deletions(-) diff --git a/ddev/src/ddev/cli/env/agent.py b/ddev/src/ddev/cli/env/agent.py index 159bc2579990b..c159207367fdc 100644 --- a/ddev/src/ddev/cli/env/agent.py +++ b/ddev/src/ddev/cli/env/agent.py @@ -79,9 +79,8 @@ def agent( """ import subprocess - from ddev.e2e.agent import get_agent_interface + from ddev.e2e.agent import create_agent from ddev.e2e.config import EnvDataStorage - from ddev.e2e.constants import DEFAULT_AGENT_TYPE, E2EMetadata from ddev.utils.fs import Path integration = app.repo.integrations.get(intg_name) @@ -91,8 +90,7 @@ def agent( app.abort(f'Environment `{environment}` for integration `{integration.name}` is not running') metadata = env_data.read_metadata() - agent_type = metadata.get(E2EMetadata.AGENT_TYPE, DEFAULT_AGENT_TYPE) - agent = get_agent_interface(agent_type)(app, integration, environment, metadata, env_data.config_file) + agent = create_agent(app, integration, environment, metadata, env_data.config_file) full_args = list(args) trigger_run = False diff --git a/ddev/src/ddev/cli/env/logs.py b/ddev/src/ddev/cli/env/logs.py index 052d37ef8be23..67cd3dd663a03 100644 --- a/ddev/src/ddev/cli/env/logs.py +++ b/ddev/src/ddev/cli/env/logs.py @@ -17,9 +17,8 @@ @click.pass_obj def logs(app: Application, *, intg_name: str, environment: str): """Show backend-specific diagnostics for the Agent.""" - from ddev.e2e.agent import get_agent_interface + from ddev.e2e.agent import create_agent from ddev.e2e.config import EnvDataStorage - from ddev.e2e.constants import DEFAULT_AGENT_TYPE, E2EMetadata integration = app.repo.integrations.get(intg_name) env_data = EnvDataStorage(app.data_dir).get(integration.name, environment) @@ -28,8 +27,7 @@ def logs(app: Application, *, intg_name: str, environment: str): app.abort(f'Environment `{environment}` for integration `{integration.name}` is not running') metadata = env_data.read_metadata() - agent_type = metadata.get(E2EMetadata.AGENT_TYPE, DEFAULT_AGENT_TYPE) - agent = get_agent_interface(agent_type)(app, integration, environment, metadata, env_data.config_file) + agent = create_agent(app, integration, environment, metadata, env_data.config_file) try: agent.show_logs() diff --git a/ddev/src/ddev/cli/env/reload.py b/ddev/src/ddev/cli/env/reload.py index 284c8c1e50563..786587303ed1f 100644 --- a/ddev/src/ddev/cli/env/reload.py +++ b/ddev/src/ddev/cli/env/reload.py @@ -19,9 +19,8 @@ def reload_command(app: Application, *, intg_name: str, environment: str): """ Restart the Agent to detect environment changes. """ - from ddev.e2e.agent import get_agent_interface + from ddev.e2e.agent import create_agent from ddev.e2e.config import EnvDataStorage - from ddev.e2e.constants import DEFAULT_AGENT_TYPE, E2EMetadata integration = app.repo.integrations.get(intg_name) env_data = EnvDataStorage(app.data_dir).get(integration.name, environment) @@ -30,8 +29,7 @@ def reload_command(app: Application, *, intg_name: str, environment: str): app.abort(f'Environment `{environment}` for integration `{integration.name}` is not running') metadata = env_data.read_metadata() - agent_type = metadata.get(E2EMetadata.AGENT_TYPE, DEFAULT_AGENT_TYPE) - agent = get_agent_interface(agent_type)(app, integration, environment, metadata, env_data.config_file) + agent = create_agent(app, integration, environment, metadata, env_data.config_file) try: agent.restart() diff --git a/ddev/src/ddev/cli/env/shell.py b/ddev/src/ddev/cli/env/shell.py index 21ce3f864cf89..f28b5974fa0bb 100644 --- a/ddev/src/ddev/cli/env/shell.py +++ b/ddev/src/ddev/cli/env/shell.py @@ -21,9 +21,8 @@ def shell(app: Application, *, intg_name: str, environment: str): """ import subprocess - from ddev.e2e.agent import get_agent_interface + from ddev.e2e.agent import create_agent from ddev.e2e.config import EnvDataStorage - from ddev.e2e.constants import DEFAULT_AGENT_TYPE, E2EMetadata integration = app.repo.integrations.get(intg_name) env_data = EnvDataStorage(app.data_dir).get(integration.name, environment) @@ -32,8 +31,7 @@ def shell(app: Application, *, intg_name: str, environment: str): app.abort(f'Environment `{environment}` for integration `{integration.name}` is not running') metadata = env_data.read_metadata() - agent_type = metadata.get(E2EMetadata.AGENT_TYPE, DEFAULT_AGENT_TYPE) - agent = get_agent_interface(agent_type)(app, integration, environment, metadata, env_data.config_file) + agent = create_agent(app, integration, environment, metadata, env_data.config_file) try: agent.enter_shell() diff --git a/ddev/src/ddev/cli/env/show.py b/ddev/src/ddev/cli/env/show.py index fca3946d8d23a..3953227dfcd51 100644 --- a/ddev/src/ddev/cli/env/show.py +++ b/ddev/src/ddev/cli/env/show.py @@ -72,7 +72,7 @@ def show(app: Application, *, intg_name: str | None, environment: str | None, fo app.display_table('Available', available_columns, show_lines=True, force_ascii=force_ascii) # Display information about a specific environment else: - from ddev.e2e.agent import get_agent_interface + from ddev.e2e.agent import create_agent integration = app.repo.integrations.get(intg_name) env_data = storage.get(integration.name, environment) @@ -82,7 +82,7 @@ def show(app: Application, *, intg_name: str | None, environment: str | None, fo metadata = env_data.read_metadata() agent_type = metadata.get(E2EMetadata.AGENT_TYPE, DEFAULT_AGENT_TYPE) - agent = get_agent_interface(agent_type)(app, integration, environment, metadata, env_data.config_file) + agent = create_agent(app, integration, environment, metadata, env_data.config_file) app.display_pair('Agent type', agent_type) app.display_pair('Agent ID', agent.get_id()) diff --git a/ddev/src/ddev/cli/env/start.py b/ddev/src/ddev/cli/env/start.py index f86ff42537daa..06e85206eb22d 100644 --- a/ddev/src/ddev/cli/env/start.py +++ b/ddev/src/ddev/cli/env/start.py @@ -133,12 +133,6 @@ def start( metadata = result['metadata'] agent_type = metadata.get(E2EMetadata.AGENT_TYPE, DEFAULT_AGENT_TYPE) - if agent_type == 'kubernetes': - from secrets import token_hex - - # Persist a unique owner before backend startup so error cleanup cannot - # delete resources belonging to another environment with the same name. - metadata['_kubernetes_owner_id'] = token_hex(16) # TODO Remove once we have migrated the `docker_run` function if serialized_volumes := metadata.get(E2EMetadata.ENV_VARS, {}).get(E2EEnvVars.DOCKER_VOLUMES): @@ -146,21 +140,19 @@ def start( volumes.extend(deserialize_data(serialized_volumes)) metadata[E2EMetadata.DOCKER_VOLUMES] = volumes + agent_class = get_agent_interface(agent_type) + if running_on_ci() and not agent_class.supports_ci: + app.abort(text=f'{agent_type.capitalize()} is not supported on CI', code=0) + + agent = agent_class(app, integration, environment, metadata, env_data.config_file) + agent.prepare_start() env_data.write_metadata(metadata) config = result['config'] env_data.write_config(config) - if agent_type == "vagrant" and running_on_ci(): - app.abort(text="Vagrant is not supported on CI", code=0) - - agent = get_agent_interface(agent_type)(app, integration, environment, metadata, env_data.config_file) - if not agent_build: - configured_agent_build = app.config.agent.config.get(agent_type) - if configured_agent_build is None and agent_type == 'kubernetes': - # Kubernetes uses the same container image as the Docker backend. - configured_agent_build = app.config.agent.config.get('docker') + configured_agent_build = agent.get_configured_build(app.config.agent.config) agent_build = ( os.getenv(E2EEnvVars.AGENT_BUILD_PY2 if agent.python_version[0] == 2 else E2EEnvVars.AGENT_BUILD) or configured_agent_build diff --git a/ddev/src/ddev/cli/env/stop.py b/ddev/src/ddev/cli/env/stop.py index e20b33f74f88d..f5ba1f0a020c4 100644 --- a/ddev/src/ddev/cli/env/stop.py +++ b/ddev/src/ddev/cli/env/stop.py @@ -20,9 +20,9 @@ def stop(app: Application, *, intg_name: str, environment: str, ignore_state: bo """ Stop environments. To stop all the running environments, use `all` as the integration name and the environment. """ - from ddev.e2e.agent import get_agent_interface + from ddev.e2e.agent import create_agent from ddev.e2e.config import EnvDataStorage - from ddev.e2e.constants import DEFAULT_AGENT_TYPE, E2EEnvVars, E2EMetadata + from ddev.e2e.constants import E2EEnvVars, E2EMetadata from ddev.e2e.run import E2EEnvironmentRunner from ddev.utils.fs import temp_directory @@ -63,8 +63,7 @@ def stop(app: Application, *, intg_name: str, environment: str, ignore_state: bo metadata = env_data.read_metadata() env_vars.update(metadata.get(E2EMetadata.ENV_VARS, {})) - agent_type = metadata.get(E2EMetadata.AGENT_TYPE, DEFAULT_AGENT_TYPE) - agent = get_agent_interface(agent_type)(app, integration, env, metadata, env_data.config_file) + agent = create_agent(app, integration, env, metadata, env_data.config_file) try: agent.stop() diff --git a/ddev/src/ddev/e2e/agent/__init__.py b/ddev/src/ddev/e2e/agent/__init__.py index a7a04fd0b7f09..0aebf65695075 100644 --- a/ddev/src/ddev/e2e/agent/__init__.py +++ b/ddev/src/ddev/e2e/agent/__init__.py @@ -3,10 +3,26 @@ # Licensed under a 3-clause BSD style license (see LICENSE) from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: + from ddev.cli.application import Application from ddev.e2e.agent.interface import AgentInterface + from ddev.integration.core import Integration + from ddev.utils.fs import Path + + +def create_agent( + app: Application, + integration: Integration, + environment: str, + metadata: dict[str, Any], + config_file: Path, +) -> AgentInterface: + from ddev.e2e.constants import DEFAULT_AGENT_TYPE, E2EMetadata + + agent_type = metadata.get(E2EMetadata.AGENT_TYPE, DEFAULT_AGENT_TYPE) + return get_agent_interface(agent_type)(app, integration, environment, metadata, config_file) def get_agent_interface(agent_type: str) -> type[AgentInterface]: diff --git a/ddev/src/ddev/e2e/agent/docker.py b/ddev/src/ddev/e2e/agent/docker.py index 37a90fc7dacde..319cb12fba5ab 100644 --- a/ddev/src/ddev/e2e/agent/docker.py +++ b/ddev/src/ddev/e2e/agent/docker.py @@ -82,6 +82,8 @@ def _normalize_agent_image_name(agent_build: str | None, python_major: int, use_ class DockerAgent(AgentInterface): + build_config_keys = ('docker',) + @cached_property def _isatty(self) -> bool: isatty: Callable[[], bool] | None = getattr(sys.stdout, 'isatty', None) diff --git a/ddev/src/ddev/e2e/agent/interface.py b/ddev/src/ddev/e2e/agent/interface.py index c0592bbc1443d..85047b3a3fb63 100644 --- a/ddev/src/ddev/e2e/agent/interface.py +++ b/ddev/src/ddev/e2e/agent/interface.py @@ -8,6 +8,8 @@ from typing import TYPE_CHECKING, Any if TYPE_CHECKING: + from collections.abc import Mapping + from ddev.cli.application import Application from ddev.integration.core import Integration from ddev.utils.fs import Path @@ -15,6 +17,9 @@ class AgentInterface(ABC): + build_config_keys: tuple[str, ...] = () + supports_ci = True + def __init__( self, app: Application, integration: Integration, env: str, metadata: dict[str, Any], config_file: Path ) -> None: @@ -63,6 +68,16 @@ def python_version(self) -> tuple[int, int]: def get_id(self) -> str: return f'{self.integration.name}_{self.env}' + def prepare_start(self) -> None: + """Populate backend state that must be persisted before startup.""" + + def get_configured_build(self, config: Mapping[str, str]) -> str | None: + for key in self.build_config_keys: + if agent_build := config.get(key): + return agent_build + + return None + @abstractmethod def start(self, *, agent_build: str, local_packages: dict[Path, str], env_vars: dict[str, str]) -> None: ... diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index ea171e1433e45..71644b9283d99 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -35,6 +35,15 @@ class KubernetesAgent(AgentInterface): provisioner (for example, ``kind_run``), not this backend. """ + build_config_keys = ('kubernetes', 'docker') + + def prepare_start(self) -> None: + from secrets import token_hex + + # Persist a unique owner before backend startup so error cleanup cannot + # delete resources belonging to another environment with the same name. + self.metadata[_OWNER_ID_METADATA] = token_hex(16) + @cached_property def _kubernetes_metadata(self) -> dict[str, Any]: metadata = self.metadata.get('kubernetes') diff --git a/ddev/src/ddev/e2e/agent/vagrant.py b/ddev/src/ddev/e2e/agent/vagrant.py index f79e7e1a911dd..368132fc77ea8 100644 --- a/ddev/src/ddev/e2e/agent/vagrant.py +++ b/ddev/src/ddev/e2e/agent/vagrant.py @@ -57,6 +57,8 @@ def disable_integration_before_install(config_file: Path): class VagrantAgent(AgentInterface): + build_config_keys = ('vagrant',) + supports_ci = False VM_HOST_IP = "172.30.1.5" def __init__( From 04d2bc51e6a2c3d3b02c95249d573f03745748f7 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Wed, 22 Jul 2026 13:08:23 +0000 Subject: [PATCH 09/50] Add Kubernetes discovery candidate stability helper --- .../datadog_checks/dev/kubernetes.py | 254 ++++++++++++++++++ datadog_checks_dev/tests/test_kubernetes.py | 209 ++++++++++++++ 2 files changed, 463 insertions(+) create mode 100644 datadog_checks_dev/datadog_checks/dev/kubernetes.py create mode 100644 datadog_checks_dev/tests/test_kubernetes.py diff --git a/datadog_checks_dev/datadog_checks/dev/kubernetes.py b/datadog_checks_dev/datadog_checks/dev/kubernetes.py new file mode 100644 index 0000000000000..5f7c71051674f --- /dev/null +++ b/datadog_checks_dev/datadog_checks/dev/kubernetes.py @@ -0,0 +1,254 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import json +import logging +import os +import re +from collections.abc import Callable, Mapping, Sequence +from types import SimpleNamespace +from typing import Any + +from .docker import CONTAINER_STABILITY_LOG_PATTERNS +from .subprocess import run_command + + +def assert_all_discovery_candidates_stable_kubernetes( + dd_agent_check: Callable[..., Any], + check_cls: type[Any], + kubeconfig: str | os.PathLike[str], + *, + namespace: str, + pod_name: str | None = None, + pod_selector: str | None = None, + container_name: str | None = None, + service_id: str | None = None, + dd_agent_check_kwargs: Mapping[str, Any] | None = None, + log_patterns: Sequence[str] = CONTAINER_STABILITY_LOG_PATTERNS, +) -> None: + """Run generated discovery candidates and assert that the target Kubernetes pod stays stable.""" + if bool(pod_name) == bool(pod_selector): + raise TypeError('Exactly one of `pod_name` or `pod_selector` must be provided') + + if pod_selector: + initial_state = _get_selected_pod(kubeconfig, namespace, pod_selector) + pod_name = initial_state['metadata']['name'] + else: + assert pod_name is not None + initial_state = _get_pod(kubeconfig, namespace, pod_name) + + previous_logs = _get_pod_logs(kubeconfig, namespace, initial_state) + service = _build_service_from_pod(initial_state, service_id, container_name=container_name) + candidates = tuple(check_cls.generate_configs(service)) + if not candidates: + raise AssertionError(f'No discovery candidates generated for service {service.id!r}') + + check_kwargs = {'check_rate': True} + if dd_agent_check_kwargs: + check_kwargs.update(dd_agent_check_kwargs) + + for index, candidate in enumerate(candidates, 1): + logging.debug('Probing candidate #%d: %r', index, candidate) + try: + dd_agent_check(candidate, **check_kwargs) + except Exception: + # A failed check may still have contacted the workload. As with the Docker helper, + # candidate errors do not skip the workload stability assertions. + logging.debug('Error probing candidate #%d: %r', index, candidate, exc_info=True) + + if pod_selector: + current_state = _get_selected_pod(kubeconfig, namespace, pod_selector) + else: + current_state = _get_pod(kubeconfig, namespace, pod_name) + _assert_pod_stable(initial_state, current_state, index) + + current_logs = _get_pod_logs(kubeconfig, namespace, current_state) + _assert_no_new_log_patterns(previous_logs, current_logs, log_patterns, index) + previous_logs = current_logs + + +def _run_kubectl(kubeconfig: str | os.PathLike[str], args: Sequence[str], **kwargs: Any) -> Any: + return run_command(['kubectl', '--kubeconfig', os.fspath(kubeconfig), *args], **kwargs) + + +def _get_selected_pod(kubeconfig: str | os.PathLike[str], namespace: str, pod_selector: str) -> dict[str, Any]: + result = _run_kubectl( + kubeconfig, + ['get', 'pods', '--namespace', namespace, '--selector', pod_selector, '--output', 'json'], + capture='out', + check=True, + ) + pods = [pod for pod in json.loads(result.stdout)['items'] if not pod.get('metadata', {}).get('deletionTimestamp')] + if len(pods) != 1: + raise AssertionError( + f'Expected exactly one active pod in namespace {namespace!r} matching selector {pod_selector!r}, ' + f'found {len(pods)}' + ) + return pods[0] + + +def _get_pod(kubeconfig: str | os.PathLike[str], namespace: str, pod_name: str) -> dict[str, Any]: + result = _run_kubectl( + kubeconfig, + ['get', 'pod', pod_name, '--namespace', namespace, '--output', 'json'], + capture='out', + check=True, + ) + return json.loads(result.stdout) + + +def _get_pod_logs( + kubeconfig: str | os.PathLike[str], namespace: str, pod: Mapping[str, Any] +) -> dict[str, tuple[str, str]]: + pod_name = pod['metadata']['name'] + containers = [ + *pod.get('spec', {}).get('initContainers', []), + *pod.get('spec', {}).get('containers', []), + *pod.get('spec', {}).get('ephemeralContainers', []), + ] + logs = {} + for container in containers: + container_name = container['name'] + result = _run_kubectl( + kubeconfig, + ['logs', pod_name, '--namespace', namespace, '--container', container_name], + capture=True, + check=False, + ) + if result.code != 0: + raise AssertionError( + f'Could not read logs for container {container_name!r} in pod {pod_name!r}: ' + f'{result.stdout}{result.stderr}' + ) + logs[container_name] = (result.stdout, result.stderr) + return logs + + +def _build_service_from_pod( + pod: Mapping[str, Any], service_id: str | None, *, container_name: str | None = None +) -> SimpleNamespace: + host = pod.get('status', {}).get('podIP') + if not host: + raise AssertionError(f'Pod {pod.get("metadata", {}).get("name", "")!r} has no IP address') + + containers = pod.get('spec', {}).get('containers', []) + if container_name is None: + if len(containers) != 1: + raise AssertionError('`container_name` is required when the pod has more than one regular container') + container = containers[0] + else: + container = next((candidate for candidate in containers if candidate['name'] == container_name), None) + if container is None: + raise AssertionError(f'Pod has no regular container named {container_name!r}') + + ports = tuple( + sorted( + ( + SimpleNamespace(number=port_spec['containerPort'], name=port_spec.get('name', '')) + for port_spec in container.get('ports', []) + ), + key=lambda port: port.number, + ) + ) + + if service_id is None: + container_status = next( + ( + status + for status in pod.get('status', {}).get('containerStatuses', []) + if status['name'] == container['name'] + ), + None, + ) + service_id = container_status.get('containerID') if container_status else None + if not service_id: + raise AssertionError(f'Container {container["name"]!r} has no runtime ID') + + return SimpleNamespace(id=service_id, host=host, ports=ports) + + +def _assert_pod_stable(initial: Mapping[str, Any], current: Mapping[str, Any], candidate_index: int) -> None: + initial_uid = initial['metadata']['uid'] + current_uid = current['metadata']['uid'] + if current_uid != initial_uid: + raise AssertionError(f'Pod changed while probing candidate #{candidate_index}: {initial_uid} -> {current_uid}') + if current.get('metadata', {}).get('deletionTimestamp'): + raise AssertionError(f'Pod is terminating after probing candidate #{candidate_index}') + + phase = current.get('status', {}).get('phase') + if phase != 'Running': + raise AssertionError(f'Pod phase is {phase!r} after probing candidate #{candidate_index}') + + ready = next( + ( + condition.get('status') == 'True' + for condition in current.get('status', {}).get('conditions', []) + if condition.get('type') == 'Ready' + ), + False, + ) + if not ready: + raise AssertionError(f'Pod is not ready after probing candidate #{candidate_index}') + + for status_key in ('initContainerStatuses', 'containerStatuses', 'ephemeralContainerStatuses'): + initial_statuses = {status['name']: status for status in initial.get('status', {}).get(status_key, [])} + current_statuses = {status['name']: status for status in current.get('status', {}).get(status_key, [])} + if current_statuses.keys() != initial_statuses.keys(): + raise AssertionError(f'Pod containers changed while probing candidate #{candidate_index}') + + for name, current_status in current_statuses.items(): + initial_status = initial_statuses[name] + initial_container_id = initial_status.get('containerID') + current_container_id = current_status.get('containerID') + if current_container_id != initial_container_id: + raise AssertionError( + f'Container {name!r} changed while probing candidate #{candidate_index}: ' + f'{initial_container_id} -> {current_container_id}' + ) + + initial_restarts = initial_status.get('restartCount', 0) + current_restarts = current_status.get('restartCount', 0) + if current_restarts != initial_restarts: + raise AssertionError( + f'Container {name!r} restart count changed while probing candidate #{candidate_index}: ' + f'{initial_restarts} -> {current_restarts}' + ) + if status_key == 'containerStatuses' and not current_status.get('ready', False): + raise AssertionError(f'Container {name!r} is not ready after probing candidate #{candidate_index}') + + for state_key in ('state', 'lastState'): + initial_terminated = initial_status.get(state_key, {}).get('terminated') + current_terminated = current_status.get(state_key, {}).get('terminated') + if current_terminated is not None and initial_terminated is None: + reason = current_terminated.get('reason') or '' + raise AssertionError( + f'Container {name!r} terminated with reason {reason!r} ' + f'after probing candidate #{candidate_index}' + ) + + +def _assert_no_new_log_patterns( + previous: Mapping[str, tuple[str, str]], + current: Mapping[str, tuple[str, str]], + patterns: Sequence[str], + candidate_index: int, +) -> None: + if current.keys() != previous.keys(): + raise AssertionError(f'Pod log streams changed while probing candidate #{candidate_index}') + + for container_name, (current_stdout, current_stderr) in current.items(): + previous_stdout, previous_stderr = previous[container_name] + new_logs = _diff_logs(previous_stdout, current_stdout) + _diff_logs(previous_stderr, current_stderr) + for line in new_logs.splitlines(): + logging.debug('New log line from container %s: %s', container_name, line) + for pattern in patterns: + match = re.search(pattern, new_logs, re.IGNORECASE) + if match: + raise AssertionError( + f'Pod logs for container {container_name!r} matched {pattern!r} after probing ' + f'candidate #{candidate_index}: {match.group(0)!r}' + ) + + +def _diff_logs(previous: str, current: str) -> str: + return current[len(previous) :] if current.startswith(previous) else current diff --git a/datadog_checks_dev/tests/test_kubernetes.py b/datadog_checks_dev/tests/test_kubernetes.py new file mode 100644 index 0000000000000..2c0d7417ecec5 --- /dev/null +++ b/datadog_checks_dev/tests/test_kubernetes.py @@ -0,0 +1,209 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +import copy +import json +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from datadog_checks.dev import kubernetes + + +def _pod( + *, + uid='pod-uid', + container_id='containerd://velero-id', + restart_count=0, + ready=True, + phase='Running', + last_reason=None, +): + return { + 'metadata': {'name': 'velero-123', 'uid': uid}, + 'spec': { + 'containers': [ + { + 'name': 'velero', + 'ports': [ + {'containerPort': 8085, 'name': 'http-monitoring'}, + {'containerPort': 9999, 'name': 'admin'}, + ], + } + ] + }, + 'status': { + 'phase': phase, + 'podIP': '10.0.0.1', + 'conditions': [{'type': 'Ready', 'status': 'True' if ready else 'False'}], + 'containerStatuses': [ + { + 'name': 'velero', + 'containerID': container_id, + 'ready': ready, + 'restartCount': restart_count, + 'lastState': {'terminated': {'reason': last_reason}} if last_reason else {}, + } + ], + }, + } + + +def _result(data='', *, stderr='', code=0): + return SimpleNamespace(stdout=data, stderr=stderr, code=code) + + +def _mock_kubectl(monkeypatch, *, initial=None, current=None, initial_logs='old\n', current_logs='old\nnew\n'): + pod_states = iter((initial or _pod(), current or _pod())) + logs = iter((initial_logs, current_logs)) + + def run_command(command, **kwargs): + assert command[:3] == ['kubectl', '--kubeconfig', '/tmp/kubeconfig'] + if command[3:5] == ['get', 'pods']: + return _result(json.dumps({'items': [next(pod_states)]})) + if command[3:5] == ['get', 'pod']: + return _result(json.dumps(next(pod_states))) + if command[3] == 'logs': + return _result(next(logs)) + raise AssertionError(f'Unexpected command: {command}') + + monkeypatch.setattr(kubernetes, 'run_command', run_command) + + +class _Check: + services = [] + + @classmethod + def generate_configs(cls, service): + cls.services.append(service) + return [ + { + 'init_config': {}, + 'instances': [{'openmetrics_endpoint': f'http://{service.host}:{port.number}/metrics'}], + } + for port in service.ports + if port.name == 'http-monitoring' + ] + + +def test_assert_all_discovery_candidates_stable_kubernetes(monkeypatch): + _mock_kubectl(monkeypatch) + dd_agent_check = Mock() + _Check.services = [] + + kubernetes.assert_all_discovery_candidates_stable_kubernetes( + dd_agent_check, + _Check, + '/tmp/kubeconfig', + namespace='velero', + pod_selector='name=velero', + ) + + service = _Check.services[0] + assert service.id == 'containerd://velero-id' + assert service.host == '10.0.0.1' + assert [(port.number, port.name) for port in service.ports] == [ + (8085, 'http-monitoring'), + (9999, 'admin'), + ] + dd_agent_check.assert_called_once_with( + { + 'init_config': {}, + 'instances': [{'openmetrics_endpoint': 'http://10.0.0.1:8085/metrics'}], + }, + check_rate=True, + ) + + +def test_candidate_error_still_checks_workload_stability(monkeypatch): + _mock_kubectl(monkeypatch, current=_pod(restart_count=1)) + dd_agent_check = Mock(side_effect=RuntimeError('check failed')) + + with pytest.raises(AssertionError, match='restart count changed'): + kubernetes.assert_all_discovery_candidates_stable_kubernetes( + dd_agent_check, + _Check, + '/tmp/kubeconfig', + namespace='velero', + pod_name='velero-123', + ) + + +def test_new_dangerous_workload_logs_fail(monkeypatch): + _mock_kubectl(monkeypatch, current_logs='old\npanic: crashed\n') + + with pytest.raises(AssertionError, match="Pod logs for container 'velero' matched 'panic'"): + kubernetes.assert_all_discovery_candidates_stable_kubernetes( + Mock(), + _Check, + '/tmp/kubeconfig', + namespace='velero', + pod_name='velero-123', + ) + + +def test_selector_detects_replacement_pod(monkeypatch): + _mock_kubectl(monkeypatch, current=_pod(uid='replacement-uid')) + + with pytest.raises(AssertionError, match='Pod changed'): + kubernetes.assert_all_discovery_candidates_stable_kubernetes( + Mock(), + _Check, + '/tmp/kubeconfig', + namespace='velero', + pod_selector='name=velero', + ) + + +def test_service_uses_only_target_container_and_preserves_named_ports(): + pod = _pod() + pod['spec']['containers'][0]['ports'].append({'containerPort': 8085, 'name': 'metrics'}) + pod['spec']['containers'].append({'name': 'sidecar', 'ports': [{'containerPort': 9000, 'name': 'sidecar-metrics'}]}) + + with pytest.raises(AssertionError, match='container_name'): + kubernetes._build_service_from_pod(pod, 'svc') + + service = kubernetes._build_service_from_pod(pod, 'svc', container_name='velero') + assert [(port.number, port.name) for port in service.ports] == [ + (8085, 'http-monitoring'), + (8085, 'metrics'), + (9999, 'admin'), + ] + + +def test_termination_without_reason_is_detected(): + initial = _pod() + current = copy.deepcopy(initial) + current['status']['containerStatuses'][0]['state'] = {'terminated': {'exitCode': 1}} + + with pytest.raises(AssertionError, match="terminated with reason ''"): + kubernetes._assert_pod_stable(initial, current, 1) + + +def test_log_streams_are_diffed_independently(): + kubernetes._assert_no_new_log_patterns( + {'velero': ('old error\n', 'old warning\n')}, + {'velero': ('old error\n', 'old warning\nnew healthy line\n')}, + ('error',), + 1, + ) + + +@pytest.mark.parametrize( + ('pod_name', 'pod_selector'), + [ + (None, None), + ('velero-123', 'name=velero'), + ], +) +def test_exactly_one_pod_identifier_is_required(pod_name, pod_selector): + with pytest.raises(TypeError, match='Exactly one'): + kubernetes.assert_all_discovery_candidates_stable_kubernetes( + Mock(), + _Check, + '/tmp/kubeconfig', + namespace='velero', + pod_name=pod_name, + pod_selector=pod_selector, + ) From fb00f3e616836e0fb6a64202f34c2a93d6f86886 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Wed, 22 Jul 2026 13:26:40 +0000 Subject: [PATCH 10/50] Document Kubernetes candidate stability support --- datadog_checks_dev/changelog.d/24639.added | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datadog_checks_dev/changelog.d/24639.added b/datadog_checks_dev/changelog.d/24639.added index d4a5b54b8f258..fbb612fe4e935 100644 --- a/datadog_checks_dev/changelog.d/24639.added +++ b/datadog_checks_dev/changelog.d/24639.added @@ -1 +1 @@ -Use backend-neutral Agent log retrieval for E2E test diagnostics. +Add Kubernetes discovery candidate-stability checks and backend-neutral Agent log retrieval for E2E tests. From 154ce6a5c222969318a659d854fd07b8a2726a1d Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 18:10:10 +0000 Subject: [PATCH 11/50] Clean up unsupported CI environments --- ddev/src/ddev/cli/env/start.py | 9 +++++ ddev/src/ddev/cli/env/stop.py | 6 ++-- ddev/src/ddev/e2e/constants.py | 1 + ddev/tests/cli/env/test_start.py | 60 ++++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 2 deletions(-) diff --git a/ddev/src/ddev/cli/env/start.py b/ddev/src/ddev/cli/env/start.py index 06e85206eb22d..1ffe38fa541d4 100644 --- a/ddev/src/ddev/cli/env/start.py +++ b/ddev/src/ddev/cli/env/start.py @@ -142,6 +142,15 @@ def start( agent_class = get_agent_interface(agent_type) if running_on_ci() and not agent_class.supports_ci: + from ddev.cli.env.stop import stop + + # The pytest fixture has already provisioned its environment by the time it returns Agent + # metadata. Persist enough state for `stop` to tear the fixture down, while recording that + # no Agent backend was started. If teardown fails, a later `ddev env stop` can safely retry. + metadata[E2EMetadata.SKIP_AGENT_STOP] = True + env_data.write_metadata(metadata) + env_data.write_config(result['config']) + ctx.invoke(stop, intg_name=intg_name, environment=environment) app.abort(text=f'{agent_type.capitalize()} is not supported on CI', code=0) agent = agent_class(app, integration, environment, metadata, env_data.config_file) diff --git a/ddev/src/ddev/cli/env/stop.py b/ddev/src/ddev/cli/env/stop.py index f5ba1f0a020c4..d540fc19b61a1 100644 --- a/ddev/src/ddev/cli/env/stop.py +++ b/ddev/src/ddev/cli/env/stop.py @@ -63,10 +63,12 @@ def stop(app: Application, *, intg_name: str, environment: str, ignore_state: bo metadata = env_data.read_metadata() env_vars.update(metadata.get(E2EMetadata.ENV_VARS, {})) - agent = create_agent(app, integration, env, metadata, env_data.config_file) + skip_agent_stop = metadata.get(E2EMetadata.SKIP_AGENT_STOP, False) + agent = None if skip_agent_stop else create_agent(app, integration, env, metadata, env_data.config_file) try: - agent.stop() + if agent is not None: + agent.stop() finally: with integration.path.as_cwd(env_vars=env_vars), runner.stop() as command: process = app.platform.run_command(command) diff --git a/ddev/src/ddev/e2e/constants.py b/ddev/src/ddev/e2e/constants.py index e52a15d26a180..4924daf591a73 100644 --- a/ddev/src/ddev/e2e/constants.py +++ b/ddev/src/ddev/e2e/constants.py @@ -22,3 +22,4 @@ class E2EMetadata: AGENT_TYPE = 'agent_type' ENV_VARS = 'e2e_env_vars' DOCKER_VOLUMES = 'docker_volumes' + SKIP_AGENT_STOP = '_ddev_skip_agent_stop' diff --git a/ddev/tests/cli/env/test_start.py b/ddev/tests/cli/env/test_start.py index cc4a5576c82d4..cfbe0473fa240 100644 --- a/ddev/tests/cli/env/test_start.py +++ b/ddev/tests/cli/env/test_start.py @@ -98,6 +98,66 @@ def test_stop_on_error(ddev, helpers, data_dir, write_result_file, mocker): stop.assert_called_once() +def test_unsupported_agent_on_ci_cleans_up_environment(ddev, data_dir, write_result_file, mocker): + metadata = {'agent_type': 'vagrant'} + config = {} + run = mocker.patch('subprocess.run', side_effect=write_result_file({'metadata': metadata, 'config': config})) + mocker.patch('ddev.cli.env.start.running_on_ci', return_value=True) + stop_agent = mocker.patch('ddev.e2e.agent.vagrant.VagrantAgent.stop') + + integration = 'postgres' + environment = 'py3.12' + env_data = EnvDataStorage(data_dir).get(integration, environment) + + result = ddev('env', 'start', integration, environment) + + assert result.exit_code == 0, result.output + assert 'Starting: py3.12' in result.output + assert 'Stopping: postgres:py3.12' in result.output + assert 'Vagrant is not supported on CI' in result.output + assert not env_data.exists() + assert run.call_count == 2 + stop_agent.assert_not_called() + + +def test_unsupported_agent_failed_cleanup_can_be_retried(ddev, data_dir, write_result_file, mocker): + metadata = {'agent_type': 'vagrant'} + config = {} + write_result = write_result_file({'metadata': metadata, 'config': config}) + call_count = 0 + + def run_command(*args, **kwargs): + nonlocal call_count + call_count += 1 + result = write_result(*args, **kwargs) + if call_count == 2: + result.returncode = 1 + return result + + run = mocker.patch('subprocess.run', side_effect=run_command) + mocker.patch('ddev.cli.env.start.running_on_ci', return_value=True) + stop_agent = mocker.patch('ddev.e2e.agent.vagrant.VagrantAgent.stop') + + integration = 'postgres' + environment = 'py3.12' + env_data = EnvDataStorage(data_dir).get(integration, environment) + + result = ddev('env', 'start', integration, environment) + + assert result.exit_code == 1, result.output + assert env_data.exists() + assert env_data.read_metadata() == {**metadata, E2EMetadata.SKIP_AGENT_STOP: True} + stop_agent.assert_not_called() + + run.side_effect = None + run.return_value = mocker.MagicMock(returncode=0) + result = ddev('env', 'stop', integration, environment) + + assert result.exit_code == 0, result.output + assert not env_data.exists() + stop_agent.assert_not_called() + + def test_basic(ddev, helpers, data_dir, write_result_file, mocker): metadata = {} config = {} From 9b88ee8fb4aa89cde7738c43eb33952fe12776f0 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 18:13:42 +0000 Subject: [PATCH 12/50] Clarify Agent interface creation --- ddev/src/ddev/cli/env/agent.py | 4 ++-- ddev/src/ddev/cli/env/logs.py | 4 ++-- ddev/src/ddev/cli/env/reload.py | 4 ++-- ddev/src/ddev/cli/env/shell.py | 4 ++-- ddev/src/ddev/cli/env/show.py | 4 ++-- ddev/src/ddev/cli/env/stop.py | 8 ++++++-- ddev/src/ddev/e2e/agent/__init__.py | 2 +- 7 files changed, 17 insertions(+), 13 deletions(-) diff --git a/ddev/src/ddev/cli/env/agent.py b/ddev/src/ddev/cli/env/agent.py index c159207367fdc..c6ef4e08dbf1c 100644 --- a/ddev/src/ddev/cli/env/agent.py +++ b/ddev/src/ddev/cli/env/agent.py @@ -79,7 +79,7 @@ def agent( """ import subprocess - from ddev.e2e.agent import create_agent + from ddev.e2e.agent import create_agent_interface from ddev.e2e.config import EnvDataStorage from ddev.utils.fs import Path @@ -90,7 +90,7 @@ def agent( app.abort(f'Environment `{environment}` for integration `{integration.name}` is not running') metadata = env_data.read_metadata() - agent = create_agent(app, integration, environment, metadata, env_data.config_file) + agent = create_agent_interface(app, integration, environment, metadata, env_data.config_file) full_args = list(args) trigger_run = False diff --git a/ddev/src/ddev/cli/env/logs.py b/ddev/src/ddev/cli/env/logs.py index 67cd3dd663a03..c72c3d5280278 100644 --- a/ddev/src/ddev/cli/env/logs.py +++ b/ddev/src/ddev/cli/env/logs.py @@ -17,7 +17,7 @@ @click.pass_obj def logs(app: Application, *, intg_name: str, environment: str): """Show backend-specific diagnostics for the Agent.""" - from ddev.e2e.agent import create_agent + from ddev.e2e.agent import create_agent_interface from ddev.e2e.config import EnvDataStorage integration = app.repo.integrations.get(intg_name) @@ -27,7 +27,7 @@ def logs(app: Application, *, intg_name: str, environment: str): app.abort(f'Environment `{environment}` for integration `{integration.name}` is not running') metadata = env_data.read_metadata() - agent = create_agent(app, integration, environment, metadata, env_data.config_file) + agent = create_agent_interface(app, integration, environment, metadata, env_data.config_file) try: agent.show_logs() diff --git a/ddev/src/ddev/cli/env/reload.py b/ddev/src/ddev/cli/env/reload.py index 786587303ed1f..e302c4f33fc29 100644 --- a/ddev/src/ddev/cli/env/reload.py +++ b/ddev/src/ddev/cli/env/reload.py @@ -19,7 +19,7 @@ def reload_command(app: Application, *, intg_name: str, environment: str): """ Restart the Agent to detect environment changes. """ - from ddev.e2e.agent import create_agent + from ddev.e2e.agent import create_agent_interface from ddev.e2e.config import EnvDataStorage integration = app.repo.integrations.get(intg_name) @@ -29,7 +29,7 @@ def reload_command(app: Application, *, intg_name: str, environment: str): app.abort(f'Environment `{environment}` for integration `{integration.name}` is not running') metadata = env_data.read_metadata() - agent = create_agent(app, integration, environment, metadata, env_data.config_file) + agent = create_agent_interface(app, integration, environment, metadata, env_data.config_file) try: agent.restart() diff --git a/ddev/src/ddev/cli/env/shell.py b/ddev/src/ddev/cli/env/shell.py index f28b5974fa0bb..7a4bc5947a70b 100644 --- a/ddev/src/ddev/cli/env/shell.py +++ b/ddev/src/ddev/cli/env/shell.py @@ -21,7 +21,7 @@ def shell(app: Application, *, intg_name: str, environment: str): """ import subprocess - from ddev.e2e.agent import create_agent + from ddev.e2e.agent import create_agent_interface from ddev.e2e.config import EnvDataStorage integration = app.repo.integrations.get(intg_name) @@ -31,7 +31,7 @@ def shell(app: Application, *, intg_name: str, environment: str): app.abort(f'Environment `{environment}` for integration `{integration.name}` is not running') metadata = env_data.read_metadata() - agent = create_agent(app, integration, environment, metadata, env_data.config_file) + agent = create_agent_interface(app, integration, environment, metadata, env_data.config_file) try: agent.enter_shell() diff --git a/ddev/src/ddev/cli/env/show.py b/ddev/src/ddev/cli/env/show.py index 3953227dfcd51..5d50c5ec39821 100644 --- a/ddev/src/ddev/cli/env/show.py +++ b/ddev/src/ddev/cli/env/show.py @@ -72,7 +72,7 @@ def show(app: Application, *, intg_name: str | None, environment: str | None, fo app.display_table('Available', available_columns, show_lines=True, force_ascii=force_ascii) # Display information about a specific environment else: - from ddev.e2e.agent import create_agent + from ddev.e2e.agent import create_agent_interface integration = app.repo.integrations.get(intg_name) env_data = storage.get(integration.name, environment) @@ -82,7 +82,7 @@ def show(app: Application, *, intg_name: str | None, environment: str | None, fo metadata = env_data.read_metadata() agent_type = metadata.get(E2EMetadata.AGENT_TYPE, DEFAULT_AGENT_TYPE) - agent = create_agent(app, integration, environment, metadata, env_data.config_file) + agent = create_agent_interface(app, integration, environment, metadata, env_data.config_file) app.display_pair('Agent type', agent_type) app.display_pair('Agent ID', agent.get_id()) diff --git a/ddev/src/ddev/cli/env/stop.py b/ddev/src/ddev/cli/env/stop.py index d540fc19b61a1..c16500c377be1 100644 --- a/ddev/src/ddev/cli/env/stop.py +++ b/ddev/src/ddev/cli/env/stop.py @@ -20,7 +20,7 @@ def stop(app: Application, *, intg_name: str, environment: str, ignore_state: bo """ Stop environments. To stop all the running environments, use `all` as the integration name and the environment. """ - from ddev.e2e.agent import create_agent + from ddev.e2e.agent import create_agent_interface from ddev.e2e.config import EnvDataStorage from ddev.e2e.constants import E2EEnvVars, E2EMetadata from ddev.e2e.run import E2EEnvironmentRunner @@ -64,7 +64,11 @@ def stop(app: Application, *, intg_name: str, environment: str, ignore_state: bo env_vars.update(metadata.get(E2EMetadata.ENV_VARS, {})) skip_agent_stop = metadata.get(E2EMetadata.SKIP_AGENT_STOP, False) - agent = None if skip_agent_stop else create_agent(app, integration, env, metadata, env_data.config_file) + agent = ( + None + if skip_agent_stop + else create_agent_interface(app, integration, env, metadata, env_data.config_file) + ) try: if agent is not None: diff --git a/ddev/src/ddev/e2e/agent/__init__.py b/ddev/src/ddev/e2e/agent/__init__.py index 0aebf65695075..522d5abfc7553 100644 --- a/ddev/src/ddev/e2e/agent/__init__.py +++ b/ddev/src/ddev/e2e/agent/__init__.py @@ -12,7 +12,7 @@ from ddev.utils.fs import Path -def create_agent( +def create_agent_interface( app: Application, integration: Integration, environment: str, From 5206dbd58741ac67cce32d0049cdbb6c10c52cb4 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 18:23:21 +0000 Subject: [PATCH 13/50] refactor(e2e): move Kubernetes discovery helper to consumer PR --- datadog_checks_dev/changelog.d/24639.added | 2 +- .../datadog_checks/dev/kubernetes.py | 254 ------------------ datadog_checks_dev/tests/test_kubernetes.py | 209 -------------- 3 files changed, 1 insertion(+), 464 deletions(-) delete mode 100644 datadog_checks_dev/datadog_checks/dev/kubernetes.py delete mode 100644 datadog_checks_dev/tests/test_kubernetes.py diff --git a/datadog_checks_dev/changelog.d/24639.added b/datadog_checks_dev/changelog.d/24639.added index fbb612fe4e935..e2df056d40f0f 100644 --- a/datadog_checks_dev/changelog.d/24639.added +++ b/datadog_checks_dev/changelog.d/24639.added @@ -1 +1 @@ -Add Kubernetes discovery candidate-stability checks and backend-neutral Agent log retrieval for E2E tests. +Add backend-neutral Agent log retrieval for E2E test diagnostics. diff --git a/datadog_checks_dev/datadog_checks/dev/kubernetes.py b/datadog_checks_dev/datadog_checks/dev/kubernetes.py deleted file mode 100644 index 5f7c71051674f..0000000000000 --- a/datadog_checks_dev/datadog_checks/dev/kubernetes.py +++ /dev/null @@ -1,254 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) -import json -import logging -import os -import re -from collections.abc import Callable, Mapping, Sequence -from types import SimpleNamespace -from typing import Any - -from .docker import CONTAINER_STABILITY_LOG_PATTERNS -from .subprocess import run_command - - -def assert_all_discovery_candidates_stable_kubernetes( - dd_agent_check: Callable[..., Any], - check_cls: type[Any], - kubeconfig: str | os.PathLike[str], - *, - namespace: str, - pod_name: str | None = None, - pod_selector: str | None = None, - container_name: str | None = None, - service_id: str | None = None, - dd_agent_check_kwargs: Mapping[str, Any] | None = None, - log_patterns: Sequence[str] = CONTAINER_STABILITY_LOG_PATTERNS, -) -> None: - """Run generated discovery candidates and assert that the target Kubernetes pod stays stable.""" - if bool(pod_name) == bool(pod_selector): - raise TypeError('Exactly one of `pod_name` or `pod_selector` must be provided') - - if pod_selector: - initial_state = _get_selected_pod(kubeconfig, namespace, pod_selector) - pod_name = initial_state['metadata']['name'] - else: - assert pod_name is not None - initial_state = _get_pod(kubeconfig, namespace, pod_name) - - previous_logs = _get_pod_logs(kubeconfig, namespace, initial_state) - service = _build_service_from_pod(initial_state, service_id, container_name=container_name) - candidates = tuple(check_cls.generate_configs(service)) - if not candidates: - raise AssertionError(f'No discovery candidates generated for service {service.id!r}') - - check_kwargs = {'check_rate': True} - if dd_agent_check_kwargs: - check_kwargs.update(dd_agent_check_kwargs) - - for index, candidate in enumerate(candidates, 1): - logging.debug('Probing candidate #%d: %r', index, candidate) - try: - dd_agent_check(candidate, **check_kwargs) - except Exception: - # A failed check may still have contacted the workload. As with the Docker helper, - # candidate errors do not skip the workload stability assertions. - logging.debug('Error probing candidate #%d: %r', index, candidate, exc_info=True) - - if pod_selector: - current_state = _get_selected_pod(kubeconfig, namespace, pod_selector) - else: - current_state = _get_pod(kubeconfig, namespace, pod_name) - _assert_pod_stable(initial_state, current_state, index) - - current_logs = _get_pod_logs(kubeconfig, namespace, current_state) - _assert_no_new_log_patterns(previous_logs, current_logs, log_patterns, index) - previous_logs = current_logs - - -def _run_kubectl(kubeconfig: str | os.PathLike[str], args: Sequence[str], **kwargs: Any) -> Any: - return run_command(['kubectl', '--kubeconfig', os.fspath(kubeconfig), *args], **kwargs) - - -def _get_selected_pod(kubeconfig: str | os.PathLike[str], namespace: str, pod_selector: str) -> dict[str, Any]: - result = _run_kubectl( - kubeconfig, - ['get', 'pods', '--namespace', namespace, '--selector', pod_selector, '--output', 'json'], - capture='out', - check=True, - ) - pods = [pod for pod in json.loads(result.stdout)['items'] if not pod.get('metadata', {}).get('deletionTimestamp')] - if len(pods) != 1: - raise AssertionError( - f'Expected exactly one active pod in namespace {namespace!r} matching selector {pod_selector!r}, ' - f'found {len(pods)}' - ) - return pods[0] - - -def _get_pod(kubeconfig: str | os.PathLike[str], namespace: str, pod_name: str) -> dict[str, Any]: - result = _run_kubectl( - kubeconfig, - ['get', 'pod', pod_name, '--namespace', namespace, '--output', 'json'], - capture='out', - check=True, - ) - return json.loads(result.stdout) - - -def _get_pod_logs( - kubeconfig: str | os.PathLike[str], namespace: str, pod: Mapping[str, Any] -) -> dict[str, tuple[str, str]]: - pod_name = pod['metadata']['name'] - containers = [ - *pod.get('spec', {}).get('initContainers', []), - *pod.get('spec', {}).get('containers', []), - *pod.get('spec', {}).get('ephemeralContainers', []), - ] - logs = {} - for container in containers: - container_name = container['name'] - result = _run_kubectl( - kubeconfig, - ['logs', pod_name, '--namespace', namespace, '--container', container_name], - capture=True, - check=False, - ) - if result.code != 0: - raise AssertionError( - f'Could not read logs for container {container_name!r} in pod {pod_name!r}: ' - f'{result.stdout}{result.stderr}' - ) - logs[container_name] = (result.stdout, result.stderr) - return logs - - -def _build_service_from_pod( - pod: Mapping[str, Any], service_id: str | None, *, container_name: str | None = None -) -> SimpleNamespace: - host = pod.get('status', {}).get('podIP') - if not host: - raise AssertionError(f'Pod {pod.get("metadata", {}).get("name", "")!r} has no IP address') - - containers = pod.get('spec', {}).get('containers', []) - if container_name is None: - if len(containers) != 1: - raise AssertionError('`container_name` is required when the pod has more than one regular container') - container = containers[0] - else: - container = next((candidate for candidate in containers if candidate['name'] == container_name), None) - if container is None: - raise AssertionError(f'Pod has no regular container named {container_name!r}') - - ports = tuple( - sorted( - ( - SimpleNamespace(number=port_spec['containerPort'], name=port_spec.get('name', '')) - for port_spec in container.get('ports', []) - ), - key=lambda port: port.number, - ) - ) - - if service_id is None: - container_status = next( - ( - status - for status in pod.get('status', {}).get('containerStatuses', []) - if status['name'] == container['name'] - ), - None, - ) - service_id = container_status.get('containerID') if container_status else None - if not service_id: - raise AssertionError(f'Container {container["name"]!r} has no runtime ID') - - return SimpleNamespace(id=service_id, host=host, ports=ports) - - -def _assert_pod_stable(initial: Mapping[str, Any], current: Mapping[str, Any], candidate_index: int) -> None: - initial_uid = initial['metadata']['uid'] - current_uid = current['metadata']['uid'] - if current_uid != initial_uid: - raise AssertionError(f'Pod changed while probing candidate #{candidate_index}: {initial_uid} -> {current_uid}') - if current.get('metadata', {}).get('deletionTimestamp'): - raise AssertionError(f'Pod is terminating after probing candidate #{candidate_index}') - - phase = current.get('status', {}).get('phase') - if phase != 'Running': - raise AssertionError(f'Pod phase is {phase!r} after probing candidate #{candidate_index}') - - ready = next( - ( - condition.get('status') == 'True' - for condition in current.get('status', {}).get('conditions', []) - if condition.get('type') == 'Ready' - ), - False, - ) - if not ready: - raise AssertionError(f'Pod is not ready after probing candidate #{candidate_index}') - - for status_key in ('initContainerStatuses', 'containerStatuses', 'ephemeralContainerStatuses'): - initial_statuses = {status['name']: status for status in initial.get('status', {}).get(status_key, [])} - current_statuses = {status['name']: status for status in current.get('status', {}).get(status_key, [])} - if current_statuses.keys() != initial_statuses.keys(): - raise AssertionError(f'Pod containers changed while probing candidate #{candidate_index}') - - for name, current_status in current_statuses.items(): - initial_status = initial_statuses[name] - initial_container_id = initial_status.get('containerID') - current_container_id = current_status.get('containerID') - if current_container_id != initial_container_id: - raise AssertionError( - f'Container {name!r} changed while probing candidate #{candidate_index}: ' - f'{initial_container_id} -> {current_container_id}' - ) - - initial_restarts = initial_status.get('restartCount', 0) - current_restarts = current_status.get('restartCount', 0) - if current_restarts != initial_restarts: - raise AssertionError( - f'Container {name!r} restart count changed while probing candidate #{candidate_index}: ' - f'{initial_restarts} -> {current_restarts}' - ) - if status_key == 'containerStatuses' and not current_status.get('ready', False): - raise AssertionError(f'Container {name!r} is not ready after probing candidate #{candidate_index}') - - for state_key in ('state', 'lastState'): - initial_terminated = initial_status.get(state_key, {}).get('terminated') - current_terminated = current_status.get(state_key, {}).get('terminated') - if current_terminated is not None and initial_terminated is None: - reason = current_terminated.get('reason') or '' - raise AssertionError( - f'Container {name!r} terminated with reason {reason!r} ' - f'after probing candidate #{candidate_index}' - ) - - -def _assert_no_new_log_patterns( - previous: Mapping[str, tuple[str, str]], - current: Mapping[str, tuple[str, str]], - patterns: Sequence[str], - candidate_index: int, -) -> None: - if current.keys() != previous.keys(): - raise AssertionError(f'Pod log streams changed while probing candidate #{candidate_index}') - - for container_name, (current_stdout, current_stderr) in current.items(): - previous_stdout, previous_stderr = previous[container_name] - new_logs = _diff_logs(previous_stdout, current_stdout) + _diff_logs(previous_stderr, current_stderr) - for line in new_logs.splitlines(): - logging.debug('New log line from container %s: %s', container_name, line) - for pattern in patterns: - match = re.search(pattern, new_logs, re.IGNORECASE) - if match: - raise AssertionError( - f'Pod logs for container {container_name!r} matched {pattern!r} after probing ' - f'candidate #{candidate_index}: {match.group(0)!r}' - ) - - -def _diff_logs(previous: str, current: str) -> str: - return current[len(previous) :] if current.startswith(previous) else current diff --git a/datadog_checks_dev/tests/test_kubernetes.py b/datadog_checks_dev/tests/test_kubernetes.py deleted file mode 100644 index 2c0d7417ecec5..0000000000000 --- a/datadog_checks_dev/tests/test_kubernetes.py +++ /dev/null @@ -1,209 +0,0 @@ -# (C) Datadog, Inc. 2026-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) -import copy -import json -from types import SimpleNamespace -from unittest.mock import Mock - -import pytest - -from datadog_checks.dev import kubernetes - - -def _pod( - *, - uid='pod-uid', - container_id='containerd://velero-id', - restart_count=0, - ready=True, - phase='Running', - last_reason=None, -): - return { - 'metadata': {'name': 'velero-123', 'uid': uid}, - 'spec': { - 'containers': [ - { - 'name': 'velero', - 'ports': [ - {'containerPort': 8085, 'name': 'http-monitoring'}, - {'containerPort': 9999, 'name': 'admin'}, - ], - } - ] - }, - 'status': { - 'phase': phase, - 'podIP': '10.0.0.1', - 'conditions': [{'type': 'Ready', 'status': 'True' if ready else 'False'}], - 'containerStatuses': [ - { - 'name': 'velero', - 'containerID': container_id, - 'ready': ready, - 'restartCount': restart_count, - 'lastState': {'terminated': {'reason': last_reason}} if last_reason else {}, - } - ], - }, - } - - -def _result(data='', *, stderr='', code=0): - return SimpleNamespace(stdout=data, stderr=stderr, code=code) - - -def _mock_kubectl(monkeypatch, *, initial=None, current=None, initial_logs='old\n', current_logs='old\nnew\n'): - pod_states = iter((initial or _pod(), current or _pod())) - logs = iter((initial_logs, current_logs)) - - def run_command(command, **kwargs): - assert command[:3] == ['kubectl', '--kubeconfig', '/tmp/kubeconfig'] - if command[3:5] == ['get', 'pods']: - return _result(json.dumps({'items': [next(pod_states)]})) - if command[3:5] == ['get', 'pod']: - return _result(json.dumps(next(pod_states))) - if command[3] == 'logs': - return _result(next(logs)) - raise AssertionError(f'Unexpected command: {command}') - - monkeypatch.setattr(kubernetes, 'run_command', run_command) - - -class _Check: - services = [] - - @classmethod - def generate_configs(cls, service): - cls.services.append(service) - return [ - { - 'init_config': {}, - 'instances': [{'openmetrics_endpoint': f'http://{service.host}:{port.number}/metrics'}], - } - for port in service.ports - if port.name == 'http-monitoring' - ] - - -def test_assert_all_discovery_candidates_stable_kubernetes(monkeypatch): - _mock_kubectl(monkeypatch) - dd_agent_check = Mock() - _Check.services = [] - - kubernetes.assert_all_discovery_candidates_stable_kubernetes( - dd_agent_check, - _Check, - '/tmp/kubeconfig', - namespace='velero', - pod_selector='name=velero', - ) - - service = _Check.services[0] - assert service.id == 'containerd://velero-id' - assert service.host == '10.0.0.1' - assert [(port.number, port.name) for port in service.ports] == [ - (8085, 'http-monitoring'), - (9999, 'admin'), - ] - dd_agent_check.assert_called_once_with( - { - 'init_config': {}, - 'instances': [{'openmetrics_endpoint': 'http://10.0.0.1:8085/metrics'}], - }, - check_rate=True, - ) - - -def test_candidate_error_still_checks_workload_stability(monkeypatch): - _mock_kubectl(monkeypatch, current=_pod(restart_count=1)) - dd_agent_check = Mock(side_effect=RuntimeError('check failed')) - - with pytest.raises(AssertionError, match='restart count changed'): - kubernetes.assert_all_discovery_candidates_stable_kubernetes( - dd_agent_check, - _Check, - '/tmp/kubeconfig', - namespace='velero', - pod_name='velero-123', - ) - - -def test_new_dangerous_workload_logs_fail(monkeypatch): - _mock_kubectl(monkeypatch, current_logs='old\npanic: crashed\n') - - with pytest.raises(AssertionError, match="Pod logs for container 'velero' matched 'panic'"): - kubernetes.assert_all_discovery_candidates_stable_kubernetes( - Mock(), - _Check, - '/tmp/kubeconfig', - namespace='velero', - pod_name='velero-123', - ) - - -def test_selector_detects_replacement_pod(monkeypatch): - _mock_kubectl(monkeypatch, current=_pod(uid='replacement-uid')) - - with pytest.raises(AssertionError, match='Pod changed'): - kubernetes.assert_all_discovery_candidates_stable_kubernetes( - Mock(), - _Check, - '/tmp/kubeconfig', - namespace='velero', - pod_selector='name=velero', - ) - - -def test_service_uses_only_target_container_and_preserves_named_ports(): - pod = _pod() - pod['spec']['containers'][0]['ports'].append({'containerPort': 8085, 'name': 'metrics'}) - pod['spec']['containers'].append({'name': 'sidecar', 'ports': [{'containerPort': 9000, 'name': 'sidecar-metrics'}]}) - - with pytest.raises(AssertionError, match='container_name'): - kubernetes._build_service_from_pod(pod, 'svc') - - service = kubernetes._build_service_from_pod(pod, 'svc', container_name='velero') - assert [(port.number, port.name) for port in service.ports] == [ - (8085, 'http-monitoring'), - (8085, 'metrics'), - (9999, 'admin'), - ] - - -def test_termination_without_reason_is_detected(): - initial = _pod() - current = copy.deepcopy(initial) - current['status']['containerStatuses'][0]['state'] = {'terminated': {'exitCode': 1}} - - with pytest.raises(AssertionError, match="terminated with reason ''"): - kubernetes._assert_pod_stable(initial, current, 1) - - -def test_log_streams_are_diffed_independently(): - kubernetes._assert_no_new_log_patterns( - {'velero': ('old error\n', 'old warning\n')}, - {'velero': ('old error\n', 'old warning\nnew healthy line\n')}, - ('error',), - 1, - ) - - -@pytest.mark.parametrize( - ('pod_name', 'pod_selector'), - [ - (None, None), - ('velero-123', 'name=velero'), - ], -) -def test_exactly_one_pod_identifier_is_required(pod_name, pod_selector): - with pytest.raises(TypeError, match='Exactly one'): - kubernetes.assert_all_discovery_candidates_stable_kubernetes( - Mock(), - _Check, - '/tmp/kubeconfig', - namespace='velero', - pod_name=pod_name, - pod_selector=pod_selector, - ) From a65f9218f9f468fbfc1dc310f42422f7962a7185 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 18:28:04 +0000 Subject: [PATCH 14/50] refactor(e2e): avoid caching Kubernetes Agent properties --- ddev/src/ddev/e2e/agent/kubernetes.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index 71644b9283d99..73f30c6f34ef3 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -7,7 +7,6 @@ import json import re import time -from functools import cached_property from typing import TYPE_CHECKING, Any from ddev.e2e.agent.docker import _normalize_agent_image_name @@ -44,21 +43,21 @@ def prepare_start(self) -> None: # delete resources belonging to another environment with the same name. self.metadata[_OWNER_ID_METADATA] = token_hex(16) - @cached_property + @property def _kubernetes_metadata(self) -> dict[str, Any]: metadata = self.metadata.get('kubernetes') if not isinstance(metadata, dict): raise ValueError('Kubernetes Agent metadata must contain a `kubernetes` mapping') return metadata - @cached_property + @property def _kubeconfig(self) -> str: kubeconfig = self._kubernetes_metadata.get('kubeconfig') if not isinstance(kubeconfig, str) or not kubeconfig: raise ValueError('Kubernetes Agent metadata must define a non-empty `kubeconfig` path') return kubeconfig - @cached_property + @property def _namespace(self) -> str: namespace = self._kubernetes_metadata.get('namespace') if namespace is not None: @@ -75,11 +74,11 @@ def _namespace(self) -> str: normalized_id = normalized_id[: 63 - len(_DEFAULT_NAMESPACE_PREFIX) - len(digest) - 2].rstrip('-') return f'{_DEFAULT_NAMESPACE_PREFIX}-{normalized_id}-{digest}' - @cached_property + @property def _cluster_resource_name(self) -> str: return self._namespace - @cached_property + @property def _owner_id(self) -> str: owner_id = self.metadata.get(_OWNER_ID_METADATA) if not isinstance(owner_id, str) or not re.fullmatch(r'[a-z0-9]([-a-z0-9]*[a-z0-9])?', owner_id): @@ -88,26 +87,26 @@ def _owner_id(self) -> str: raise ValueError('Kubernetes Agent internal owner ID must contain at most 63 characters') return owner_id - @cached_property + @property def _resource_labels(self) -> dict[str, str]: return { 'app.kubernetes.io/managed-by': 'ddev', _OWNER_LABEL: self._owner_id, } - @cached_property + @property def _config_dir(self) -> str: return f'/etc/datadog-agent/conf.d/{self.integration.name}.d' - @cached_property + @property def _python_path(self) -> str: return f'/opt/datadog-agent/embedded/bin/python{self.python_version[0]}' - @cached_property + @property def _kubectl_prefix(self) -> list[str]: return ['kubectl', '--kubeconfig', self._kubeconfig] - @cached_property + @property def _wait_timeout(self) -> int: timeout = self._kubernetes_metadata.get('wait_timeout', _DEFAULT_WAIT_TIMEOUT) if isinstance(timeout, bool) or not isinstance(timeout, int) or timeout <= 0: From aad49960dff3c029a5cf4ab65d8a65f0cbd6fb79 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 18:43:23 +0000 Subject: [PATCH 15/50] Restore unsupported CI backend state handling --- ddev/src/ddev/cli/env/start.py | 17 ++++------- ddev/src/ddev/cli/env/stop.py | 10 ++----- ddev/src/ddev/e2e/constants.py | 1 - ddev/tests/cli/env/test_start.py | 51 +++++--------------------------- 4 files changed, 14 insertions(+), 65 deletions(-) diff --git a/ddev/src/ddev/cli/env/start.py b/ddev/src/ddev/cli/env/start.py index 1ffe38fa541d4..34733a152a155 100644 --- a/ddev/src/ddev/cli/env/start.py +++ b/ddev/src/ddev/cli/env/start.py @@ -140,26 +140,19 @@ def start( volumes.extend(deserialize_data(serialized_volumes)) metadata[E2EMetadata.DOCKER_VOLUMES] = volumes + env_data.write_metadata(metadata) + + config = result['config'] + env_data.write_config(config) + agent_class = get_agent_interface(agent_type) if running_on_ci() and not agent_class.supports_ci: - from ddev.cli.env.stop import stop - - # The pytest fixture has already provisioned its environment by the time it returns Agent - # metadata. Persist enough state for `stop` to tear the fixture down, while recording that - # no Agent backend was started. If teardown fails, a later `ddev env stop` can safely retry. - metadata[E2EMetadata.SKIP_AGENT_STOP] = True - env_data.write_metadata(metadata) - env_data.write_config(result['config']) - ctx.invoke(stop, intg_name=intg_name, environment=environment) app.abort(text=f'{agent_type.capitalize()} is not supported on CI', code=0) agent = agent_class(app, integration, environment, metadata, env_data.config_file) agent.prepare_start() env_data.write_metadata(metadata) - config = result['config'] - env_data.write_config(config) - if not agent_build: configured_agent_build = agent.get_configured_build(app.config.agent.config) agent_build = ( diff --git a/ddev/src/ddev/cli/env/stop.py b/ddev/src/ddev/cli/env/stop.py index c16500c377be1..0b9457536ad5d 100644 --- a/ddev/src/ddev/cli/env/stop.py +++ b/ddev/src/ddev/cli/env/stop.py @@ -63,16 +63,10 @@ def stop(app: Application, *, intg_name: str, environment: str, ignore_state: bo metadata = env_data.read_metadata() env_vars.update(metadata.get(E2EMetadata.ENV_VARS, {})) - skip_agent_stop = metadata.get(E2EMetadata.SKIP_AGENT_STOP, False) - agent = ( - None - if skip_agent_stop - else create_agent_interface(app, integration, env, metadata, env_data.config_file) - ) + agent = create_agent_interface(app, integration, env, metadata, env_data.config_file) try: - if agent is not None: - agent.stop() + agent.stop() finally: with integration.path.as_cwd(env_vars=env_vars), runner.stop() as command: process = app.platform.run_command(command) diff --git a/ddev/src/ddev/e2e/constants.py b/ddev/src/ddev/e2e/constants.py index 4924daf591a73..e52a15d26a180 100644 --- a/ddev/src/ddev/e2e/constants.py +++ b/ddev/src/ddev/e2e/constants.py @@ -22,4 +22,3 @@ class E2EMetadata: AGENT_TYPE = 'agent_type' ENV_VARS = 'e2e_env_vars' DOCKER_VOLUMES = 'docker_volumes' - SKIP_AGENT_STOP = '_ddev_skip_agent_stop' diff --git a/ddev/tests/cli/env/test_start.py b/ddev/tests/cli/env/test_start.py index cfbe0473fa240..9c001ba6a7136 100644 --- a/ddev/tests/cli/env/test_start.py +++ b/ddev/tests/cli/env/test_start.py @@ -98,12 +98,12 @@ def test_stop_on_error(ddev, helpers, data_dir, write_result_file, mocker): stop.assert_called_once() -def test_unsupported_agent_on_ci_cleans_up_environment(ddev, data_dir, write_result_file, mocker): +def test_unsupported_agent_on_ci_persists_environment_state(ddev, data_dir, write_result_file, mocker): metadata = {'agent_type': 'vagrant'} config = {} run = mocker.patch('subprocess.run', side_effect=write_result_file({'metadata': metadata, 'config': config})) mocker.patch('ddev.cli.env.start.running_on_ci', return_value=True) - stop_agent = mocker.patch('ddev.e2e.agent.vagrant.VagrantAgent.stop') + start_agent = mocker.patch('ddev.e2e.agent.vagrant.VagrantAgent.start') integration = 'postgres' environment = 'py3.12' @@ -113,49 +113,12 @@ def test_unsupported_agent_on_ci_cleans_up_environment(ddev, data_dir, write_res assert result.exit_code == 0, result.output assert 'Starting: py3.12' in result.output - assert 'Stopping: postgres:py3.12' in result.output + assert 'Stopping:' not in result.output assert 'Vagrant is not supported on CI' in result.output - assert not env_data.exists() - assert run.call_count == 2 - stop_agent.assert_not_called() - - -def test_unsupported_agent_failed_cleanup_can_be_retried(ddev, data_dir, write_result_file, mocker): - metadata = {'agent_type': 'vagrant'} - config = {} - write_result = write_result_file({'metadata': metadata, 'config': config}) - call_count = 0 - - def run_command(*args, **kwargs): - nonlocal call_count - call_count += 1 - result = write_result(*args, **kwargs) - if call_count == 2: - result.returncode = 1 - return result - - run = mocker.patch('subprocess.run', side_effect=run_command) - mocker.patch('ddev.cli.env.start.running_on_ci', return_value=True) - stop_agent = mocker.patch('ddev.e2e.agent.vagrant.VagrantAgent.stop') - - integration = 'postgres' - environment = 'py3.12' - env_data = EnvDataStorage(data_dir).get(integration, environment) - - result = ddev('env', 'start', integration, environment) - - assert result.exit_code == 1, result.output - assert env_data.exists() - assert env_data.read_metadata() == {**metadata, E2EMetadata.SKIP_AGENT_STOP: True} - stop_agent.assert_not_called() - - run.side_effect = None - run.return_value = mocker.MagicMock(returncode=0) - result = ddev('env', 'stop', integration, environment) - - assert result.exit_code == 0, result.output - assert not env_data.exists() - stop_agent.assert_not_called() + assert env_data.read_metadata() == metadata + assert env_data.read_config() == {'instances': [config]} + assert run.call_count == 1 + start_agent.assert_not_called() def test_basic(ddev, helpers, data_dir, write_result_file, mocker): From d0dbbf8ac93f1795b38b4a25188e528ecd77fa8d Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 18:49:26 +0000 Subject: [PATCH 16/50] Clarify Kubernetes Agent constraints --- ddev/src/ddev/e2e/agent/kubernetes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index 73f30c6f34ef3..29748f8370e02 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -29,9 +29,9 @@ class KubernetesAgent(AgentInterface): """Run the E2E Agent inside a Kubernetes test cluster. - The initial implementation intentionally supports one schedulable node and - one Agent pod. Cluster creation and image loading belong to the environment - provisioner (for example, ``kind_run``), not this backend. + Exactly one schedulable node is required, and the backend runs one Agent pod. + Cluster creation and image loading are the responsibility of the environment + provisioner (for example, ``kind_run``). """ build_config_keys = ('kubernetes', 'docker') From 45d98e27a4e1c3d198a50e610427e33c4b455280 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 18:54:21 +0000 Subject: [PATCH 17/50] refactor(ddev): use singular Agent build config keys --- ddev/src/ddev/e2e/agent/docker.py | 2 +- ddev/src/ddev/e2e/agent/interface.py | 9 ++++----- ddev/src/ddev/e2e/agent/kubernetes.py | 2 +- ddev/src/ddev/e2e/agent/vagrant.py | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/docker.py b/ddev/src/ddev/e2e/agent/docker.py index 319cb12fba5ab..95760858cb193 100644 --- a/ddev/src/ddev/e2e/agent/docker.py +++ b/ddev/src/ddev/e2e/agent/docker.py @@ -82,7 +82,7 @@ def _normalize_agent_image_name(agent_build: str | None, python_major: int, use_ class DockerAgent(AgentInterface): - build_config_keys = ('docker',) + build_config_key = 'docker' @cached_property def _isatty(self) -> bool: diff --git a/ddev/src/ddev/e2e/agent/interface.py b/ddev/src/ddev/e2e/agent/interface.py index 85047b3a3fb63..9f0400866ddfa 100644 --- a/ddev/src/ddev/e2e/agent/interface.py +++ b/ddev/src/ddev/e2e/agent/interface.py @@ -17,7 +17,7 @@ class AgentInterface(ABC): - build_config_keys: tuple[str, ...] = () + build_config_key: str | None = None supports_ci = True def __init__( @@ -72,11 +72,10 @@ def prepare_start(self) -> None: """Populate backend state that must be persisted before startup.""" def get_configured_build(self, config: Mapping[str, str]) -> str | None: - for key in self.build_config_keys: - if agent_build := config.get(key): - return agent_build + if self.build_config_key is None: + return None - return None + return config.get(self.build_config_key) @abstractmethod def start(self, *, agent_build: str, local_packages: dict[Path, str], env_vars: dict[str, str]) -> None: ... diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index 29748f8370e02..dccbe86220847 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -34,7 +34,7 @@ class KubernetesAgent(AgentInterface): provisioner (for example, ``kind_run``). """ - build_config_keys = ('kubernetes', 'docker') + build_config_key = 'docker' def prepare_start(self) -> None: from secrets import token_hex diff --git a/ddev/src/ddev/e2e/agent/vagrant.py b/ddev/src/ddev/e2e/agent/vagrant.py index 368132fc77ea8..32b4218de1df5 100644 --- a/ddev/src/ddev/e2e/agent/vagrant.py +++ b/ddev/src/ddev/e2e/agent/vagrant.py @@ -57,7 +57,7 @@ def disable_integration_before_install(config_file: Path): class VagrantAgent(AgentInterface): - build_config_keys = ('vagrant',) + build_config_key = 'vagrant' supports_ci = False VM_HOST_IP = "172.30.1.5" From c6b694dfa1b8eebcf2d80502a8e3bab03c684331 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 19:04:00 +0000 Subject: [PATCH 18/50] Remove Kubernetes Agent pod label option --- ddev/src/ddev/e2e/agent/kubernetes.py | 7 +------ ddev/tests/e2e/agent/test_kubernetes.py | 12 ++++++++++-- docs/developer/ddev/plugins.md | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index dccbe86220847..167b72d84c6d0 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -209,12 +209,7 @@ def _manifest(self, agent_build: str, env_vars: dict[str, str]) -> dict[str, Any ] ) - extra_labels = self._kubernetes_metadata.get('pod_labels', {}) - if not isinstance(extra_labels, dict) or not all( - isinstance(key, str) and isinstance(value, str) for key, value in extra_labels.items() - ): - raise ValueError('Kubernetes Agent `pod_labels` must be a mapping of strings to strings') - labels = {**extra_labels, **self._resource_labels, 'app.kubernetes.io/name': _POD_NAME} + labels = {**self._resource_labels, 'app.kubernetes.io/name': _POD_NAME} image_pull_policy = self._kubernetes_metadata.get('image_pull_policy', 'Always') if image_pull_policy not in {'Always', 'IfNotPresent', 'Never'}: diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index ba57ae9b9684f..84a0c36af0e96 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -107,8 +107,16 @@ def test_start_uses_selected_image_rbac_config_and_local_packages( assert resources['Pod']['spec']['containers'][0]['image'] == 'registry.example.com/datadog-agent:test' assert resources['Pod']['spec']['containers'][0]['imagePullPolicy'] == 'Always' assert resources['Pod']['spec']['serviceAccountName'] == 'ddev-agent' - for kind in ('Namespace', 'ServiceAccount', 'ClusterRole', 'ClusterRoleBinding', 'Pod'): - assert resources[kind]['metadata']['labels']['ddev.datadoghq.com/environment'] == 'test-owner' + resource_labels = { + 'app.kubernetes.io/managed-by': 'ddev', + 'ddev.datadoghq.com/environment': 'test-owner', + } + for kind in ('Namespace', 'ServiceAccount', 'ClusterRole', 'ClusterRoleBinding'): + assert resources[kind]['metadata']['labels'] == resource_labels + assert resources['Pod']['metadata']['labels'] == { + **resource_labels, + 'app.kubernetes.io/name': 'ddev-agent', + } assert resources['ClusterRole']['rules'] == [ {'apiGroups': [''], 'resources': ['nodes'], 'verbs': ['get', 'list', 'watch']}, { diff --git a/docs/developer/ddev/plugins.md b/docs/developer/ddev/plugins.md index fbf1c4599fa1d..a784239a297cb 100644 --- a/docs/developer/ddev/plugins.md +++ b/docs/developer/ddev/plugins.md @@ -141,5 +141,5 @@ is desired. This fixture is responsible for starting and stopping environments a - `env_vars` - A `dict` of environment variables and their values that will be present when starting the Agent. - `docker_volumes` - A `list` of `str` representing [Docker volume mounts][docker-volume-docs] if `agent_type` is `docker` e.g. `/local/path:/agent/container/path:ro`. - `docker_platform` - The container architecture to use if `agent_type` is `docker`. Currently, we support `linux` (default) and `windows`. -- `kubernetes` - Configuration used when `agent_type` is `kubernetes`. It requires `kubeconfig` and optionally accepts `namespace`, `auto_conf`, `pod_labels`, `image_pull_policy`, and `wait_timeout`. The backend owns the namespace, which must not already exist. +- `kubernetes` - Configuration used when `agent_type` is `kubernetes`. It requires `kubeconfig` and optionally accepts `namespace`, `auto_conf`, `image_pull_policy`, and `wait_timeout`. The backend owns the namespace, which must not already exist. - `logs_config` - A `list` of configs that will be used by the Logs Agent. You will never need to use this directly, but rather via [higher level abstractions](test.md#logs). From 18267e74ba6eaabb1b6940f4277346d1bd2d37fb Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 19:04:00 +0000 Subject: [PATCH 19/50] Remove custom Kubernetes Agent namespace option --- ddev/src/ddev/e2e/agent/kubernetes.py | 10 +--------- ddev/tests/e2e/agent/test_kubernetes.py | 18 ++++++++++++++++-- docs/developer/ddev/plugins.md | 2 +- docs/developer/ddev/test.md | 4 ++-- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index 167b72d84c6d0..82167f4a9c70f 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -59,14 +59,6 @@ def _kubeconfig(self) -> str: @property def _namespace(self) -> str: - namespace = self._kubernetes_metadata.get('namespace') - if namespace is not None: - if not isinstance(namespace, str) or not re.fullmatch(r'[a-z0-9]([-a-z0-9]*[a-z0-9])?', namespace): - raise ValueError(f'Invalid Kubernetes Agent namespace: {namespace!r}') - if len(namespace) > 63: - raise ValueError('Kubernetes Agent namespace must contain at most 63 characters') - return namespace - raw_id = super().get_id().lower() normalized_id = re.sub(r'[^a-z0-9]+', '-', raw_id).strip('-') digest = hashlib.sha256(raw_id.encode()).hexdigest()[:8] @@ -438,7 +430,7 @@ def stop(self) -> None: errors: list[Exception] = [] # Do not execute commands in a namespace that merely happens to have - # the configured name. Startup may have failed because it was already + # the generated name. Startup may have failed because it was already # owned by the cluster's caller. try: namespace_owned = self._resource_is_owned('namespace', self._namespace) diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index 84a0c36af0e96..18f211e792ba5 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -2,6 +2,7 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) import json +import re import subprocess import pytest @@ -81,6 +82,20 @@ def command_calls(run_command): return [call.args[0] for call in run_command.call_args_list] +def test_namespace_metadata_cannot_override_generated_name(agent, metadata): + metadata['kubernetes']['namespace'] = 'custom-namespace' + + assert agent._namespace == 'ddev-agent-velero-py3-12-ad72b57a' + + +def test_generated_namespace_is_a_valid_kubernetes_name(app, get_integration, metadata, config_file): + integration = get_integration('Very_Long.Integration-' + 'x' * 100) + agent = KubernetesAgent(app, integration, 'Py3.12+FIPS', metadata, config_file) + + assert len(agent._namespace) == 63 + assert re.fullmatch(r'[a-z0-9]([-a-z0-9]*[a-z0-9])?', agent._namespace) + + def test_start_uses_selected_image_rbac_config_and_local_packages( agent, metadata, config_file, auto_conf, temp_dir, run_command ): @@ -503,10 +518,9 @@ def test_shell_and_logs_use_backend_commands(agent, run_command): [ ({}, 'must contain a `kubernetes` mapping'), ({'kubernetes': {}}, 'non-empty `kubeconfig`'), - ({'kubernetes': {'kubeconfig': '/tmp/config', 'namespace': 'INVALID'}}, 'Invalid Kubernetes Agent namespace'), ], ) def test_metadata_validation(app, get_integration, config_file, metadata, match): agent = KubernetesAgent(app, get_integration('velero'), 'py3.12', metadata, config_file) with pytest.raises(ValueError, match=match): - _ = agent._namespace if 'namespace' in metadata.get('kubernetes', {}) else agent._kubeconfig + _ = agent._kubeconfig diff --git a/docs/developer/ddev/plugins.md b/docs/developer/ddev/plugins.md index a784239a297cb..27a76dd6c5e8e 100644 --- a/docs/developer/ddev/plugins.md +++ b/docs/developer/ddev/plugins.md @@ -141,5 +141,5 @@ is desired. This fixture is responsible for starting and stopping environments a - `env_vars` - A `dict` of environment variables and their values that will be present when starting the Agent. - `docker_volumes` - A `list` of `str` representing [Docker volume mounts][docker-volume-docs] if `agent_type` is `docker` e.g. `/local/path:/agent/container/path:ro`. - `docker_platform` - The container architecture to use if `agent_type` is `docker`. Currently, we support `linux` (default) and `windows`. -- `kubernetes` - Configuration used when `agent_type` is `kubernetes`. It requires `kubeconfig` and optionally accepts `namespace`, `auto_conf`, `image_pull_policy`, and `wait_timeout`. The backend owns the namespace, which must not already exist. +- `kubernetes` - Configuration used when `agent_type` is `kubernetes`. It requires `kubeconfig` and optionally accepts `auto_conf`, `image_pull_policy`, and `wait_timeout`. The backend owns its generated namespace, which must not already exist. - `logs_config` - A `list` of configs that will be used by the Logs Agent. You will never need to use this directly, but rather via [higher level abstractions](test.md#logs). diff --git a/docs/developer/ddev/test.md b/docs/developer/ddev/test.md index 1e6ed2b49f1a9..db3ae37eea515 100644 --- a/docs/developer/ddev/test.md +++ b/docs/developer/ddev/test.md @@ -121,8 +121,8 @@ local packages requested by `--dev` or `--base`, and implements Agent commands t discovery E2E tests therefore continue to use `dd_agent_check` and `dd_agent_check_discovery`. Agent images default to the `Always` pull policy so mutable release and development tags are refreshed. Environments -that import a local image into the cluster can set `image_pull_policy` to `IfNotPresent` or `Never`. A custom `namespace` -must not already exist; the backend owns and deletes the namespace and its cluster-scoped RBAC resources. +that import a local image into the cluster can set `image_pull_policy` to `IfNotPresent` or `Never`. The backend generates +a namespace from the integration and environment names, then owns and deletes it and its cluster-scoped RBAC resources. The initial implementation supports exactly one schedulable Kubernetes node. It rejects multi-node clusters until Agent targeting or fan-out semantics are defined. From 475a6e9a29a16934c56c9d7d9951e655585bdc13 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 20:32:42 +0000 Subject: [PATCH 20/50] Deduplicate Agent log commands --- ddev/src/ddev/e2e/agent/docker.py | 6 +++--- ddev/src/ddev/e2e/agent/kubernetes.py | 9 +++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/docker.py b/ddev/src/ddev/e2e/agent/docker.py index 95760858cb193..6daa11d26558b 100644 --- a/ddev/src/ddev/e2e/agent/docker.py +++ b/ddev/src/ddev/e2e/agent/docker.py @@ -175,11 +175,11 @@ def _execute_lifecycle_command( self._show_logs() raise RuntimeError(error_message) - def _show_logs(self) -> None: - self._run_command(['docker', 'logs', self._container_name]) + def _show_logs(self, *, check: bool = False) -> None: + self._run_command(['docker', 'logs', self._container_name], check=check) def show_logs(self) -> None: - self._run_command(['docker', 'logs', self._container_name], check=True) + self._show_logs(check=True) def get_id(self) -> str: return self._container_name diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index 82167f4a9c70f..7bb0db47b0582 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -527,14 +527,11 @@ def enter_shell(self) -> None: check=True, ) - def _show_logs(self) -> None: + def _show_logs(self, *, check: bool = False) -> None: self._kubectl( ['logs', '--namespace', self._namespace, f'pod/{_POD_NAME}', '--container', _CONTAINER_NAME], - check=False, + check=check, ) def show_logs(self) -> None: - self._kubectl( - ['logs', '--namespace', self._namespace, f'pod/{_POD_NAME}', '--container', _CONTAINER_NAME], - check=True, - ) + self._show_logs(check=True) From c28ca047aadbc1560341ace5f150ae1ff174a286 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 20:36:05 +0000 Subject: [PATCH 21/50] Simplify Kubernetes metadata access --- ddev/src/ddev/e2e/agent/kubernetes.py | 7 ++----- ddev/tests/e2e/agent/test_kubernetes.py | 13 +++---------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index 7bb0db47b0582..a487d98559710 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -7,7 +7,7 @@ import json import re import time -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast from ddev.e2e.agent.docker import _normalize_agent_image_name from ddev.e2e.agent.interface import AgentInterface @@ -45,10 +45,7 @@ def prepare_start(self) -> None: @property def _kubernetes_metadata(self) -> dict[str, Any]: - metadata = self.metadata.get('kubernetes') - if not isinstance(metadata, dict): - raise ValueError('Kubernetes Agent metadata must contain a `kubernetes` mapping') - return metadata + return cast(dict[str, Any], self.metadata['kubernetes']) @property def _kubeconfig(self) -> str: diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index 18f211e792ba5..01cafa93bcf62 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -513,14 +513,7 @@ def test_shell_and_logs_use_backend_commands(agent, run_command): assert run_command.call_args_list[-1].kwargs['check'] is True -@pytest.mark.parametrize( - 'metadata, match', - [ - ({}, 'must contain a `kubernetes` mapping'), - ({'kubernetes': {}}, 'non-empty `kubeconfig`'), - ], -) -def test_metadata_validation(app, get_integration, config_file, metadata, match): - agent = KubernetesAgent(app, get_integration('velero'), 'py3.12', metadata, config_file) - with pytest.raises(ValueError, match=match): +def test_kubeconfig_validation(app, get_integration, config_file): + agent = KubernetesAgent(app, get_integration('velero'), 'py3.12', {'kubernetes': {}}, config_file) + with pytest.raises(ValueError, match='non-empty `kubeconfig`'): _ = agent._kubeconfig From e0d4f0d5a8f250aa5a8738dbd6ddb0ac159990cd Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 20:44:04 +0000 Subject: [PATCH 22/50] refactor(ddev): remove Kubernetes Agent owner ID --- ddev/src/ddev/cli/env/start.py | 2 - ddev/src/ddev/cli/env/stop.py | 4 +- ddev/src/ddev/e2e/agent/interface.py | 3 - ddev/src/ddev/e2e/agent/kubernetes.py | 110 ++++-------------------- ddev/tests/cli/env/test_start.py | 3 +- ddev/tests/e2e/agent/test_kubernetes.py | 76 ++++------------ 6 files changed, 37 insertions(+), 161 deletions(-) diff --git a/ddev/src/ddev/cli/env/start.py b/ddev/src/ddev/cli/env/start.py index 34733a152a155..0a76d849d3f32 100644 --- a/ddev/src/ddev/cli/env/start.py +++ b/ddev/src/ddev/cli/env/start.py @@ -150,8 +150,6 @@ def start( app.abort(text=f'{agent_type.capitalize()} is not supported on CI', code=0) agent = agent_class(app, integration, environment, metadata, env_data.config_file) - agent.prepare_start() - env_data.write_metadata(metadata) if not agent_build: configured_agent_build = agent.get_configured_build(app.config.agent.config) diff --git a/ddev/src/ddev/cli/env/stop.py b/ddev/src/ddev/cli/env/stop.py index 0b9457536ad5d..58d4f3799343b 100644 --- a/ddev/src/ddev/cli/env/stop.py +++ b/ddev/src/ddev/cli/env/stop.py @@ -73,6 +73,6 @@ def stop(app: Application, *, intg_name: str, environment: str, ignore_state: bo if process.returncode: app.abort(code=process.returncode) - # Keep the owner metadata when either cleanup stage fails so a - # later `ddev env stop` can safely retry resource deletion. + # Keep the environment state when either cleanup stage fails so + # a later `ddev env stop` can retry the cleanup. env_data.remove() diff --git a/ddev/src/ddev/e2e/agent/interface.py b/ddev/src/ddev/e2e/agent/interface.py index 9f0400866ddfa..13b0a8089bd67 100644 --- a/ddev/src/ddev/e2e/agent/interface.py +++ b/ddev/src/ddev/e2e/agent/interface.py @@ -68,9 +68,6 @@ def python_version(self) -> tuple[int, int]: def get_id(self) -> str: return f'{self.integration.name}_{self.env}' - def prepare_start(self) -> None: - """Populate backend state that must be persisted before startup.""" - def get_configured_build(self, config: Mapping[str, str]) -> str | None: if self.build_config_key is None: return None diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index a487d98559710..36fa29036db4b 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -22,27 +22,18 @@ _DEFAULT_NAMESPACE_PREFIX = 'ddev-agent' _DEFAULT_WAIT_TIMEOUT = 120 _LOCAL_PACKAGES_METADATA = 'local_packages' -_OWNER_ID_METADATA = '_kubernetes_owner_id' -_OWNER_LABEL = 'ddev.datadoghq.com/environment' class KubernetesAgent(AgentInterface): - """Run the E2E Agent inside a Kubernetes test cluster. + """Run the E2E Agent inside a dedicated Kubernetes test cluster. Exactly one schedulable node is required, and the backend runs one Agent pod. - Cluster creation and image loading are the responsibility of the environment - provisioner (for example, ``kind_run``). + Cluster creation, image loading, and final teardown are the responsibility of + the environment provisioner (for example, ``kind_run``). """ build_config_key = 'docker' - def prepare_start(self) -> None: - from secrets import token_hex - - # Persist a unique owner before backend startup so error cleanup cannot - # delete resources belonging to another environment with the same name. - self.metadata[_OWNER_ID_METADATA] = token_hex(16) - @property def _kubernetes_metadata(self) -> dict[str, Any]: return cast(dict[str, Any], self.metadata['kubernetes']) @@ -67,21 +58,9 @@ def _namespace(self) -> str: def _cluster_resource_name(self) -> str: return self._namespace - @property - def _owner_id(self) -> str: - owner_id = self.metadata.get(_OWNER_ID_METADATA) - if not isinstance(owner_id, str) or not re.fullmatch(r'[a-z0-9]([-a-z0-9]*[a-z0-9])?', owner_id): - raise ValueError('Kubernetes Agent metadata is missing a valid internal owner ID') - if len(owner_id) > 63: - raise ValueError('Kubernetes Agent internal owner ID must contain at most 63 characters') - return owner_id - @property def _resource_labels(self) -> dict[str, str]: - return { - 'app.kubernetes.io/managed-by': 'ddev', - _OWNER_LABEL: self._owner_id, - } + return {'app.kubernetes.io/managed-by': 'ddev'} @property def _config_dir(self) -> str: @@ -137,28 +116,6 @@ def _exec( return self._captured_kubectl(args, check=check) return self._kubectl(args, check=check) - def _validate_resource_ownership(self) -> None: - resources = ( - ('namespace', self._namespace), - ('clusterrole', self._cluster_resource_name), - ('clusterrolebinding', self._cluster_resource_name), - ) - existing_resources = [] - for resource, name in resources: - process = self._captured_kubectl(['get', resource, name, '--ignore-not-found=true', '-o', 'name']) - if process.returncode: - raise RuntimeError( - f'Unable to inspect Kubernetes Agent {resource} `{name}`: {self._process_output(process)}' - ) - if self._process_output(process).strip(): - existing_resources.append(f'{resource}/{name}') - - if existing_resources: - raise RuntimeError( - 'Refusing to overwrite Kubernetes resources not owned by this environment: ' - + ', '.join(existing_resources) - ) - def _validate_topology(self) -> None: process = self._captured_kubectl(['get', 'nodes', '-o', 'json']) if process.returncode: @@ -278,14 +235,6 @@ def _create_payload(self, payload: dict[str, Any]) -> None: if process.returncode: raise RuntimeError(f'Unable to create Kubernetes Agent resources: {self._process_output(process)}') - def _create_manifest(self, manifest: dict[str, Any]) -> None: - # Acquire the namespace as an atomic ownership lock before creating any - # other resource. `create`, unlike `apply`, fails rather than overwriting - # a namespace that appears after the ownership preflight. - namespace, *resources = manifest['items'] - self._create_payload(namespace) - self._create_payload({'apiVersion': 'v1', 'kind': 'List', 'items': resources}) - def _wait_for_pod(self) -> None: process = self._captured_kubectl( [ @@ -397,10 +346,9 @@ def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env agent_build, self.python_version[0], self.metadata.get('use_jmx', False) ) # Validate values needed by teardown before creating any resources. - _ = self._owner_id, self._wait_timeout + _ = self._wait_timeout self._validate_topology() - self._validate_resource_ownership() - self._create_manifest(self._manifest(agent_build, env_vars)) + self._create_payload(self._manifest(agent_build, env_vars)) self._wait_for_pod() self._run_metadata_commands('start_commands') self._remember_local_packages(local_packages) @@ -410,48 +358,23 @@ def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env self._run_metadata_commands('post_install_commands') self._restart_agent_process() - def _resource_is_owned(self, resource: str, name: str) -> bool: - process = self._captured_kubectl(['get', resource, name, '--ignore-not-found=true', '-o', 'json']) - if process.returncode: - output = self._process_output(process) - raise RuntimeError(f'Unable to inspect Kubernetes Agent {resource} `{name}`: {output}') - if not self._process_output(process).strip(): - return False - try: - data = json.loads(self._process_output(process)) - return data.get('metadata', {}).get('labels', {}).get(_OWNER_LABEL) == self._owner_id - except (AttributeError, json.JSONDecodeError) as e: - raise RuntimeError(f'Unable to parse Kubernetes Agent {resource} `{name}`: {e}') from e - def stop(self) -> None: errors: list[Exception] = [] - # Do not execute commands in a namespace that merely happens to have - # the generated name. Startup may have failed because it was already - # owned by the cluster's caller. - try: - namespace_owned = self._resource_is_owned('namespace', self._namespace) - except Exception as e: - errors.append(e) - namespace_owned = False - - if namespace_owned: - pod = self._captured_kubectl( - ['get', 'pod', _POD_NAME, '--namespace', self._namespace, '--ignore-not-found=true', '-o', 'name'] - ) - if pod.returncode == 0 and self._process_output(pod).strip(): - try: - self._run_metadata_commands('stop_commands') - except Exception as e: - errors.append(e) + pod = self._captured_kubectl( + ['get', 'pod', _POD_NAME, '--namespace', self._namespace, '--ignore-not-found=true', '-o', 'name'] + ) + if pod.returncode == 0 and self._process_output(pod).strip(): + try: + self._run_metadata_commands('stop_commands') + except Exception as e: + errors.append(e) - selector = f'{_OWNER_LABEL}={self._owner_id}' for args in ( [ 'delete', 'namespace', - '--selector', - selector, + self._namespace, '--ignore-not-found=true', '--wait=true', f'--timeout={self._wait_timeout}s', @@ -459,8 +382,7 @@ def stop(self) -> None: [ 'delete', 'clusterrole,clusterrolebinding', - '--selector', - selector, + self._cluster_resource_name, '--ignore-not-found=true', ], ): diff --git a/ddev/tests/cli/env/test_start.py b/ddev/tests/cli/env/test_start.py index 9c001ba6a7136..7264886eff496 100644 --- a/ddev/tests/cli/env/test_start.py +++ b/ddev/tests/cli/env/test_start.py @@ -204,7 +204,6 @@ def test_kubernetes_agent_build_uses_configured_docker_image(ddev, config_file, metadata = {'agent_type': 'kubernetes', 'kubernetes': {'kubeconfig': '/tmp/kubeconfig'}} config = {} - mocker.patch('secrets.token_hex', return_value='test-owner') mocker.patch('subprocess.run', side_effect=write_result_file({'metadata': metadata, 'config': config})) start = mocker.patch('ddev.e2e.agent.kubernetes.KubernetesAgent.start') @@ -215,7 +214,7 @@ def test_kubernetes_agent_build_uses_configured_docker_image(ddev, config_file, result = ddev('env', 'start', integration, environment) assert result.exit_code == 0, result.output - assert env_data.read_metadata() == {**metadata, '_kubernetes_owner_id': 'test-owner'} + assert env_data.read_metadata() == metadata start.assert_called_once_with( agent_build='registry.datadoghq.com/agent:7', local_packages={}, diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index 01cafa93bcf62..60b9ab53f1f0a 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -50,7 +50,6 @@ def auto_conf(temp_dir): @pytest.fixture def metadata(auto_conf): return { - '_kubernetes_owner_id': 'test-owner', 'kubernetes': { 'kubeconfig': '/tmp/kubeconfig', 'auto_conf': str(auto_conf), @@ -114,18 +113,13 @@ def test_start_uses_selected_image_rbac_config_and_local_packages( prefix = ['kubectl', '--kubeconfig', '/tmp/kubeconfig'] assert calls[0] == [*prefix, 'get', 'nodes', '-o', 'json'] - create_calls = [call for call in run_command.call_args_list if call.args[0] == [*prefix, 'create', '-f', '-']] - namespace = json.loads(create_calls[0].kwargs['input']) - manifest = json.loads(create_calls[1].kwargs['input']) - resources = {item['kind']: item for item in [namespace, *manifest['items']]} - assert namespace['kind'] == 'Namespace' + create_call = next(call for call in run_command.call_args_list if call.args[0] == [*prefix, 'create', '-f', '-']) + manifest = json.loads(create_call.kwargs['input']) + resources = {item['kind']: item for item in manifest['items']} assert resources['Pod']['spec']['containers'][0]['image'] == 'registry.example.com/datadog-agent:test' assert resources['Pod']['spec']['containers'][0]['imagePullPolicy'] == 'Always' assert resources['Pod']['spec']['serviceAccountName'] == 'ddev-agent' - resource_labels = { - 'app.kubernetes.io/managed-by': 'ddev', - 'ddev.datadoghq.com/environment': 'test-owner', - } + resource_labels = {'app.kubernetes.io/managed-by': 'ddev'} for kind in ('Namespace', 'ServiceAccount', 'ClusterRole', 'ClusterRoleBinding'): assert resources[kind]['metadata']['labels'] == resource_labels assert resources['Pod']['metadata']['labels'] == { @@ -238,25 +232,7 @@ def test_rejects_multi_node_clusters(agent, app, mocker): run_command.assert_called_once() -def test_rejects_preexisting_owned_resources(agent, app, mocker): - nodes = {'items': [{'metadata': {'name': 'kind-control-plane'}, 'spec': {}}]} - - def run(command, **kwargs): - if command[-4:] == ['get', 'nodes', '-o', 'json']: - return successful_process(command, stdout=json.dumps(nodes).encode()) - if 'namespace' in command and '--ignore-not-found=true' in command: - return successful_process(command, stdout=f'namespace/{agent._namespace}\n'.encode()) - return successful_process(command) - - run_command = mocker.patch.object(app.platform, 'run_command', side_effect=run) - - with pytest.raises(RuntimeError, match='Refusing to overwrite Kubernetes resources'): - agent.start(agent_build='', local_packages={}, env_vars={}) - - assert not any(call.args[0][-3:] == ['create', '-f', '-'] for call in run_command.call_args_list) - - -def test_creation_fails_if_a_resource_appears_after_preflight(agent, app, mocker): +def test_creation_failure_stops_startup(agent, app, mocker): nodes = {'items': [{'metadata': {'name': 'kind-control-plane'}, 'spec': {}}]} def run(command, **kwargs): @@ -374,15 +350,23 @@ def test_stop_is_idempotent_when_pod_is_absent(agent, run_command): calls = command_calls(run_command) prefix = ['kubectl', '--kubeconfig', '/tmp/kubeconfig'] - selector = 'ddev.datadoghq.com/environment=test-owner' assert calls == [ - [*prefix, 'get', 'namespace', agent._namespace, '--ignore-not-found=true', '-o', 'json'], + [ + *prefix, + 'get', + 'pod', + 'ddev-agent', + '--namespace', + agent._namespace, + '--ignore-not-found=true', + '-o', + 'name', + ], [ *prefix, 'delete', 'namespace', - '--selector', - selector, + agent._namespace, '--ignore-not-found=true', '--wait=true', '--timeout=120s', @@ -391,33 +375,12 @@ def test_stop_is_idempotent_when_pod_is_absent(agent, run_command): *prefix, 'delete', 'clusterrole,clusterrolebinding', - '--selector', - selector, + agent._cluster_resource_name, '--ignore-not-found=true', ], ] -def test_stop_does_not_enter_or_delete_another_environment_namespace(agent, app, mocker): - namespace = {'metadata': {'labels': {'ddev.datadoghq.com/environment': 'another-owner'}}} - - def run(command, **kwargs): - if command[-1:] == ['json'] and 'namespace' in command: - return successful_process(command, stdout=json.dumps(namespace).encode()) - return successful_process(command) - - run_command = mocker.patch.object(app.platform, 'run_command', side_effect=run) - - agent.stop() - - calls = command_calls(run_command) - assert not any('pod' in command for command in calls) - delete_calls = [command for command in calls if 'delete' in command] - assert len(delete_calls) == 2 - assert all('ddev.datadoghq.com/environment=test-owner' in command for command in delete_calls) - assert all('another-owner' not in command for command in delete_calls) - - def test_restart_resynchronizes_config_auto_conf_and_editable_sources( agent, metadata, config_file, temp_dir, run_command ): @@ -470,9 +433,6 @@ def test_stop_cleans_resources_after_stop_command_failure(agent, app, mocker): agent.metadata['stop_commands'] = ['false'] def run(command, **kwargs): - if command[-1:] == ['json'] and 'namespace' in command: - namespace = {'metadata': {'labels': {'ddev.datadoghq.com/environment': 'test-owner'}}} - return successful_process(command, stdout=json.dumps(namespace).encode()) if command[-1:] == ['name'] and 'pod' in command: return successful_process(command, stdout=b'pod/ddev-agent\n') if command[-1:] == ['false']: From 79f0f9fb17a97930a58655b0ff1db87a60000b71 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 20:54:21 +0000 Subject: [PATCH 23/50] Simplify Kubernetes Agent cluster ownership --- ddev/changelog.d/24639.added | 2 +- ddev/src/ddev/cli/env/stop.py | 4 +- ddev/src/ddev/e2e/agent/kubernetes.py | 64 +++---------- ddev/tests/cli/env/test_stop.py | 18 ++++ ddev/tests/e2e/agent/test_kubernetes.py | 119 ++++++++++-------------- docs/developer/ddev/plugins.md | 2 +- docs/developer/ddev/test.md | 11 ++- 7 files changed, 86 insertions(+), 134 deletions(-) diff --git a/ddev/changelog.d/24639.added b/ddev/changelog.d/24639.added index 41944a4c74b1b..1c49c31d51137 100644 --- a/ddev/changelog.d/24639.added +++ b/ddev/changelog.d/24639.added @@ -1 +1 @@ -Add a Kubernetes Agent backend for running integration E2E checks and Autodiscovery tests inside a cluster. +Add a Kubernetes Agent backend for running integration E2E checks and Autodiscovery tests inside disposable Kind clusters. diff --git a/ddev/src/ddev/cli/env/stop.py b/ddev/src/ddev/cli/env/stop.py index 58d4f3799343b..b3d40ebf0b455 100644 --- a/ddev/src/ddev/cli/env/stop.py +++ b/ddev/src/ddev/cli/env/stop.py @@ -73,6 +73,6 @@ def stop(app: Application, *, intg_name: str, environment: str, ignore_state: bo if process.returncode: app.abort(code=process.returncode) - # Keep the environment state when either cleanup stage fails so - # a later `ddev env stop` can retry the cleanup. + # Keep environment state when either cleanup stage fails so a + # later `ddev env stop` can retry the fixture teardown. env_data.remove() diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index 36fa29036db4b..8fd7022b6ef1e 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -3,7 +3,6 @@ # Licensed under a 3-clause BSD style license (see LICENSE) from __future__ import annotations -import hashlib import json import re import time @@ -19,17 +18,17 @@ _POD_NAME = 'ddev-agent' _CONTAINER_NAME = 'agent' -_DEFAULT_NAMESPACE_PREFIX = 'ddev-agent' +NAMESPACE = 'ddev-agent' +CLUSTER_RESOURCE_NAME = 'ddev-agent' _DEFAULT_WAIT_TIMEOUT = 120 _LOCAL_PACKAGES_METADATA = 'local_packages' class KubernetesAgent(AgentInterface): - """Run the E2E Agent inside a dedicated Kubernetes test cluster. + """Run the E2E Agent in a disposable, fixture-owned Kubernetes cluster. - Exactly one schedulable node is required, and the backend runs one Agent pod. - Cluster creation, image loading, and final teardown are the responsibility of - the environment provisioner (for example, ``kind_run``). + The backend currently supports clusters provisioned by ``kind_run``. It requires + exactly one schedulable node and runs one Agent pod in the cluster. """ build_config_key = 'docker' @@ -47,16 +46,11 @@ def _kubeconfig(self) -> str: @property def _namespace(self) -> str: - raw_id = super().get_id().lower() - normalized_id = re.sub(r'[^a-z0-9]+', '-', raw_id).strip('-') - digest = hashlib.sha256(raw_id.encode()).hexdigest()[:8] - # Leave room for the prefix, separators, and digest. - normalized_id = normalized_id[: 63 - len(_DEFAULT_NAMESPACE_PREFIX) - len(digest) - 2].rstrip('-') - return f'{_DEFAULT_NAMESPACE_PREFIX}-{normalized_id}-{digest}' + return NAMESPACE @property def _cluster_resource_name(self) -> str: - return self._namespace + return CLUSTER_RESOURCE_NAME @property def _resource_labels(self) -> dict[str, str]: @@ -230,8 +224,8 @@ def _manifest(self, agent_build: str, env_vars: dict[str, str]) -> dict[str, Any ], } - def _create_payload(self, payload: dict[str, Any]) -> None: - process = self._captured_kubectl(['create', '-f', '-'], input=json.dumps(payload).encode()) + def _create_manifest(self, manifest: dict[str, Any]) -> None: + process = self._captured_kubectl(['create', '-f', '-'], input=json.dumps(manifest).encode()) if process.returncode: raise RuntimeError(f'Unable to create Kubernetes Agent resources: {self._process_output(process)}') @@ -345,10 +339,9 @@ def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env agent_build = _normalize_agent_image_name( agent_build, self.python_version[0], self.metadata.get('use_jmx', False) ) - # Validate values needed by teardown before creating any resources. _ = self._wait_timeout self._validate_topology() - self._create_payload(self._manifest(agent_build, env_vars)) + self._create_manifest(self._manifest(agent_build, env_vars)) self._wait_for_pod() self._run_metadata_commands('start_commands') self._remember_local_packages(local_packages) @@ -359,42 +352,7 @@ def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env self._restart_agent_process() def stop(self) -> None: - errors: list[Exception] = [] - - pod = self._captured_kubectl( - ['get', 'pod', _POD_NAME, '--namespace', self._namespace, '--ignore-not-found=true', '-o', 'name'] - ) - if pod.returncode == 0 and self._process_output(pod).strip(): - try: - self._run_metadata_commands('stop_commands') - except Exception as e: - errors.append(e) - - for args in ( - [ - 'delete', - 'namespace', - self._namespace, - '--ignore-not-found=true', - '--wait=true', - f'--timeout={self._wait_timeout}s', - ], - [ - 'delete', - 'clusterrole,clusterrolebinding', - self._cluster_resource_name, - '--ignore-not-found=true', - ], - ): - process = self._captured_kubectl(args) - if process.returncode: - errors.append( - RuntimeError(f'Unable to remove Kubernetes Agent resources: {self._process_output(process)}') - ) - - if errors: - details = '; '.join(str(error) for error in errors) - raise RuntimeError(f'Errors while stopping Kubernetes Agent: {details}') from errors[0] + """Leave cleanup to the fixture that deletes the disposable cluster.""" def _restart_agent_process(self) -> None: # The Agent image's s6 finish handler normally shuts down the whole diff --git a/ddev/tests/cli/env/test_stop.py b/ddev/tests/cli/env/test_stop.py index 3990bb2385a22..792bbbddcb935 100644 --- a/ddev/tests/cli/env/test_stop.py +++ b/ddev/tests/cli/env/test_stop.py @@ -66,6 +66,24 @@ def test_failed_agent_cleanup_preserves_environment_state(ddev, data_dir, mocker teardown.assert_called_once() +def test_failed_kubernetes_fixture_teardown_preserves_environment_state(ddev, data_dir, mocker): + teardown = mocker.patch('subprocess.run', return_value=mocker.MagicMock(returncode=1)) + stop = mocker.patch('ddev.e2e.agent.kubernetes.KubernetesAgent.stop') + + integration = 'postgres' + environment = 'py3.12' + env_data = EnvDataStorage(data_dir).get(integration, environment) + metadata = {'agent_type': 'kubernetes', 'kubernetes': {'kubeconfig': '/tmp/kubeconfig'}} + env_data.write_metadata(metadata) + + result = ddev('env', 'stop', integration, environment) + + assert result.exit_code == 1, result.output + assert env_data.read_metadata() == metadata + stop.assert_called_once_with() + teardown.assert_called_once() + + def test_stop_all(ddev, helpers, data_dir, mocker): mocker.patch('subprocess.run', return_value=mocker.MagicMock(returncode=0)) stop = mocker.patch('ddev.e2e.agent.docker.DockerAgent.stop') diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index 60b9ab53f1f0a..bc4f109270e84 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -2,7 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) import json -import re import subprocess import pytest @@ -81,20 +80,6 @@ def command_calls(run_command): return [call.args[0] for call in run_command.call_args_list] -def test_namespace_metadata_cannot_override_generated_name(agent, metadata): - metadata['kubernetes']['namespace'] = 'custom-namespace' - - assert agent._namespace == 'ddev-agent-velero-py3-12-ad72b57a' - - -def test_generated_namespace_is_a_valid_kubernetes_name(app, get_integration, metadata, config_file): - integration = get_integration('Very_Long.Integration-' + 'x' * 100) - agent = KubernetesAgent(app, integration, 'Py3.12+FIPS', metadata, config_file) - - assert len(agent._namespace) == 63 - assert re.fullmatch(r'[a-z0-9]([-a-z0-9]*[a-z0-9])?', agent._namespace) - - def test_start_uses_selected_image_rbac_config_and_local_packages( agent, metadata, config_file, auto_conf, temp_dir, run_command ): @@ -102,6 +87,8 @@ def test_start_uses_selected_image_rbac_config_and_local_packages( local_base.ensure_dir_exists() integration = temp_dir / 'velero' integration.ensure_dir_exists() + metadata['start_commands'] = ['echo start'] + metadata['post_install_commands'] = ['echo post-install'] agent.start( agent_build='registry.example.com/datadog-agent:test', @@ -113,9 +100,18 @@ def test_start_uses_selected_image_rbac_config_and_local_packages( prefix = ['kubectl', '--kubeconfig', '/tmp/kubeconfig'] assert calls[0] == [*prefix, 'get', 'nodes', '-o', 'json'] - create_call = next(call for call in run_command.call_args_list if call.args[0] == [*prefix, 'create', '-f', '-']) - manifest = json.loads(create_call.kwargs['input']) + create_calls = [call for call in run_command.call_args_list if call.args[0] == [*prefix, 'create', '-f', '-']] + assert len(create_calls) == 1 + manifest = json.loads(create_calls[0].kwargs['input']) resources = {item['kind']: item for item in manifest['items']} + assert resources['Namespace']['metadata']['name'] == 'ddev-agent' + assert resources['ServiceAccount']['metadata']['namespace'] == 'ddev-agent' + assert resources['ClusterRole']['metadata']['name'] == 'ddev-agent' + assert resources['ClusterRoleBinding']['metadata']['name'] == 'ddev-agent' + assert resources['ClusterRoleBinding']['roleRef']['name'] == 'ddev-agent' + assert resources['ClusterRoleBinding']['subjects'][0]['namespace'] == 'ddev-agent' + assert resources['Pod']['metadata']['namespace'] == 'ddev-agent' + assert resources['Pod']['spec']['containers'][0]['name'] == 'agent' assert resources['Pod']['spec']['containers'][0]['image'] == 'registry.example.com/datadog-agent:test' assert resources['Pod']['spec']['containers'][0]['imagePullPolicy'] == 'Always' assert resources['Pod']['spec']['serviceAccountName'] == 'ddev-agent' @@ -146,6 +142,33 @@ def test_start_uses_selected_image_rbac_config_and_local_packages( assert 'DD_KUBERNETES_KUBELET_HOST' in env assert 'DD_KUBERNETES_KUBELET_NODENAME' in env + start_command = [ + *prefix, + 'exec', + '--namespace', + 'ddev-agent', + 'pod/ddev-agent', + '--container', + 'agent', + '--', + 'echo', + 'start', + ] + post_install_command = [ + *prefix, + 'exec', + '--namespace', + 'ddev-agent', + 'pod/ddev-agent', + '--container', + 'agent', + '--', + 'echo', + 'post-install', + ] + assert start_command in calls + assert post_install_command in calls + assert [ *prefix, 'cp', @@ -187,7 +210,7 @@ def test_start_uses_selected_image_rbac_config_and_local_packages( auto_conf.name, f'{agent._namespace}/ddev-agent:/etc/datadog-agent/conf.d/velero.d/auto_conf.yaml', ] in calls - assert [ + restart_command = [ *prefix, 'exec', '--namespace', @@ -199,7 +222,9 @@ def test_start_uses_selected_image_rbac_config_and_local_packages( 'sh', '-c', RESTART_COMMAND, - ] in calls + ] + assert restart_command in calls + assert calls.index(start_command) < calls.index(post_install_command) < calls.index(restart_command) assert metadata['kubernetes']['local_packages'] == [ {'path': str(local_base), 'name': 'datadog_checks_base', 'features': '[kube]'}, {'path': str(integration), 'name': 'velero', 'features': '[deps]'}, @@ -232,7 +257,7 @@ def test_rejects_multi_node_clusters(agent, app, mocker): run_command.assert_called_once() -def test_creation_failure_stops_startup(agent, app, mocker): +def test_partial_manifest_creation_failure_is_propagated(agent, app, mocker): nodes = {'items': [{'metadata': {'name': 'kind-control-plane'}, 'spec': {}}]} def run(command, **kwargs): @@ -345,40 +370,10 @@ def test_invoke_removes_pod_config_when_host_config_is_absent(agent, config_file ] in calls -def test_stop_is_idempotent_when_pod_is_absent(agent, run_command): +def test_stop_is_no_op(agent, run_command): agent.stop() - calls = command_calls(run_command) - prefix = ['kubectl', '--kubeconfig', '/tmp/kubeconfig'] - assert calls == [ - [ - *prefix, - 'get', - 'pod', - 'ddev-agent', - '--namespace', - agent._namespace, - '--ignore-not-found=true', - '-o', - 'name', - ], - [ - *prefix, - 'delete', - 'namespace', - agent._namespace, - '--ignore-not-found=true', - '--wait=true', - '--timeout=120s', - ], - [ - *prefix, - 'delete', - 'clusterrole,clusterrolebinding', - agent._cluster_resource_name, - '--ignore-not-found=true', - ], - ] + run_command.assert_not_called() def test_restart_resynchronizes_config_auto_conf_and_editable_sources( @@ -429,26 +424,6 @@ def test_restart_resynchronizes_config_auto_conf_and_editable_sources( assert not any('pip' in command for command in calls) -def test_stop_cleans_resources_after_stop_command_failure(agent, app, mocker): - agent.metadata['stop_commands'] = ['false'] - - def run(command, **kwargs): - if command[-1:] == ['name'] and 'pod' in command: - return successful_process(command, stdout=b'pod/ddev-agent\n') - if command[-1:] == ['false']: - raise subprocess.CalledProcessError(1, command) - return successful_process(command) - - run_command = mocker.patch.object(app.platform, 'run_command', side_effect=run) - - with pytest.raises(RuntimeError, match='Errors while stopping Kubernetes Agent'): - agent.stop() - - calls = command_calls(run_command) - assert any('namespace' in command and 'delete' in command for command in calls) - assert any('clusterrole,clusterrolebinding' in command and 'delete' in command for command in calls) - - def test_shell_and_logs_use_backend_commands(agent, run_command): agent.enter_shell() agent.show_logs() diff --git a/docs/developer/ddev/plugins.md b/docs/developer/ddev/plugins.md index 27a76dd6c5e8e..2e772797c47a4 100644 --- a/docs/developer/ddev/plugins.md +++ b/docs/developer/ddev/plugins.md @@ -141,5 +141,5 @@ is desired. This fixture is responsible for starting and stopping environments a - `env_vars` - A `dict` of environment variables and their values that will be present when starting the Agent. - `docker_volumes` - A `list` of `str` representing [Docker volume mounts][docker-volume-docs] if `agent_type` is `docker` e.g. `/local/path:/agent/container/path:ro`. - `docker_platform` - The container architecture to use if `agent_type` is `docker`. Currently, we support `linux` (default) and `windows`. -- `kubernetes` - Configuration used when `agent_type` is `kubernetes`. It requires `kubeconfig` and optionally accepts `auto_conf`, `image_pull_policy`, and `wait_timeout`. The backend owns its generated namespace, which must not already exist. +- `kubernetes` - Configuration used when `agent_type` is `kubernetes`. It requires `kubeconfig` and optionally accepts `auto_conf`, `image_pull_policy`, and `wait_timeout`. The backend is supported only with a disposable, fixture-owned cluster provisioned by `kind_run`; it uses the fixed `ddev-agent` namespace, and fixture teardown deletes the whole cluster. - `logs_config` - A `list` of configs that will be used by the Logs Agent. You will never need to use this directly, but rather via [higher level abstractions](test.md#logs). diff --git a/docs/developer/ddev/test.md b/docs/developer/ddev/test.md index db3ae37eea515..308d363a75b89 100644 --- a/docs/developer/ddev/test.md +++ b/docs/developer/ddev/test.md @@ -93,9 +93,9 @@ Note: Vagrant environments are not supported in CI environments due to virtualiz ### Kubernetes Agent -The Kubernetes Agent backend runs the Datadog Agent inside an existing Kubernetes test cluster. The cluster lifecycle -remains the responsibility of an environment helper such as `kind_run`; the backend only requires a kubeconfig and uses -standard `kubectl` operations, so it is not tied to Kind. +The Kubernetes Agent backend runs the Datadog Agent only in a disposable Kubernetes cluster that is owned by the E2E +fixture. The currently supported provisioner is `kind_run`, which creates the cluster before the fixture yields and deletes +the whole cluster during fixture teardown. The backend is not designed for shared or pre-existing clusters. ```python @pytest.fixture(scope='session') @@ -121,8 +121,9 @@ local packages requested by `--dev` or `--base`, and implements Agent commands t discovery E2E tests therefore continue to use `dd_agent_check` and `dd_agent_check_discovery`. Agent images default to the `Always` pull policy so mutable release and development tags are refreshed. Environments -that import a local image into the cluster can set `image_pull_policy` to `IfNotPresent` or `Never`. The backend generates -a namespace from the integration and environment names, then owns and deletes it and its cluster-scoped RBAC resources. +that import a local image into the cluster can set `image_pull_policy` to `IfNotPresent` or `Never`. The backend creates +one Agent in the fixed `ddev-agent` namespace with fixed cluster-scoped RBAC names. Resource cleanup happens when +`kind_run` deletes the whole disposable cluster. The initial implementation supports exactly one schedulable Kubernetes node. It rejects multi-node clusters until Agent targeting or fan-out semantics are defined. From cc84c312ed3d632bd59af6bb65934431fe8cf1a7 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 21:00:53 +0000 Subject: [PATCH 24/50] Update logs tests to Python 3.13 --- ddev/tests/cli/env/test_logs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ddev/tests/cli/env/test_logs.py b/ddev/tests/cli/env/test_logs.py index b0bae1a361d59..6c2e207eab46a 100644 --- a/ddev/tests/cli/env/test_logs.py +++ b/ddev/tests/cli/env/test_logs.py @@ -8,7 +8,7 @@ def test_nonexistent(ddev, helpers, mocker): show_logs = mocker.patch('ddev.e2e.agent.docker.DockerAgent.show_logs') integration = 'postgres' - environment = 'py3.12' + environment = 'py3.13' result = ddev('env', 'logs', integration, environment) @@ -25,7 +25,7 @@ def test_basic(ddev, data_dir, mocker): show_logs = mocker.patch('ddev.e2e.agent.docker.DockerAgent.show_logs') integration = 'postgres' - environment = 'py3.12' + environment = 'py3.13' env_data = EnvDataStorage(data_dir).get(integration, environment) env_data.write_metadata({}) @@ -42,7 +42,7 @@ def test_log_failure_is_propagated(ddev, data_dir, mocker): ) integration = 'postgres' - environment = 'py3.12' + environment = 'py3.13' EnvDataStorage(data_dir).get(integration, environment).write_metadata({}) result = ddev('env', 'logs', integration, environment) From 4c14f0eae3226c4c81ba5bad21a9833c1c50cdda Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 21:20:56 +0000 Subject: [PATCH 25/50] Refine Kubernetes command contract tests --- ddev/tests/e2e/agent/test_kubernetes.py | 386 +++++++++++------------- 1 file changed, 178 insertions(+), 208 deletions(-) diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index bc4f109270e84..1cb8bd0fe0588 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -10,17 +10,10 @@ from ddev.integration.core import Integration from ddev.repo.config import RepositoryConfig -RESTART_COMMAND = ( - 'old_pid=$(pidof agent) || exit 1; ' - 'set -- $old_pid; old_pid=$1; ' - 'rm -f /var/run/s6/services/agent/finish; ' - 'kill "$old_pid" || exit 1; ' - 'elapsed=0; ' - 'while kill -0 "$old_pid" 2>/dev/null; do ' - '[ "$elapsed" -ge 120 ] && exit 1; ' - 'sleep 1; elapsed=$((elapsed + 1)); ' - 'done' -) +TEST_KUBECONFIG = '/tmp/kubeconfig' +TEST_NAMESPACE = 'ddev-agent' +TEST_POD = 'ddev-agent' +TEST_CONTAINER = 'agent' @pytest.fixture(scope='module') @@ -50,7 +43,7 @@ def auto_conf(temp_dir): def metadata(auto_conf): return { 'kubernetes': { - 'kubeconfig': '/tmp/kubeconfig', + 'kubeconfig': TEST_KUBECONFIG, 'auto_conf': str(auto_conf), }, } @@ -80,6 +73,117 @@ def command_calls(run_command): return [call.args[0] for call in run_command.call_args_list] +def option_value(command: list[str], option: str) -> str: + return command[command.index(option) + 1] + + +def find_exec_command(calls: list[list[str]], expected: list[str], *, prefix: bool = False) -> int: + matches = [] + for index, command in enumerate(calls): + if 'exec' not in command or '--' not in command: + continue + + separator = command.index('--') + payload = command[separator + 1 :] + matches_expected = payload[: len(expected)] == expected if prefix else payload == expected + if not matches_expected: + continue + + kubectl_args = command[:separator] + assert command[0] == 'kubectl' + assert option_value(kubectl_args, '--kubeconfig') == TEST_KUBECONFIG + assert option_value(kubectl_args, '--namespace') == TEST_NAMESPACE + assert f'pod/{TEST_POD}' in kubectl_args + assert option_value(kubectl_args, '--container') == TEST_CONTAINER + matches.append(index) + + assert len(matches) == 1, f'Expected one kubectl exec payload matching {expected!r}, found {len(matches)}' + return matches[0] + + +def find_copy_command(calls: list[list[str]], source: str, destination: str) -> int: + matches = [] + for index, command in enumerate(calls): + if 'cp' not in command or source not in command or destination not in command: + continue + + assert command[0] == 'kubectl' + assert option_value(command, '--kubeconfig') == TEST_KUBECONFIG + assert option_value(command, '--container') == TEST_CONTAINER + assert command.index(source) < command.index(destination) + matches.append(index) + + assert len(matches) == 1, f'Expected one kubectl cp from {source!r} to {destination!r}, found {len(matches)}' + return matches[0] + + +def test_exec_builds_scoped_command_and_sorts_environment(agent, run_command): + agent._exec(['agent', 'check', 'velero'], env_vars={'ZED': 'last', 'ALPHA': 'first'}) + + run_command.assert_called_once_with( + [ + 'kubectl', + '--kubeconfig', + TEST_KUBECONFIG, + 'exec', + '--namespace', + TEST_NAMESPACE, + f'pod/{TEST_POD}', + '--container', + TEST_CONTAINER, + '--', + 'env', + 'ALPHA=first', + 'ZED=last', + 'agent', + 'check', + 'velero', + ], + check=True, + ) + + +def test_copy_file_builds_scoped_command(agent, config_file, run_command): + destination = '/tmp/conf.yaml' + + agent._copy_file(str(config_file), destination) + + run_command.assert_called_once_with( + [ + 'kubectl', + '--kubeconfig', + TEST_KUBECONFIG, + 'cp', + '--container', + TEST_CONTAINER, + config_file.name, + f'{TEST_NAMESPACE}/{TEST_POD}:{destination}', + ], + check=True, + cwd=config_file.resolve().parent, + ) + + +def test_restart_command_preserves_s6_and_timeout_contract(agent, metadata, mocker): + metadata['kubernetes']['wait_timeout'] = 17 + operations = mocker.Mock() + execute = mocker.patch.object(agent, '_exec') + wait_for_agent = mocker.patch.object(agent, '_wait_for_agent') + operations.attach_mock(execute, 'execute') + operations.attach_mock(wait_for_agent, 'wait_for_agent') + + agent._restart_agent_process() + + operations.assert_has_calls([mocker.call.execute(mocker.ANY), mocker.call.wait_for_agent()]) + command = execute.call_args.args[0] + assert command[:2] == ['sh', '-c'] + script = command[2] + assert 'pidof agent' in script + assert 'rm -f /var/run/s6/services/agent/finish' in script + assert 'kill "$old_pid"' in script + assert '[ "$elapsed" -ge 17 ]' in script + + def test_start_uses_selected_image_rbac_config_and_local_packages( agent, metadata, config_file, auto_conf, temp_dir, run_command ): @@ -97,7 +201,7 @@ def test_start_uses_selected_image_rbac_config_and_local_packages( ) calls = command_calls(run_command) - prefix = ['kubectl', '--kubeconfig', '/tmp/kubeconfig'] + prefix = ['kubectl', '--kubeconfig', TEST_KUBECONFIG] assert calls[0] == [*prefix, 'get', 'nodes', '-o', 'json'] create_calls = [call for call in run_command.call_args_list if call.args[0] == [*prefix, 'create', '-f', '-']] @@ -142,94 +246,42 @@ def test_start_uses_selected_image_rbac_config_and_local_packages( assert 'DD_KUBERNETES_KUBELET_HOST' in env assert 'DD_KUBERNETES_KUBELET_NODENAME' in env - start_command = [ - *prefix, - 'exec', - '--namespace', - 'ddev-agent', - 'pod/ddev-agent', - '--container', - 'agent', - '--', - 'echo', - 'start', - ] - post_install_command = [ - *prefix, - 'exec', - '--namespace', - 'ddev-agent', - 'pod/ddev-agent', - '--container', - 'agent', - '--', - 'echo', - 'post-install', - ] - assert start_command in calls - assert post_install_command in calls - - assert [ - *prefix, - 'cp', - '--container', - 'agent', + start_index = find_exec_command(calls, ['echo', 'start']) + post_install_index = find_exec_command(calls, ['echo', 'post-install']) + local_base_copy_index = find_copy_command( + calls, local_base.name, - f'{agent._namespace}/ddev-agent:/home/datadog_checks_base', - ] in calls - assert [ - *prefix, - 'exec', - '--namespace', - agent._namespace, - 'pod/ddev-agent', - '--container', - 'agent', - '--', - '/opt/datadog-agent/embedded/bin/python3', - '-m', - 'pip', - 'install', - '--disable-pip-version-check', - '-e', - '/home/datadog_checks_base[kube]', - ] in calls - assert [ - *prefix, - 'cp', - '--container', - 'agent', + f'{TEST_NAMESPACE}/{TEST_POD}:/home/datadog_checks_base', + ) + find_exec_command( + calls, + [ + '/opt/datadog-agent/embedded/bin/python3', + '-m', + 'pip', + 'install', + '--disable-pip-version-check', + '-e', + '/home/datadog_checks_base[kube]', + ], + ) + find_copy_command( + calls, config_file.name, - f'{agent._namespace}/ddev-agent:/etc/datadog-agent/conf.d/velero.d/conf.yaml', - ] in calls - assert [ - *prefix, - 'cp', - '--container', - 'agent', + f'{TEST_NAMESPACE}/{TEST_POD}:/etc/datadog-agent/conf.d/velero.d/conf.yaml', + ) + find_copy_command( + calls, auto_conf.name, - f'{agent._namespace}/ddev-agent:/etc/datadog-agent/conf.d/velero.d/auto_conf.yaml', - ] in calls - restart_command = [ - *prefix, - 'exec', - '--namespace', - agent._namespace, - 'pod/ddev-agent', - '--container', - 'agent', - '--', - 'sh', - '-c', - RESTART_COMMAND, - ] - assert restart_command in calls - assert calls.index(start_command) < calls.index(post_install_command) < calls.index(restart_command) + f'{TEST_NAMESPACE}/{TEST_POD}:/etc/datadog-agent/conf.d/velero.d/auto_conf.yaml', + ) + restart_index = find_exec_command(calls, ['sh', '-c'], prefix=True) + assert start_index < post_install_index < restart_index assert metadata['kubernetes']['local_packages'] == [ {'path': str(local_base), 'name': 'datadog_checks_base', 'features': '[kube]'}, {'path': str(integration), 'name': 'velero', 'features': '[deps]'}, ] - local_base_copy = next(call for call in run_command.call_args_list if call.args[0][-2] == local_base.name) + local_base_copy = run_command.call_args_list[local_base_copy_index] assert local_base_copy.kwargs['cwd'] == local_base.resolve().parent @@ -277,74 +329,27 @@ def run(command, **kwargs): assert not any('apply' in command or 'wait' in command for command in calls) -def test_invoke_synchronizes_config_and_environment(agent, metadata, config_file, auto_conf, run_command): +def test_invoke_synchronizes_config_and_environment(agent, config_file, auto_conf, run_command): config_file.write_text('instances:\n - openmetrics_endpoint: http://velero:8085/metrics\n') agent.invoke(['check', 'velero', '--json'], env_vars={'ZED': 'last', 'ALPHA': 'first'}) calls = command_calls(run_command) - prefix = ['kubectl', '--kubeconfig', '/tmp/kubeconfig'] - assert calls == [ - [ - *prefix, - 'exec', - '--namespace', - agent._namespace, - 'pod/ddev-agent', - '--container', - 'agent', - '--', - 'mkdir', - '-p', - '/etc/datadog-agent/conf.d/velero.d', - ], - [ - *prefix, - 'cp', - '--container', - 'agent', - config_file.name, - f'{agent._namespace}/ddev-agent:/etc/datadog-agent/conf.d/velero.d/conf.yaml', - ], - [ - *prefix, - 'exec', - '--namespace', - agent._namespace, - 'pod/ddev-agent', - '--container', - 'agent', - '--', - 'mkdir', - '-p', - '/etc/datadog-agent/conf.d/velero.d', - ], - [ - *prefix, - 'cp', - '--container', - 'agent', - auto_conf.name, - f'{agent._namespace}/ddev-agent:/etc/datadog-agent/conf.d/velero.d/auto_conf.yaml', - ], - [ - *prefix, - 'exec', - '--namespace', - agent._namespace, - 'pod/ddev-agent', - '--container', - 'agent', - '--', - 'env', - 'ALPHA=first', - 'ZED=last', - 'agent', - 'check', - 'velero', - '--json', - ], - ] + config_copy_index = find_copy_command( + calls, + config_file.name, + f'{TEST_NAMESPACE}/{TEST_POD}:/etc/datadog-agent/conf.d/velero.d/conf.yaml', + ) + auto_conf_copy_index = find_copy_command( + calls, + auto_conf.name, + f'{TEST_NAMESPACE}/{TEST_POD}:/etc/datadog-agent/conf.d/velero.d/auto_conf.yaml', + ) + invoke_index = find_exec_command( + calls, + ['env', 'ALPHA=first', 'ZED=last', 'agent', 'check', 'velero', '--json'], + ) + assert config_copy_index < auto_conf_copy_index < invoke_index def test_invoke_removes_pod_config_when_host_config_is_absent(agent, config_file, run_command): @@ -353,21 +358,7 @@ def test_invoke_removes_pod_config_when_host_config_is_absent(agent, config_file agent.invoke(['status']) calls = command_calls(run_command) - assert [ - 'kubectl', - '--kubeconfig', - '/tmp/kubeconfig', - 'exec', - '--namespace', - agent._namespace, - 'pod/ddev-agent', - '--container', - 'agent', - '--', - 'rm', - '-f', - '/etc/datadog-agent/conf.d/velero.d/conf.yaml', - ] in calls + find_exec_command(calls, ['rm', '-f', '/etc/datadog-agent/conf.d/velero.d/conf.yaml']) def test_stop_is_no_op(agent, run_command): @@ -388,39 +379,18 @@ def test_restart_resynchronizes_config_auto_conf_and_editable_sources( agent.restart() calls = command_calls(run_command) - prefix = ['kubectl', '--kubeconfig', '/tmp/kubeconfig'] - package_copy = [ - *prefix, - 'cp', - '--container', - 'agent', + package_copy_index = find_copy_command( + calls, local_package.name, - f'{agent._namespace}/ddev-agent:/home/velero-source', - ] - config_copy = [ - *prefix, - 'cp', - '--container', - 'agent', + f'{TEST_NAMESPACE}/{TEST_POD}:/home/velero-source', + ) + config_copy_index = find_copy_command( + calls, config_file.name, - f'{agent._namespace}/ddev-agent:/etc/datadog-agent/conf.d/velero.d/conf.yaml', - ] - restart = [ - *prefix, - 'exec', - '--namespace', - agent._namespace, - 'pod/ddev-agent', - '--container', - 'agent', - '--', - 'sh', - '-c', - RESTART_COMMAND, - ] - assert package_copy in calls - assert config_copy in calls - assert calls.index(package_copy) < calls.index(config_copy) < calls.index(restart) + f'{TEST_NAMESPACE}/{TEST_POD}:/etc/datadog-agent/conf.d/velero.d/conf.yaml', + ) + restart_index = find_exec_command(calls, ['sh', '-c'], prefix=True) + assert package_copy_index < config_copy_index < restart_index assert not any('pip' in command for command in calls) @@ -429,21 +399,21 @@ def test_shell_and_logs_use_backend_commands(agent, run_command): agent.show_logs() calls = command_calls(run_command) - prefix = ['kubectl', '--kubeconfig', '/tmp/kubeconfig'] + prefix = ['kubectl', '--kubeconfig', TEST_KUBECONFIG] assert calls == [ [ *prefix, 'exec', '-it', '--namespace', - agent._namespace, - 'pod/ddev-agent', + TEST_NAMESPACE, + f'pod/{TEST_POD}', '--container', - 'agent', + TEST_CONTAINER, '--', 'bash', ], - [*prefix, 'logs', '--namespace', agent._namespace, 'pod/ddev-agent', '--container', 'agent'], + [*prefix, 'logs', '--namespace', TEST_NAMESPACE, f'pod/{TEST_POD}', '--container', TEST_CONTAINER], ] assert run_command.call_args_list[-1].kwargs['check'] is True From 449f6c18f963b0a55eccbed96cc221f487338165 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 22:12:10 +0000 Subject: [PATCH 26/50] Test Kubernetes topology command directly --- ddev/tests/e2e/agent/test_kubernetes.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index 1cb8bd0fe0588..89fcefc777765 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -202,7 +202,6 @@ def test_start_uses_selected_image_rbac_config_and_local_packages( calls = command_calls(run_command) prefix = ['kubectl', '--kubeconfig', TEST_KUBECONFIG] - assert calls[0] == [*prefix, 'get', 'nodes', '-o', 'json'] create_calls = [call for call in run_command.call_args_list if call.args[0] == [*prefix, 'create', '-f', '-']] assert len(create_calls) == 1 @@ -306,7 +305,11 @@ def test_rejects_multi_node_clusters(agent, app, mocker): with pytest.raises(NotImplementedError, match='exactly one schedulable node'): agent.start(agent_build='', local_packages={}, env_vars={}) - run_command.assert_called_once() + run_command.assert_called_once_with( + ['kubectl', '--kubeconfig', TEST_KUBECONFIG, 'get', 'nodes', '-o', 'json'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) def test_partial_manifest_creation_failure_is_propagated(agent, app, mocker): From 79e4a9d15996395954cde379b341d3543b6b113d Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 22:13:54 +0000 Subject: [PATCH 27/50] Reject non-Kind Kubernetes Agent contexts --- ddev/src/ddev/e2e/agent/kubernetes.py | 10 ++++++ ddev/tests/e2e/agent/test_kubernetes.py | 44 ++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index 8fd7022b6ef1e..b8a6b9e3bd419 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -110,6 +110,15 @@ def _exec( return self._captured_kubectl(args, check=check) return self._kubectl(args, check=check) + def _validate_context(self) -> None: + process = self._captured_kubectl(['config', 'current-context']) + if process.returncode: + raise RuntimeError(f'Unable to inspect Kubernetes context: {self._process_output(process)}') + + context = self._process_output(process).strip() + if not context.startswith('kind-'): + raise RuntimeError(f'Refusing to use non-Kind Kubernetes context `{context}`') + def _validate_topology(self) -> None: process = self._captured_kubectl(['get', 'nodes', '-o', 'json']) if process.returncode: @@ -340,6 +349,7 @@ def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env agent_build, self.python_version[0], self.metadata.get('use_jmx', False) ) _ = self._wait_timeout + self._validate_context() self._validate_topology() self._create_manifest(self._manifest(agent_build, env_vars)) self._wait_for_pod() diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index 89fcefc777765..811cac6b490eb 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -61,6 +61,8 @@ def successful_process(command, *, stdout=b''): @pytest.fixture def run_command(app, mocker): def run(command, **kwargs): + if command[-2:] == ['config', 'current-context']: + return successful_process(command, stdout=b'kind-test\n') if command[-4:] == ['get', 'nodes', '-o', 'json']: nodes = {'items': [{'metadata': {'name': 'kind-control-plane'}, 'spec': {}}]} return successful_process(command, stdout=json.dumps(nodes).encode()) @@ -202,6 +204,10 @@ def test_start_uses_selected_image_rbac_config_and_local_packages( calls = command_calls(run_command) prefix = ['kubectl', '--kubeconfig', TEST_KUBECONFIG] + assert calls[:2] == [ + [*prefix, 'config', 'current-context'], + [*prefix, 'get', 'nodes', '-o', 'json'], + ] create_calls = [call for call in run_command.call_args_list if call.args[0] == [*prefix, 'create', '-f', '-']] assert len(create_calls) == 1 @@ -294,28 +300,56 @@ def test_rejects_invalid_wait_timeout_before_creating_resources(agent, metadata, run_command.assert_not_called() -def test_rejects_multi_node_clusters(agent, app, mocker): - nodes = {'items': [{'metadata': {'name': 'one'}, 'spec': {}}, {'metadata': {'name': 'two'}, 'spec': {}}]} +def test_rejects_non_kind_context_before_inspecting_cluster(agent, app, mocker): run_command = mocker.patch.object( app.platform, 'run_command', - return_value=successful_process([], stdout=json.dumps(nodes).encode()), + return_value=successful_process([], stdout=b'prod\n'), ) - with pytest.raises(NotImplementedError, match='exactly one schedulable node'): + with pytest.raises(RuntimeError, match='non-Kind Kubernetes context `prod`'): agent.start(agent_build='', local_packages={}, env_vars={}) run_command.assert_called_once_with( - ['kubectl', '--kubeconfig', TEST_KUBECONFIG, 'get', 'nodes', '-o', 'json'], + ['kubectl', '--kubeconfig', TEST_KUBECONFIG, 'config', 'current-context'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) +def test_rejects_multi_node_clusters(agent, app, mocker): + nodes = {'items': [{'metadata': {'name': 'one'}, 'spec': {}}, {'metadata': {'name': 'two'}, 'spec': {}}]} + + def run(command, **kwargs): + if command[-2:] == ['config', 'current-context']: + return successful_process(command, stdout=b'kind-test\n') + return successful_process(command, stdout=json.dumps(nodes).encode()) + + run_command = mocker.patch.object(app.platform, 'run_command', side_effect=run) + + with pytest.raises(NotImplementedError, match='exactly one schedulable node'): + agent.start(agent_build='', local_packages={}, env_vars={}) + + assert run_command.call_args_list == [ + mocker.call( + ['kubectl', '--kubeconfig', TEST_KUBECONFIG, 'config', 'current-context'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ), + mocker.call( + ['kubectl', '--kubeconfig', TEST_KUBECONFIG, 'get', 'nodes', '-o', 'json'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ), + ] + + def test_partial_manifest_creation_failure_is_propagated(agent, app, mocker): nodes = {'items': [{'metadata': {'name': 'kind-control-plane'}, 'spec': {}}]} def run(command, **kwargs): + if command[-2:] == ['config', 'current-context']: + return successful_process(command, stdout=b'kind-test\n') if command[-4:] == ['get', 'nodes', '-o', 'json']: return successful_process(command, stdout=json.dumps(nodes).encode()) if command[-3:] == ['create', '-f', '-']: From b530ccd8d35ba374c30a13465759add0a2a16451 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 23 Jul 2026 22:21:37 +0000 Subject: [PATCH 28/50] Relax Kubernetes orchestration assertions --- ddev/tests/e2e/agent/test_kubernetes.py | 48 ++++++++++++++++--------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index 811cac6b490eb..c62638aabbb33 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -79,6 +79,25 @@ def option_value(command: list[str], option: str) -> str: return command[command.index(option) + 1] +def find_kubectl_command(calls: list[list[str]], expected: list[str]) -> int: + matches = [] + for index, command in enumerate(calls): + kubectl_args = command[: command.index('--')] if '--' in command else command + contains_expected = any( + kubectl_args[start : start + len(expected)] == expected + for start in range(len(kubectl_args) - len(expected) + 1) + ) + if not contains_expected: + continue + + assert command[0] == 'kubectl' + assert option_value(kubectl_args, '--kubeconfig') == TEST_KUBECONFIG + matches.append(index) + + assert len(matches) == 1, f'Expected one kubectl command containing {expected!r}, found {len(matches)}' + return matches[0] + + def find_exec_command(calls: list[list[str]], expected: list[str], *, prefix: bool = False) -> int: matches = [] for index, command in enumerate(calls): @@ -204,13 +223,12 @@ def test_start_uses_selected_image_rbac_config_and_local_packages( calls = command_calls(run_command) prefix = ['kubectl', '--kubeconfig', TEST_KUBECONFIG] - assert calls[:2] == [ - [*prefix, 'config', 'current-context'], - [*prefix, 'get', 'nodes', '-o', 'json'], - ] - + context_index = find_kubectl_command(calls, ['config', 'current-context']) + topology_index = find_kubectl_command(calls, ['get', 'nodes', '-o', 'json']) create_calls = [call for call in run_command.call_args_list if call.args[0] == [*prefix, 'create', '-f', '-']] assert len(create_calls) == 1 + create_index = run_command.call_args_list.index(create_calls[0]) + assert context_index < topology_index < create_index manifest = json.loads(create_calls[0].kwargs['input']) resources = {item['kind']: item for item in manifest['items']} assert resources['Namespace']['metadata']['name'] == 'ddev-agent' @@ -330,18 +348,14 @@ def run(command, **kwargs): with pytest.raises(NotImplementedError, match='exactly one schedulable node'): agent.start(agent_build='', local_packages={}, env_vars={}) - assert run_command.call_args_list == [ - mocker.call( - ['kubectl', '--kubeconfig', TEST_KUBECONFIG, 'config', 'current-context'], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ), - mocker.call( - ['kubectl', '--kubeconfig', TEST_KUBECONFIG, 'get', 'nodes', '-o', 'json'], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ), - ] + calls = command_calls(run_command) + context_index = find_kubectl_command(calls, ['config', 'current-context']) + topology_index = find_kubectl_command(calls, ['get', 'nodes', '-o', 'json']) + assert context_index < topology_index + assert not any('create' in command for command in calls) + for index in (context_index, topology_index): + assert run_command.call_args_list[index].kwargs['stdout'] is subprocess.PIPE + assert run_command.call_args_list[index].kwargs['stderr'] is subprocess.STDOUT def test_partial_manifest_creation_failure_is_propagated(agent, app, mocker): From 3e36fe43ebd8a96fbd039107177703f924a11149 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Fri, 24 Jul 2026 07:16:14 +0200 Subject: [PATCH 29/50] Update Kubernetes Agent description for E2E tests --- ddev/changelog.d/24639.added | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddev/changelog.d/24639.added b/ddev/changelog.d/24639.added index 1c49c31d51137..d811757c468b6 100644 --- a/ddev/changelog.d/24639.added +++ b/ddev/changelog.d/24639.added @@ -1 +1 @@ -Add a Kubernetes Agent backend for running integration E2E checks and Autodiscovery tests inside disposable Kind clusters. +Add a Kubernetes Agent interface for running the Agent in Kind-based E2E test clusters, alongside the existing Docker and Vagrant interfaces. From 1742361d709866262a19a8a83429c9d5eb06b508 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Fri, 24 Jul 2026 05:22:13 +0000 Subject: [PATCH 30/50] Simplify Kubernetes local package metadata --- ddev/src/ddev/e2e/agent/kubernetes.py | 36 +++++++++++-------------- ddev/tests/e2e/agent/test_kubernetes.py | 21 ++++++++++----- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index b8a6b9e3bd419..f67ea9d15d602 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -4,7 +4,6 @@ from __future__ import annotations import json -import re import time from typing import TYPE_CHECKING, Any, cast @@ -300,30 +299,25 @@ def _sync_auto_conf(self) -> None: self._copy_file(auto_conf, f'{self._config_dir}/auto_conf.yaml') def _remember_local_packages(self, local_packages: dict[Path, str]) -> None: - self._kubernetes_metadata[_LOCAL_PACKAGES_METADATA] = [ - {'path': str(local_package), 'name': local_package.name, 'features': features} - for local_package, features in local_packages.items() - ] + self._kubernetes_metadata[_LOCAL_PACKAGES_METADATA] = { + str(local_package): features for local_package, features in local_packages.items() + } - def _local_package_specs(self) -> list[dict[str, str]]: - specs = self._kubernetes_metadata.get(_LOCAL_PACKAGES_METADATA, []) - if not isinstance(specs, list): - raise ValueError('Kubernetes Agent `local_packages` must be a list') + def _local_packages(self) -> dict[Path, str]: + from ddev.utils.fs import Path - for spec in specs: - if not isinstance(spec, dict) or not all( - isinstance(spec.get(key), str) for key in ('path', 'name', 'features') - ): - raise ValueError('Kubernetes Agent local package entries must define path, name, and features strings') - if not re.fullmatch(r'[A-Za-z0-9_.-]+', spec['name']) or spec['name'] in {'.', '..'}: - raise ValueError(f'Invalid Kubernetes Agent local package name: {spec["name"]!r}') - return specs + local_packages = cast(dict[str, str], self._kubernetes_metadata.get(_LOCAL_PACKAGES_METADATA, {})) + return {Path(path): features for path, features in local_packages.items()} def _sync_local_packages(self, *, install: bool = False) -> None: - for spec in self._local_package_specs(): - destination = f'/home/{spec["name"]}' + for local_package, features in self._local_packages().items(): + name = local_package.name + if not name or name in {'.', '..'}: + raise ValueError(f'Invalid Kubernetes Agent local package path: {local_package}') + + destination = f'/home/{name}' self._exec(['rm', '-rf', destination]) - self._copy_file(spec['path'], destination) + self._copy_file(str(local_package), destination) if install: self._exec( [ @@ -333,7 +327,7 @@ def _sync_local_packages(self, *, install: bool = False) -> None: 'install', '--disable-pip-version-check', '-e', - f'{destination}{spec["features"]}', + f'{destination}{features}', ] ) diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index c62638aabbb33..84d2a4db98ad0 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -300,10 +300,10 @@ def test_start_uses_selected_image_rbac_config_and_local_packages( ) restart_index = find_exec_command(calls, ['sh', '-c'], prefix=True) assert start_index < post_install_index < restart_index - assert metadata['kubernetes']['local_packages'] == [ - {'path': str(local_base), 'name': 'datadog_checks_base', 'features': '[kube]'}, - {'path': str(integration), 'name': 'velero', 'features': '[deps]'}, - ] + assert metadata['kubernetes']['local_packages'] == { + str(local_base): '[kube]', + str(integration): '[deps]', + } local_base_copy = run_command.call_args_list[local_base_copy_index] assert local_base_copy.kwargs['cwd'] == local_base.resolve().parent @@ -423,9 +423,7 @@ def test_restart_resynchronizes_config_auto_conf_and_editable_sources( ): local_package = temp_dir / 'velero-source' local_package.ensure_dir_exists() - metadata['kubernetes']['local_packages'] = [ - {'path': str(local_package), 'name': 'velero-source', 'features': '[deps]'} - ] + metadata['kubernetes']['local_packages'] = {str(local_package): '[deps]'} agent.restart() @@ -445,6 +443,15 @@ def test_restart_resynchronizes_config_auto_conf_and_editable_sources( assert not any('pip' in command for command in calls) +def test_restart_rejects_unsafe_local_package_destination(agent, metadata, run_command): + metadata['kubernetes']['local_packages'] = {'..': '[deps]'} + + with pytest.raises(ValueError, match='local package path'): + agent.restart() + + run_command.assert_not_called() + + def test_shell_and_logs_use_backend_commands(agent, run_command): agent.enter_shell() agent.show_logs() From bb32463980711521464338e294f947c67f38f0f9 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Fri, 24 Jul 2026 05:39:33 +0000 Subject: [PATCH 31/50] Restore environment state cleanup ordering --- ddev/src/ddev/cli/env/stop.py | 6 ++---- ddev/tests/cli/env/test_stop.py | 8 ++++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/ddev/src/ddev/cli/env/stop.py b/ddev/src/ddev/cli/env/stop.py index b3d40ebf0b455..a73632e51bec0 100644 --- a/ddev/src/ddev/cli/env/stop.py +++ b/ddev/src/ddev/cli/env/stop.py @@ -68,11 +68,9 @@ def stop(app: Application, *, intg_name: str, environment: str, ignore_state: bo try: agent.stop() finally: + env_data.remove() + with integration.path.as_cwd(env_vars=env_vars), runner.stop() as command: process = app.platform.run_command(command) if process.returncode: app.abort(code=process.returncode) - - # Keep environment state when either cleanup stage fails so a - # later `ddev env stop` can retry the fixture teardown. - env_data.remove() diff --git a/ddev/tests/cli/env/test_stop.py b/ddev/tests/cli/env/test_stop.py index 792bbbddcb935..aa253df83776d 100644 --- a/ddev/tests/cli/env/test_stop.py +++ b/ddev/tests/cli/env/test_stop.py @@ -48,7 +48,7 @@ def test_basic(ddev, helpers, data_dir, mocker): stop.assert_called_once() -def test_failed_agent_cleanup_preserves_environment_state(ddev, data_dir, mocker): +def test_failed_agent_cleanup_removes_environment_state(ddev, data_dir, mocker): teardown = mocker.patch('subprocess.run', return_value=mocker.MagicMock(returncode=0)) stop = mocker.patch('ddev.e2e.agent.docker.DockerAgent.stop', side_effect=RuntimeError('cleanup failed')) @@ -61,12 +61,12 @@ def test_failed_agent_cleanup_preserves_environment_state(ddev, data_dir, mocker with pytest.raises(RuntimeError, match='cleanup failed'): ddev('env', 'stop', integration, environment) - assert env_data.read_metadata() == metadata + assert not env_data.exists() stop.assert_called_once_with() teardown.assert_called_once() -def test_failed_kubernetes_fixture_teardown_preserves_environment_state(ddev, data_dir, mocker): +def test_failed_kubernetes_fixture_teardown_removes_environment_state(ddev, data_dir, mocker): teardown = mocker.patch('subprocess.run', return_value=mocker.MagicMock(returncode=1)) stop = mocker.patch('ddev.e2e.agent.kubernetes.KubernetesAgent.stop') @@ -79,7 +79,7 @@ def test_failed_kubernetes_fixture_teardown_preserves_environment_state(ddev, da result = ddev('env', 'stop', integration, environment) assert result.exit_code == 1, result.output - assert env_data.read_metadata() == metadata + assert not env_data.exists() stop.assert_called_once_with() teardown.assert_called_once() From fe768f835fe35ac3fa9968faac5dafb45c7744e2 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Fri, 24 Jul 2026 07:50:06 +0000 Subject: [PATCH 32/50] Wait for Kubernetes Agent before restart --- ddev/src/ddev/e2e/agent/kubernetes.py | 3 +++ ddev/tests/e2e/agent/test_kubernetes.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index f67ea9d15d602..f9a5aca5b76e6 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -359,6 +359,9 @@ def stop(self) -> None: """Leave cleanup to the fixture that deletes the disposable cluster.""" def _restart_agent_process(self) -> None: + # Pod readiness does not guarantee that s6 has started the Agent process. + self._wait_for_agent() + # The Agent image's s6 finish handler normally shuts down the whole # service tree when the main Agent exits. Remove it before killing the # process so s6 starts a fresh Agent in the same container, preserving diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index 84d2a4db98ad0..f744d3bac5e78 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -195,7 +195,9 @@ def test_restart_command_preserves_s6_and_timeout_contract(agent, metadata, mock agent._restart_agent_process() - operations.assert_has_calls([mocker.call.execute(mocker.ANY), mocker.call.wait_for_agent()]) + operations.assert_has_calls( + [mocker.call.wait_for_agent(), mocker.call.execute(mocker.ANY), mocker.call.wait_for_agent()] + ) command = execute.call_args.args[0] assert command[:2] == ['sh', '-c'] script = command[2] From 169801dd4cc00fc21758fb4b252138efd386dc70 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Fri, 24 Jul 2026 08:33:05 +0000 Subject: [PATCH 33/50] Fail fast on unsupported Agent supervision --- ddev/src/ddev/e2e/agent/kubernetes.py | 4 +++- ddev/tests/e2e/agent/test_kubernetes.py | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index f9a5aca5b76e6..22ec3ebe31fbd 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -369,7 +369,9 @@ def _restart_agent_process(self) -> None: restart_command = ( 'old_pid=$(pidof agent) || exit 1; ' 'set -- $old_pid; old_pid=$1; ' - 'rm -f /var/run/s6/services/agent/finish; ' + 'finish=/var/run/s6/services/agent/finish; ' + '[ -f "$finish" ] || { echo "Unsupported Agent image: s6 Agent finish hook not found" >&2; exit 1; }; ' + 'rm "$finish" || exit 1; ' 'kill "$old_pid" || exit 1; ' 'elapsed=0; ' 'while kill -0 "$old_pid" 2>/dev/null; do ' diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index f744d3bac5e78..945bb2db846e4 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -202,8 +202,11 @@ def test_restart_command_preserves_s6_and_timeout_contract(agent, metadata, mock assert command[:2] == ['sh', '-c'] script = command[2] assert 'pidof agent' in script - assert 'rm -f /var/run/s6/services/agent/finish' in script - assert 'kill "$old_pid"' in script + assert 'finish=/var/run/s6/services/agent/finish' in script + assert '[ -f "$finish" ]' in script + assert 'Unsupported Agent image: s6 Agent finish hook not found' in script + assert 'rm "$finish"' in script + assert script.index('[ -f "$finish" ]') < script.index('kill "$old_pid"') assert '[ "$elapsed" -ge 17 ]' in script From 3806d7c3af4adf311328ed801c20ad62b33e380e Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Fri, 24 Jul 2026 08:37:54 +0000 Subject: [PATCH 34/50] Simplify Kubernetes Agent restart checks --- ddev/src/ddev/e2e/agent/kubernetes.py | 4 +--- ddev/tests/e2e/agent/test_kubernetes.py | 21 ++++++--------------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index 22ec3ebe31fbd..3b0cd4e0068d8 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -369,9 +369,7 @@ def _restart_agent_process(self) -> None: restart_command = ( 'old_pid=$(pidof agent) || exit 1; ' 'set -- $old_pid; old_pid=$1; ' - 'finish=/var/run/s6/services/agent/finish; ' - '[ -f "$finish" ] || { echo "Unsupported Agent image: s6 Agent finish hook not found" >&2; exit 1; }; ' - 'rm "$finish" || exit 1; ' + 'rm /var/run/s6/services/agent/finish || exit 1; ' 'kill "$old_pid" || exit 1; ' 'elapsed=0; ' 'while kill -0 "$old_pid" 2>/dev/null; do ' diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index 945bb2db846e4..a657235fe66df 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -185,8 +185,7 @@ def test_copy_file_builds_scoped_command(agent, config_file, run_command): ) -def test_restart_command_preserves_s6_and_timeout_contract(agent, metadata, mocker): - metadata['kubernetes']['wait_timeout'] = 17 +def test_restart_waits_for_agent_before_and_after_restart(agent, mocker): operations = mocker.Mock() execute = mocker.patch.object(agent, '_exec') wait_for_agent = mocker.patch.object(agent, '_wait_for_agent') @@ -195,19 +194,11 @@ def test_restart_command_preserves_s6_and_timeout_contract(agent, metadata, mock agent._restart_agent_process() - operations.assert_has_calls( - [mocker.call.wait_for_agent(), mocker.call.execute(mocker.ANY), mocker.call.wait_for_agent()] - ) - command = execute.call_args.args[0] - assert command[:2] == ['sh', '-c'] - script = command[2] - assert 'pidof agent' in script - assert 'finish=/var/run/s6/services/agent/finish' in script - assert '[ -f "$finish" ]' in script - assert 'Unsupported Agent image: s6 Agent finish hook not found' in script - assert 'rm "$finish"' in script - assert script.index('[ -f "$finish" ]') < script.index('kill "$old_pid"') - assert '[ "$elapsed" -ge 17 ]' in script + assert operations.mock_calls == [ + mocker.call.wait_for_agent(), + mocker.call.execute(mocker.ANY), + mocker.call.wait_for_agent(), + ] def test_start_uses_selected_image_rbac_config_and_local_packages( From 46fc2c7edf74f006421f46bfe1689dff525bdcfa Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Fri, 24 Jul 2026 08:51:30 +0000 Subject: [PATCH 35/50] Keep Kubernetes Agent restarts repeatable --- ddev/src/ddev/e2e/agent/kubernetes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index 3b0cd4e0068d8..f9a5aca5b76e6 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -369,7 +369,7 @@ def _restart_agent_process(self) -> None: restart_command = ( 'old_pid=$(pidof agent) || exit 1; ' 'set -- $old_pid; old_pid=$1; ' - 'rm /var/run/s6/services/agent/finish || exit 1; ' + 'rm -f /var/run/s6/services/agent/finish; ' 'kill "$old_pid" || exit 1; ' 'elapsed=0; ' 'while kill -0 "$old_pid" 2>/dev/null; do ' From ba5f7ae79ad87a55d4d8e59054df7ddb3601bc70 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Fri, 24 Jul 2026 09:24:55 +0000 Subject: [PATCH 36/50] Detect Kubernetes Agent container state loss --- ddev/src/ddev/e2e/agent/kubernetes.py | 14 +++++++++++ ddev/tests/e2e/agent/test_kubernetes.py | 33 ++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index f9a5aca5b76e6..988113138f679 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -21,6 +21,7 @@ CLUSTER_RESOURCE_NAME = 'ddev-agent' _DEFAULT_WAIT_TIMEOUT = 120 _LOCAL_PACKAGES_METADATA = 'local_packages' +_PREPARED_MARKER = '/home/.ddev-agent-prepared' class KubernetesAgent(AgentInterface): @@ -338,6 +339,14 @@ def _run_metadata_commands(self, key: str) -> None: for command in commands: self._exec(self.platform.modules.shlex.split(command)) + def _require_prepared(self) -> None: + process = self._exec(['test', '-f', _PREPARED_MARKER], check=False) + if process.returncode: + raise RuntimeError( + 'The Kubernetes Agent container is no longer prepared and may have restarted. ' + 'Recreate the environment with `ddev env stop` followed by `ddev env start`.' + ) + def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env_vars: dict[str, str]) -> None: agent_build = _normalize_agent_image_name( agent_build, self.python_version[0], self.metadata.get('use_jmx', False) @@ -354,6 +363,7 @@ def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env self._sync_auto_conf() self._run_metadata_commands('post_install_commands') self._restart_agent_process() + self._exec(['touch', _PREPARED_MARKER]) def stop(self) -> None: """Leave cleanup to the fixture that deletes the disposable cluster.""" @@ -382,18 +392,22 @@ def _restart_agent_process(self) -> None: def restart(self) -> None: self._sync_local_packages() + self._require_prepared() self._sync_config() self._sync_auto_conf() self._restart_agent_process() + self._require_prepared() def sync_config(self) -> None: self._sync_config() def invoke(self, args: list[str], *, env_vars: dict[str, str] | None = None) -> None: self._sync_local_packages() + self._require_prepared() self._sync_config() self._sync_auto_conf() self._exec(['agent', *args], env_vars=env_vars) + self._require_prepared() def enter_shell(self) -> None: self._kubectl( diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index a657235fe66df..19877cd1cb4a9 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -14,6 +14,7 @@ TEST_NAMESPACE = 'ddev-agent' TEST_POD = 'ddev-agent' TEST_CONTAINER = 'agent' +TEST_PREPARED_MARKER = '/home/.ddev-agent-prepared' @pytest.fixture(scope='module') @@ -98,6 +99,14 @@ def find_kubectl_command(calls: list[list[str]], expected: list[str]) -> int: return matches[0] +def exec_command_indices(calls: list[list[str]], expected: list[str]) -> list[int]: + return [ + index + for index, command in enumerate(calls) + if '--' in command and command[command.index('--') + 1 :] == expected + ] + + def find_exec_command(calls: list[list[str]], expected: list[str], *, prefix: bool = False) -> int: matches = [] for index, command in enumerate(calls): @@ -201,6 +210,19 @@ def test_restart_waits_for_agent_before_and_after_restart(agent, mocker): ] +def test_rejects_container_that_lost_prepared_state(agent, mocker): + execute = mocker.patch.object( + agent, + '_exec', + return_value=subprocess.CompletedProcess([], 1), + ) + + with pytest.raises(RuntimeError, match='may have restarted'): + agent._require_prepared() + + execute.assert_called_once_with(['test', '-f', TEST_PREPARED_MARKER], check=False) + + def test_start_uses_selected_image_rbac_config_and_local_packages( agent, metadata, config_file, auto_conf, temp_dir, run_command ): @@ -295,7 +317,8 @@ def test_start_uses_selected_image_rbac_config_and_local_packages( f'{TEST_NAMESPACE}/{TEST_POD}:/etc/datadog-agent/conf.d/velero.d/auto_conf.yaml', ) restart_index = find_exec_command(calls, ['sh', '-c'], prefix=True) - assert start_index < post_install_index < restart_index + prepared_index = find_exec_command(calls, ['touch', TEST_PREPARED_MARKER]) + assert start_index < post_install_index < restart_index < prepared_index assert metadata['kubernetes']['local_packages'] == { str(local_base): '[kube]', str(integration): '[deps]', @@ -396,7 +419,9 @@ def test_invoke_synchronizes_config_and_environment(agent, config_file, auto_con calls, ['env', 'ALPHA=first', 'ZED=last', 'agent', 'check', 'velero', '--json'], ) - assert config_copy_index < auto_conf_copy_index < invoke_index + prepared_indices = exec_command_indices(calls, ['test', '-f', TEST_PREPARED_MARKER]) + assert len(prepared_indices) == 2 + assert prepared_indices[0] < config_copy_index < auto_conf_copy_index < invoke_index < prepared_indices[1] def test_invoke_removes_pod_config_when_host_config_is_absent(agent, config_file, run_command): @@ -435,7 +460,9 @@ def test_restart_resynchronizes_config_auto_conf_and_editable_sources( f'{TEST_NAMESPACE}/{TEST_POD}:/etc/datadog-agent/conf.d/velero.d/conf.yaml', ) restart_index = find_exec_command(calls, ['sh', '-c'], prefix=True) - assert package_copy_index < config_copy_index < restart_index + prepared_indices = exec_command_indices(calls, ['test', '-f', TEST_PREPARED_MARKER]) + assert len(prepared_indices) == 2 + assert package_copy_index < prepared_indices[0] < config_copy_index < restart_index < prepared_indices[1] assert not any('pip' in command for command in calls) From 06de7e54148a30a37138a2b684e751be08c989b9 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Fri, 24 Jul 2026 12:13:44 +0000 Subject: [PATCH 37/50] Rename Kubernetes Agent module constants --- ddev/src/ddev/e2e/agent/kubernetes.py | 46 +++++++++++++-------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index 988113138f679..d622230dbb919 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -15,13 +15,13 @@ from ddev.utils.fs import Path -_POD_NAME = 'ddev-agent' -_CONTAINER_NAME = 'agent' +POD_NAME = 'ddev-agent' +CONTAINER_NAME = 'agent' NAMESPACE = 'ddev-agent' CLUSTER_RESOURCE_NAME = 'ddev-agent' -_DEFAULT_WAIT_TIMEOUT = 120 -_LOCAL_PACKAGES_METADATA = 'local_packages' -_PREPARED_MARKER = '/home/.ddev-agent-prepared' +DEFAULT_WAIT_TIMEOUT = 120 +LOCAL_PACKAGES_METADATA = 'local_packages' +PREPARED_MARKER = '/home/.ddev-agent-prepared' class KubernetesAgent(AgentInterface): @@ -70,7 +70,7 @@ def _kubectl_prefix(self) -> list[str]: @property def _wait_timeout(self) -> int: - timeout = self._kubernetes_metadata.get('wait_timeout', _DEFAULT_WAIT_TIMEOUT) + timeout = self._kubernetes_metadata.get('wait_timeout', DEFAULT_WAIT_TIMEOUT) if isinstance(timeout, bool) or not isinstance(timeout, int) or timeout <= 0: raise ValueError('Kubernetes Agent `wait_timeout` must be a positive integer') return timeout @@ -101,7 +101,7 @@ def _exec( check: bool = True, capture: bool = False, ) -> subprocess.CompletedProcess: - args = ['exec', '--namespace', self._namespace, f'pod/{_POD_NAME}', '--container', _CONTAINER_NAME, '--'] + args = ['exec', '--namespace', self._namespace, f'pod/{POD_NAME}', '--container', CONTAINER_NAME, '--'] if env_vars: args.append('env') args.extend(f'{key}={value}' for key, value in sorted(env_vars.items())) @@ -158,7 +158,7 @@ def _manifest(self, agent_build: str, env_vars: dict[str, str]) -> dict[str, Any ] ) - labels = {**self._resource_labels, 'app.kubernetes.io/name': _POD_NAME} + labels = {**self._resource_labels, 'app.kubernetes.io/name': POD_NAME} image_pull_policy = self._kubernetes_metadata.get('image_pull_policy', 'Always') if image_pull_policy not in {'Always', 'IfNotPresent', 'Never'}: @@ -190,7 +190,7 @@ def _manifest(self, agent_build: str, env_vars: dict[str, str]) -> dict[str, Any { 'apiVersion': 'v1', 'kind': 'ServiceAccount', - 'metadata': {'name': _POD_NAME, 'namespace': self._namespace, 'labels': self._resource_labels}, + 'metadata': {'name': POD_NAME, 'namespace': self._namespace, 'labels': self._resource_labels}, }, { 'apiVersion': 'rbac.authorization.k8s.io/v1', @@ -208,21 +208,21 @@ def _manifest(self, agent_build: str, env_vars: dict[str, str]) -> dict[str, Any 'name': self._cluster_resource_name, }, 'subjects': [ - {'kind': 'ServiceAccount', 'name': _POD_NAME, 'namespace': self._namespace}, + {'kind': 'ServiceAccount', 'name': POD_NAME, 'namespace': self._namespace}, ], }, { 'apiVersion': 'v1', 'kind': 'Pod', - 'metadata': {'name': _POD_NAME, 'namespace': self._namespace, 'labels': labels}, + 'metadata': {'name': POD_NAME, 'namespace': self._namespace, 'labels': labels}, 'spec': { - 'serviceAccountName': _POD_NAME, + 'serviceAccountName': POD_NAME, 'restartPolicy': 'Always', 'terminationGracePeriodSeconds': 0, 'tolerations': [{'operator': 'Exists'}], 'containers': [ { - 'name': _CONTAINER_NAME, + 'name': CONTAINER_NAME, 'image': agent_build, 'imagePullPolicy': image_pull_policy, 'env': container_env, @@ -245,7 +245,7 @@ def _wait_for_pod(self) -> None: '--namespace', self._namespace, '--for=condition=Ready', - f'pod/{_POD_NAME}', + f'pod/{POD_NAME}', f'--timeout={self._wait_timeout}s', ] ) @@ -274,9 +274,9 @@ def _copy_file(self, source: str, destination: str) -> None: [ 'cp', '--container', - _CONTAINER_NAME, + CONTAINER_NAME, source_path.name, - f'{self._namespace}/{_POD_NAME}:{destination}', + f'{self._namespace}/{POD_NAME}:{destination}', ], check=True, cwd=source_path.parent, @@ -300,14 +300,14 @@ def _sync_auto_conf(self) -> None: self._copy_file(auto_conf, f'{self._config_dir}/auto_conf.yaml') def _remember_local_packages(self, local_packages: dict[Path, str]) -> None: - self._kubernetes_metadata[_LOCAL_PACKAGES_METADATA] = { + self._kubernetes_metadata[LOCAL_PACKAGES_METADATA] = { str(local_package): features for local_package, features in local_packages.items() } def _local_packages(self) -> dict[Path, str]: from ddev.utils.fs import Path - local_packages = cast(dict[str, str], self._kubernetes_metadata.get(_LOCAL_PACKAGES_METADATA, {})) + local_packages = cast(dict[str, str], self._kubernetes_metadata.get(LOCAL_PACKAGES_METADATA, {})) return {Path(path): features for path, features in local_packages.items()} def _sync_local_packages(self, *, install: bool = False) -> None: @@ -340,7 +340,7 @@ def _run_metadata_commands(self, key: str) -> None: self._exec(self.platform.modules.shlex.split(command)) def _require_prepared(self) -> None: - process = self._exec(['test', '-f', _PREPARED_MARKER], check=False) + process = self._exec(['test', '-f', PREPARED_MARKER], check=False) if process.returncode: raise RuntimeError( 'The Kubernetes Agent container is no longer prepared and may have restarted. ' @@ -363,7 +363,7 @@ def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env self._sync_auto_conf() self._run_metadata_commands('post_install_commands') self._restart_agent_process() - self._exec(['touch', _PREPARED_MARKER]) + self._exec(['touch', PREPARED_MARKER]) def stop(self) -> None: """Leave cleanup to the fixture that deletes the disposable cluster.""" @@ -416,9 +416,9 @@ def enter_shell(self) -> None: '-it', '--namespace', self._namespace, - f'pod/{_POD_NAME}', + f'pod/{POD_NAME}', '--container', - _CONTAINER_NAME, + CONTAINER_NAME, '--', 'bash', ], @@ -427,7 +427,7 @@ def enter_shell(self) -> None: def _show_logs(self, *, check: bool = False) -> None: self._kubectl( - ['logs', '--namespace', self._namespace, f'pod/{_POD_NAME}', '--container', _CONTAINER_NAME], + ['logs', '--namespace', self._namespace, f'pod/{POD_NAME}', '--container', CONTAINER_NAME], check=check, ) From 09e2d72d2a85d19b92293f8a458c074ad24969f7 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Fri, 24 Jul 2026 14:19:57 +0200 Subject: [PATCH 38/50] Update Kubernetes node support description Clarified the implementation details regarding Kubernetes node support. --- docs/developer/ddev/test.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/developer/ddev/test.md b/docs/developer/ddev/test.md index 308d363a75b89..d5119efa949bd 100644 --- a/docs/developer/ddev/test.md +++ b/docs/developer/ddev/test.md @@ -125,8 +125,7 @@ that import a local image into the cluster can set `image_pull_policy` to `IfNot one Agent in the fixed `ddev-agent` namespace with fixed cluster-scoped RBAC names. Resource cleanup happens when `kind_run` deletes the whole disposable cluster. -The initial implementation supports exactly one schedulable Kubernetes node. It rejects multi-node clusters until -Agent targeting or fan-out semantics are defined. +The current implementation supports exactly one schedulable Kubernetes node. ### Terraform From 6f17438266e8faa6a1d7119710d3ea1689a866ec Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Fri, 24 Jul 2026 12:27:02 +0000 Subject: [PATCH 39/50] Decouple Kubernetes Agent tests from commands --- ddev/tests/e2e/agent/test_kubernetes.py | 83 ++++++------------------- 1 file changed, 18 insertions(+), 65 deletions(-) diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index 19877cd1cb4a9..d8c80ea3123c8 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -147,53 +147,6 @@ def find_copy_command(calls: list[list[str]], source: str, destination: str) -> return matches[0] -def test_exec_builds_scoped_command_and_sorts_environment(agent, run_command): - agent._exec(['agent', 'check', 'velero'], env_vars={'ZED': 'last', 'ALPHA': 'first'}) - - run_command.assert_called_once_with( - [ - 'kubectl', - '--kubeconfig', - TEST_KUBECONFIG, - 'exec', - '--namespace', - TEST_NAMESPACE, - f'pod/{TEST_POD}', - '--container', - TEST_CONTAINER, - '--', - 'env', - 'ALPHA=first', - 'ZED=last', - 'agent', - 'check', - 'velero', - ], - check=True, - ) - - -def test_copy_file_builds_scoped_command(agent, config_file, run_command): - destination = '/tmp/conf.yaml' - - agent._copy_file(str(config_file), destination) - - run_command.assert_called_once_with( - [ - 'kubectl', - '--kubeconfig', - TEST_KUBECONFIG, - 'cp', - '--container', - TEST_CONTAINER, - config_file.name, - f'{TEST_NAMESPACE}/{TEST_POD}:{destination}', - ], - check=True, - cwd=config_file.resolve().parent, - ) - - def test_restart_waits_for_agent_before_and_after_restart(agent, mocker): operations = mocker.Mock() execute = mocker.patch.object(agent, '_exec') @@ -324,6 +277,7 @@ def test_start_uses_selected_image_rbac_config_and_local_packages( str(integration): '[deps]', } local_base_copy = run_command.call_args_list[local_base_copy_index] + assert local_base_copy.kwargs['check'] is True assert local_base_copy.kwargs['cwd'] == local_base.resolve().parent @@ -420,6 +374,7 @@ def test_invoke_synchronizes_config_and_environment(agent, config_file, auto_con ['env', 'ALPHA=first', 'ZED=last', 'agent', 'check', 'velero', '--json'], ) prepared_indices = exec_command_indices(calls, ['test', '-f', TEST_PREPARED_MARKER]) + assert run_command.call_args_list[invoke_index].kwargs['check'] is True assert len(prepared_indices) == 2 assert prepared_indices[0] < config_copy_index < auto_conf_copy_index < invoke_index < prepared_indices[1] @@ -475,28 +430,26 @@ def test_restart_rejects_unsafe_local_package_destination(agent, metadata, run_c run_command.assert_not_called() -def test_shell_and_logs_use_backend_commands(agent, run_command): +def test_enter_shell_starts_interactive_bash(agent, run_command): agent.enter_shell() + + calls = command_calls(run_command) + shell_index = find_exec_command(calls, ['bash']) + kubectl_args = calls[shell_index][: calls[shell_index].index('--')] + assert '-it' in kubectl_args + assert run_command.call_args_list[shell_index].kwargs['check'] is True + + +def test_show_logs_targets_agent_container(agent, run_command): agent.show_logs() calls = command_calls(run_command) - prefix = ['kubectl', '--kubeconfig', TEST_KUBECONFIG] - assert calls == [ - [ - *prefix, - 'exec', - '-it', - '--namespace', - TEST_NAMESPACE, - f'pod/{TEST_POD}', - '--container', - TEST_CONTAINER, - '--', - 'bash', - ], - [*prefix, 'logs', '--namespace', TEST_NAMESPACE, f'pod/{TEST_POD}', '--container', TEST_CONTAINER], - ] - assert run_command.call_args_list[-1].kwargs['check'] is True + logs_index = find_kubectl_command(calls, ['logs']) + command = calls[logs_index] + assert option_value(command, '--namespace') == TEST_NAMESPACE + assert f'pod/{TEST_POD}' in command + assert option_value(command, '--container') == TEST_CONTAINER + assert run_command.call_args_list[logs_index].kwargs['check'] is True def test_kubeconfig_validation(app, get_integration, config_file): From 2bb169dc607dfbcf2e75e3a965b30256387d8a10 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Fri, 24 Jul 2026 12:38:25 +0000 Subject: [PATCH 40/50] Expose shared Agent image normalization --- ddev/src/ddev/e2e/agent/docker.py | 4 ++-- ddev/src/ddev/e2e/agent/kubernetes.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/docker.py b/ddev/src/ddev/e2e/agent/docker.py index 6daa11d26558b..4c882cdd4aa08 100644 --- a/ddev/src/ddev/e2e/agent/docker.py +++ b/ddev/src/ddev/e2e/agent/docker.py @@ -45,7 +45,7 @@ def disable_integration_before_install(config_file): new.rename(config_file.parent / old) -def _normalize_agent_image_name(agent_build: str | None, python_major: int, use_jmx: bool) -> str: +def normalize_agent_image_name(agent_build: str | None, python_major: int, use_jmx: bool) -> str: if not agent_build: return 'registry.datadoghq.com/agent-dev:master-py3' @@ -187,7 +187,7 @@ def get_id(self) -> str: def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env_vars: dict[str, str]) -> None: from ddev.e2e.agent.constants import AgentEnvVars - agent_build = _normalize_agent_image_name( + agent_build = normalize_agent_image_name( agent_build, self.python_version[0], self.metadata.get('use_jmx', False) ) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index d622230dbb919..6bea387f11909 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -7,7 +7,7 @@ import time from typing import TYPE_CHECKING, Any, cast -from ddev.e2e.agent.docker import _normalize_agent_image_name +from ddev.e2e.agent.docker import normalize_agent_image_name from ddev.e2e.agent.interface import AgentInterface if TYPE_CHECKING: @@ -348,7 +348,7 @@ def _require_prepared(self) -> None: ) def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env_vars: dict[str, str]) -> None: - agent_build = _normalize_agent_image_name( + agent_build = normalize_agent_image_name( agent_build, self.python_version[0], self.metadata.get('use_jmx', False) ) _ = self._wait_timeout From ed46882c0e2ebed97af64586afd165ad2cef40a8 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 30 Jul 2026 06:53:25 +0000 Subject: [PATCH 41/50] Require a Kubernetes metadata mapping Reading self.metadata['kubernetes'] unguarded turned a missing metadata block into a bare KeyError, so an environment that selects agent_type 'kubernetes' without the accompanying mapping failed with 'Unable to start the Agent: kubernetes' and gave a raw traceback from ddev env shell, which only handles CalledProcessError. Validate the mapping where it is read so the failure names the missing contract instead. --- ddev/src/ddev/e2e/agent/kubernetes.py | 7 ++++++- ddev/tests/e2e/agent/test_kubernetes.py | 9 +++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index 6bea387f11909..10e0410bd54e5 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -35,7 +35,12 @@ class KubernetesAgent(AgentInterface): @property def _kubernetes_metadata(self) -> dict[str, Any]: - return cast(dict[str, Any], self.metadata['kubernetes']) + metadata = self.metadata.get('kubernetes') + if not isinstance(metadata, dict): + raise ValueError( + 'Agent type `kubernetes` requires a `kubernetes` metadata mapping with a `kubeconfig` path' + ) + return cast(dict[str, Any], metadata) @property def _kubeconfig(self) -> str: diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index d8c80ea3123c8..185735ee5e34a 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -456,3 +456,12 @@ def test_kubeconfig_validation(app, get_integration, config_file): agent = KubernetesAgent(app, get_integration('velero'), 'py3.12', {'kubernetes': {}}, config_file) with pytest.raises(ValueError, match='non-empty `kubeconfig`'): _ = agent._kubeconfig + + +@pytest.mark.parametrize('kubernetes_metadata', [None, 'kubeconfig'], ids=['missing', 'not_a_mapping']) +def test_rejects_metadata_without_kubernetes_mapping(app, get_integration, config_file, kubernetes_metadata): + metadata = {} if kubernetes_metadata is None else {'kubernetes': kubernetes_metadata} + agent = KubernetesAgent(app, get_integration('velero'), 'py3.12', metadata, config_file) + + with pytest.raises(ValueError, match='requires a `kubernetes` metadata mapping'): + _ = agent._kubeconfig From 4eb882d1bdab1cfc6bf62901dc4ca499700bae57 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 30 Jul 2026 06:54:57 +0000 Subject: [PATCH 42/50] Stamp the prepared marker before restarting the Agent start() stamped PREPARED_MARKER after _restart_agent_process(), so a container replaced during the restart was marked prepared. The marker lives in the container filesystem and the pod declares no volumes, so a replacement loses the copied conf.yaml, auto_conf.yaml and the editable installs, while _wait_for_agent() still succeeds against the fresh container. Every later _require_prepared() check then passed and tests silently exercised the Agent-shipped integration with no configuration. Stamp the marker once preparation is complete and assert it after the restart, matching the invariant restart() already relies on. Checking after the stamp instead would only observe the gap between the two commands. --- ddev/src/ddev/e2e/agent/kubernetes.py | 6 +++++- ddev/tests/e2e/agent/test_kubernetes.py | 24 ++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index 10e0410bd54e5..a1ad767167ff2 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -367,8 +367,12 @@ def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env self._sync_config() self._sync_auto_conf() self._run_metadata_commands('post_install_commands') - self._restart_agent_process() + # Stamp the marker before the restart so the check below can still observe a + # container replacement: the marker lives in the container filesystem, so a + # fresh container loses it along with the copied configuration and packages. self._exec(['touch', PREPARED_MARKER]) + self._restart_agent_process() + self._require_prepared() def stop(self) -> None: """Leave cleanup to the fixture that deletes the disposable cluster.""" diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index 185735ee5e34a..1eb5e6a147b7a 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -270,8 +270,10 @@ def test_start_uses_selected_image_rbac_config_and_local_packages( f'{TEST_NAMESPACE}/{TEST_POD}:/etc/datadog-agent/conf.d/velero.d/auto_conf.yaml', ) restart_index = find_exec_command(calls, ['sh', '-c'], prefix=True) - prepared_index = find_exec_command(calls, ['touch', TEST_PREPARED_MARKER]) - assert start_index < post_install_index < restart_index < prepared_index + marker_index = find_exec_command(calls, ['touch', TEST_PREPARED_MARKER]) + prepared_indices = exec_command_indices(calls, ['test', '-f', TEST_PREPARED_MARKER]) + assert len(prepared_indices) == 1 + assert start_index < post_install_index < marker_index < restart_index < prepared_indices[0] assert metadata['kubernetes']['local_packages'] == { str(local_base): '[kube]', str(integration): '[deps]', @@ -291,6 +293,24 @@ def test_rejects_invalid_wait_timeout_before_creating_resources(agent, metadata, run_command.assert_not_called() +def test_start_detects_container_replacement_during_restart(agent, app, mocker): + def run(command, **kwargs): + if command[-2:] == ['config', 'current-context']: + return successful_process(command, stdout=b'kind-test\n') + if command[-4:] == ['get', 'nodes', '-o', 'json']: + nodes = {'items': [{'metadata': {'name': 'kind-control-plane'}, 'spec': {}}]} + return successful_process(command, stdout=json.dumps(nodes).encode()) + # A replaced container no longer carries the marker stamped before the restart. + if command[-3:] == ['test', '-f', TEST_PREPARED_MARKER]: + return subprocess.CompletedProcess(command, 1) + return successful_process(command) + + mocker.patch.object(app.platform, 'run_command', side_effect=run) + + with pytest.raises(RuntimeError, match='may have restarted'): + agent.start(agent_build='', local_packages={}, env_vars={}) + + def test_rejects_non_kind_context_before_inspecting_cluster(agent, app, mocker): run_command = mocker.patch.object( app.platform, From 9cd9a3d8a99f9e4641ac6c5a3f0a384c30d0e5b9 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 30 Jul 2026 10:35:04 +0000 Subject: [PATCH 43/50] Share Agent image normalization through the interface --- ddev/src/ddev/e2e/agent/docker.py | 51 +-------------------------- ddev/src/ddev/e2e/agent/interface.py | 47 ++++++++++++++++++++++-- ddev/src/ddev/e2e/agent/kubernetes.py | 5 +-- 3 files changed, 47 insertions(+), 56 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/docker.py b/ddev/src/ddev/e2e/agent/docker.py index 4c882cdd4aa08..1014278387149 100644 --- a/ddev/src/ddev/e2e/agent/docker.py +++ b/ddev/src/ddev/e2e/agent/docker.py @@ -4,7 +4,6 @@ from __future__ import annotations import os -import re import sys from contextlib import AbstractContextManager, contextmanager, nullcontext from functools import cache, cached_property, partial @@ -20,16 +19,6 @@ from ddev.utils.fs import Path -AGENT_IMAGE_REGEX = r'^([^/]+)/([^:]+):(.*)$' -AGENT_VERSION_REGEX = ( - # Main version: 7, 7.69, 7.69.0 ... - r"^(?P\d+(?:\.[\dx]+)*|latest|main|master|nightly)" - # rcs: rc.1, rc ... - r"(?P-rc(?:\.\d+)?)?" - # Anny suffixes: -jmx, -linux, -full... - r"(?P(?:-[a-zA-Z0-9]+)*)$" -) - @contextmanager def disable_integration_before_install(config_file): @@ -45,42 +34,6 @@ def disable_integration_before_install(config_file): new.rename(config_file.parent / old) -def normalize_agent_image_name(agent_build: str | None, python_major: int, use_jmx: bool) -> str: - if not agent_build: - return 'registry.datadoghq.com/agent-dev:master-py3' - - if match := re.match(AGENT_IMAGE_REGEX, agent_build): - org, image, tag = match.groups() - - if org != 'datadog' and org != 'registry.datadoghq.com': - # Some non datadog image has been selected - return agent_build - - version_match = re.match(AGENT_VERSION_REGEX, tag) - - if version_match is None: - # Not sure how to extract information for a version of this shape - return agent_build - - version = version_match.group('version') - rc = version_match.group('rc') - rc = rc if rc else '' - suffixes = version_match.group('suffixes') - - # Add -py suffix if missing, only in agent-dev the agent does not have py3 suffix after agent 6 - if image == 'agent-dev': - if not (rc != '' or any(suffix in suffixes for suffix in ['py', 'fips']) or version[0].isdigit()): - suffixes = f'-py{python_major}{suffixes}' - - # Add jmx suffix if missing - if use_jmx and '-jmx' not in suffixes: - suffixes += '-jmx' - - return f'{org}/{image}:{version}{rc}{suffixes}' - - return agent_build - - class DockerAgent(AgentInterface): build_config_key = 'docker' @@ -187,9 +140,7 @@ def get_id(self) -> str: def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env_vars: dict[str, str]) -> None: from ddev.e2e.agent.constants import AgentEnvVars - agent_build = normalize_agent_image_name( - agent_build, self.python_version[0], self.metadata.get('use_jmx', False) - ) + agent_build = self.normalize_agent_image_name(agent_build) env_vars = env_vars.copy() diff --git a/ddev/src/ddev/e2e/agent/interface.py b/ddev/src/ddev/e2e/agent/interface.py index 13b0a8089bd67..4ba24988e662f 100644 --- a/ddev/src/ddev/e2e/agent/interface.py +++ b/ddev/src/ddev/e2e/agent/interface.py @@ -3,6 +3,7 @@ # Licensed under a 3-clause BSD style license (see LICENSE) from __future__ import annotations +import re from abc import ABC, abstractmethod from functools import cached_property from typing import TYPE_CHECKING, Any @@ -16,6 +17,17 @@ from ddev.utils.platform import Platform +AGENT_IMAGE_REGEX = r'^([^/]+)/([^:]+):(.*)$' +AGENT_VERSION_REGEX = ( + # Main version: 7, 7.69, 7.69.0 ... + r"^(?P\d+(?:\.[\dx]+)*|latest|main|master|nightly)" + # rcs: rc.1, rc ... + r"(?P-rc(?:\.\d+)?)?" + # Any suffixes: -jmx, -linux, -full... + r"(?P(?:-[a-zA-Z0-9]+)*)$" +) + + class AgentInterface(ABC): build_config_key: str | None = None supports_ci = True @@ -55,8 +67,6 @@ def config_file(self) -> Path: @cached_property def python_version(self) -> tuple[int, int]: - import re - if match := re.search(r'^py(\d)\.(\d+)', self.env): return int(match.group(1)), int(match.group(2)) @@ -65,6 +75,39 @@ def python_version(self) -> tuple[int, int]: major, minor = PYTHON_VERSION.split('.') return int(major), int(minor) + def normalize_agent_image_name(self, agent_build: str | None) -> str: + if not agent_build: + return 'registry.datadoghq.com/agent-dev:master-py3' + + if match := re.match(AGENT_IMAGE_REGEX, agent_build): + org, image, tag = match.groups() + + if org not in {'datadog', 'registry.datadoghq.com'}: + # Some non-Datadog image has been selected. + return agent_build + + version_match = re.match(AGENT_VERSION_REGEX, tag) + if version_match is None: + # The tag does not follow a recognized Agent version format. + return agent_build + + version = version_match.group('version') + rc = version_match.group('rc') or '' + suffixes = version_match.group('suffixes') + + # Add a Python suffix when required by a development Agent image. + if image == 'agent-dev': + has_python_variant = any(suffix in suffixes for suffix in ('py', 'fips')) + if not (rc or has_python_variant or version[0].isdigit()): + suffixes = f'-py{self.python_version[0]}{suffixes}' + + if self.metadata.get('use_jmx', False) and '-jmx' not in suffixes: + suffixes += '-jmx' + + return f'{org}/{image}:{version}{rc}{suffixes}' + + return agent_build + def get_id(self) -> str: return f'{self.integration.name}_{self.env}' diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index a1ad767167ff2..1ab6ea1a7d2fe 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -7,7 +7,6 @@ import time from typing import TYPE_CHECKING, Any, cast -from ddev.e2e.agent.docker import normalize_agent_image_name from ddev.e2e.agent.interface import AgentInterface if TYPE_CHECKING: @@ -353,9 +352,7 @@ def _require_prepared(self) -> None: ) def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env_vars: dict[str, str]) -> None: - agent_build = normalize_agent_image_name( - agent_build, self.python_version[0], self.metadata.get('use_jmx', False) - ) + agent_build = self.normalize_agent_image_name(agent_build) _ = self._wait_timeout self._validate_context() self._validate_topology() From 63e21212aa5cc59c27cccdba4a2e28e7deaa7f7a Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 30 Jul 2026 12:15:59 +0000 Subject: [PATCH 44/50] Clarify the prepared marker invariant --- ddev/src/ddev/e2e/agent/kubernetes.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index 1ab6ea1a7d2fe..e62017c32d7b6 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -364,9 +364,8 @@ def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env self._sync_config() self._sync_auto_conf() self._run_metadata_commands('post_install_commands') - # Stamp the marker before the restart so the check below can still observe a - # container replacement: the marker lives in the container filesystem, so a - # fresh container loses it along with the copied configuration and packages. + # A replacement during restart must lose this marker so the + # post-restart check rejects the unprepared container. self._exec(['touch', PREPARED_MARKER]) self._restart_agent_process() self._require_prepared() From 6fc9574106969fde7bb5da52fb41b9d90d9a08b7 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Thu, 30 Jul 2026 12:28:34 +0000 Subject: [PATCH 45/50] Move Agent image normalization to a shared module --- ddev/src/ddev/e2e/agent/docker.py | 5 ++- ddev/src/ddev/e2e/agent/image.py | 50 +++++++++++++++++++++++++++ ddev/src/ddev/e2e/agent/interface.py | 47 ++----------------------- ddev/src/ddev/e2e/agent/kubernetes.py | 5 ++- 4 files changed, 60 insertions(+), 47 deletions(-) create mode 100644 ddev/src/ddev/e2e/agent/image.py diff --git a/ddev/src/ddev/e2e/agent/docker.py b/ddev/src/ddev/e2e/agent/docker.py index 1014278387149..a3f4978a6b496 100644 --- a/ddev/src/ddev/e2e/agent/docker.py +++ b/ddev/src/ddev/e2e/agent/docker.py @@ -11,6 +11,7 @@ import stamina +from ddev.e2e.agent.image import normalize_agent_image_name from ddev.e2e.agent.interface import AgentInterface from ddev.utils.structures import EnvVars @@ -140,7 +141,9 @@ def get_id(self) -> str: def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env_vars: dict[str, str]) -> None: from ddev.e2e.agent.constants import AgentEnvVars - agent_build = self.normalize_agent_image_name(agent_build) + agent_build = normalize_agent_image_name( + agent_build, self.python_version[0], self.metadata.get('use_jmx', False) + ) env_vars = env_vars.copy() diff --git a/ddev/src/ddev/e2e/agent/image.py b/ddev/src/ddev/e2e/agent/image.py new file mode 100644 index 0000000000000..6648c09e61904 --- /dev/null +++ b/ddev/src/ddev/e2e/agent/image.py @@ -0,0 +1,50 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from __future__ import annotations + +import re + +AGENT_IMAGE_REGEX = r'^([^/]+)/([^:]+):(.*)$' +AGENT_VERSION_REGEX = ( + # Main version: 7, 7.69, 7.69.0 ... + r"^(?P\d+(?:\.[\dx]+)*|latest|main|master|nightly)" + # rcs: rc.1, rc ... + r"(?P-rc(?:\.\d+)?)?" + # Any suffixes: -jmx, -linux, -full... + r"(?P(?:-[a-zA-Z0-9]+)*)$" +) + + +def normalize_agent_image_name(agent_build: str | None, python_major: int, use_jmx: bool) -> str: + if not agent_build: + return 'registry.datadoghq.com/agent-dev:master-py3' + + if match := re.match(AGENT_IMAGE_REGEX, agent_build): + org, image, tag = match.groups() + + if org not in {'datadog', 'registry.datadoghq.com'}: + # Some non-Datadog image has been selected. + return agent_build + + version_match = re.match(AGENT_VERSION_REGEX, tag) + if version_match is None: + # The tag does not follow a recognized Agent version format. + return agent_build + + version = version_match.group('version') + rc = version_match.group('rc') or '' + suffixes = version_match.group('suffixes') + + # Add a Python suffix when required by a development Agent image. + if image == 'agent-dev': + has_python_variant = any(suffix in suffixes for suffix in ('py', 'fips')) + if not (rc or has_python_variant or version[0].isdigit()): + suffixes = f'-py{python_major}{suffixes}' + + if use_jmx and '-jmx' not in suffixes: + suffixes += '-jmx' + + return f'{org}/{image}:{version}{rc}{suffixes}' + + return agent_build diff --git a/ddev/src/ddev/e2e/agent/interface.py b/ddev/src/ddev/e2e/agent/interface.py index 4ba24988e662f..13b0a8089bd67 100644 --- a/ddev/src/ddev/e2e/agent/interface.py +++ b/ddev/src/ddev/e2e/agent/interface.py @@ -3,7 +3,6 @@ # Licensed under a 3-clause BSD style license (see LICENSE) from __future__ import annotations -import re from abc import ABC, abstractmethod from functools import cached_property from typing import TYPE_CHECKING, Any @@ -17,17 +16,6 @@ from ddev.utils.platform import Platform -AGENT_IMAGE_REGEX = r'^([^/]+)/([^:]+):(.*)$' -AGENT_VERSION_REGEX = ( - # Main version: 7, 7.69, 7.69.0 ... - r"^(?P\d+(?:\.[\dx]+)*|latest|main|master|nightly)" - # rcs: rc.1, rc ... - r"(?P-rc(?:\.\d+)?)?" - # Any suffixes: -jmx, -linux, -full... - r"(?P(?:-[a-zA-Z0-9]+)*)$" -) - - class AgentInterface(ABC): build_config_key: str | None = None supports_ci = True @@ -67,6 +55,8 @@ def config_file(self) -> Path: @cached_property def python_version(self) -> tuple[int, int]: + import re + if match := re.search(r'^py(\d)\.(\d+)', self.env): return int(match.group(1)), int(match.group(2)) @@ -75,39 +65,6 @@ def python_version(self) -> tuple[int, int]: major, minor = PYTHON_VERSION.split('.') return int(major), int(minor) - def normalize_agent_image_name(self, agent_build: str | None) -> str: - if not agent_build: - return 'registry.datadoghq.com/agent-dev:master-py3' - - if match := re.match(AGENT_IMAGE_REGEX, agent_build): - org, image, tag = match.groups() - - if org not in {'datadog', 'registry.datadoghq.com'}: - # Some non-Datadog image has been selected. - return agent_build - - version_match = re.match(AGENT_VERSION_REGEX, tag) - if version_match is None: - # The tag does not follow a recognized Agent version format. - return agent_build - - version = version_match.group('version') - rc = version_match.group('rc') or '' - suffixes = version_match.group('suffixes') - - # Add a Python suffix when required by a development Agent image. - if image == 'agent-dev': - has_python_variant = any(suffix in suffixes for suffix in ('py', 'fips')) - if not (rc or has_python_variant or version[0].isdigit()): - suffixes = f'-py{self.python_version[0]}{suffixes}' - - if self.metadata.get('use_jmx', False) and '-jmx' not in suffixes: - suffixes += '-jmx' - - return f'{org}/{image}:{version}{rc}{suffixes}' - - return agent_build - def get_id(self) -> str: return f'{self.integration.name}_{self.env}' diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index e62017c32d7b6..b5adfc343eb1d 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -7,6 +7,7 @@ import time from typing import TYPE_CHECKING, Any, cast +from ddev.e2e.agent.image import normalize_agent_image_name from ddev.e2e.agent.interface import AgentInterface if TYPE_CHECKING: @@ -352,7 +353,9 @@ def _require_prepared(self) -> None: ) def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env_vars: dict[str, str]) -> None: - agent_build = self.normalize_agent_image_name(agent_build) + agent_build = normalize_agent_image_name( + agent_build, self.python_version[0], self.metadata.get('use_jmx', False) + ) _ = self._wait_timeout self._validate_context() self._validate_topology() From 1aa672948fdcf662591547effae0a863d8ac330b Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Mon, 3 Aug 2026 08:45:44 +0000 Subject: [PATCH 46/50] Fail fast on unsupported Kubernetes Agent inputs --- ddev/src/ddev/e2e/agent/kubernetes.py | 9 +++++++++ ddev/tests/e2e/agent/test_kubernetes.py | 15 +++++++++++++++ docs/developer/ddev/plugins.md | 2 +- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index b5adfc343eb1d..efd1d01cae247 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -7,6 +7,7 @@ import time from typing import TYPE_CHECKING, Any, cast +from ddev.e2e.agent.constants import AgentEnvVars from ddev.e2e.agent.image import normalize_agent_image_name from ddev.e2e.agent.interface import AgentInterface @@ -344,6 +345,13 @@ def _run_metadata_commands(self, key: str) -> None: for command in commands: self._exec(self.platform.modules.shlex.split(command)) + @staticmethod + def _validate_options(env_vars: dict[str, str]) -> None: + if AgentEnvVars.DOGSTATSD_PORT in env_vars: + raise NotImplementedError('Kubernetes Agent does not support DogStatsD') + if env_vars.get(AgentEnvVars.LOGS_ENABLED, '').lower() == 'true': + raise NotImplementedError('Kubernetes Agent does not support logs') + def _require_prepared(self) -> None: process = self._exec(['test', '-f', PREPARED_MARKER], check=False) if process.returncode: @@ -353,6 +361,7 @@ def _require_prepared(self) -> None: ) def start(self, *, agent_build: str | None, local_packages: dict[Path, str], env_vars: dict[str, str]) -> None: + self._validate_options(env_vars) agent_build = normalize_agent_image_name( agent_build, self.python_version[0], self.metadata.get('use_jmx', False) ) diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index 1eb5e6a147b7a..9a2ffebc9c0a2 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -6,6 +6,7 @@ import pytest +from ddev.e2e.agent.constants import AgentEnvVars from ddev.e2e.agent.kubernetes import KubernetesAgent from ddev.integration.core import Integration from ddev.repo.config import RepositoryConfig @@ -293,6 +294,20 @@ def test_rejects_invalid_wait_timeout_before_creating_resources(agent, metadata, run_command.assert_not_called() +@pytest.mark.parametrize( + ('env_vars', 'expected'), + [ + ({AgentEnvVars.DOGSTATSD_PORT: '8125'}, 'DogStatsD'), + ({AgentEnvVars.LOGS_ENABLED: 'true'}, 'logs'), + ], +) +def test_rejects_unsupported_options_before_creating_resources(agent, run_command, env_vars, expected): + with pytest.raises(NotImplementedError, match=expected): + agent.start(agent_build='', local_packages={}, env_vars=env_vars) + + run_command.assert_not_called() + + def test_start_detects_container_replacement_during_restart(agent, app, mocker): def run(command, **kwargs): if command[-2:] == ['config', 'current-context']: diff --git a/docs/developer/ddev/plugins.md b/docs/developer/ddev/plugins.md index 2e772797c47a4..f3222d134d0f1 100644 --- a/docs/developer/ddev/plugins.md +++ b/docs/developer/ddev/plugins.md @@ -142,4 +142,4 @@ is desired. This fixture is responsible for starting and stopping environments a - `docker_volumes` - A `list` of `str` representing [Docker volume mounts][docker-volume-docs] if `agent_type` is `docker` e.g. `/local/path:/agent/container/path:ro`. - `docker_platform` - The container architecture to use if `agent_type` is `docker`. Currently, we support `linux` (default) and `windows`. - `kubernetes` - Configuration used when `agent_type` is `kubernetes`. It requires `kubeconfig` and optionally accepts `auto_conf`, `image_pull_policy`, and `wait_timeout`. The backend is supported only with a disposable, fixture-owned cluster provisioned by `kind_run`; it uses the fixed `ddev-agent` namespace, and fixture teardown deletes the whole cluster. -- `logs_config` - A `list` of configs that will be used by the Logs Agent. You will never need to use this directly, but rather via [higher level abstractions](test.md#logs). +- `logs_config` - A `list` of configs that will be used by the Logs Agent when `agent_type` is `docker`. You will never need to use this directly, but rather via [higher level abstractions](test.md#logs). From cd499d3301fc76ffaa5cd6e3c52564efa1fd81af Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Mon, 3 Aug 2026 09:46:26 +0000 Subject: [PATCH 47/50] Clarify shared-log backend limitation --- docs/developer/ddev/plugins.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer/ddev/plugins.md b/docs/developer/ddev/plugins.md index f3222d134d0f1..3673721508999 100644 --- a/docs/developer/ddev/plugins.md +++ b/docs/developer/ddev/plugins.md @@ -142,4 +142,4 @@ is desired. This fixture is responsible for starting and stopping environments a - `docker_volumes` - A `list` of `str` representing [Docker volume mounts][docker-volume-docs] if `agent_type` is `docker` e.g. `/local/path:/agent/container/path:ro`. - `docker_platform` - The container architecture to use if `agent_type` is `docker`. Currently, we support `linux` (default) and `windows`. - `kubernetes` - Configuration used when `agent_type` is `kubernetes`. It requires `kubeconfig` and optionally accepts `auto_conf`, `image_pull_policy`, and `wait_timeout`. The backend is supported only with a disposable, fixture-owned cluster provisioned by `kind_run`; it uses the fixed `ddev-agent` namespace, and fixture teardown deletes the whole cluster. -- `logs_config` - A `list` of configs that will be used by the Logs Agent when `agent_type` is `docker`. You will never need to use this directly, but rather via [higher level abstractions](test.md#logs). +- `logs_config` - A `list` of configs used by the Logs Agent. The [higher-level shared-file log helper](test.md#logs) currently supports only `agent_type: docker`. From 00d1ea0cf8c6e0e7c45c3115deca3249822c957b Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Mon, 3 Aug 2026 08:46:11 +0000 Subject: [PATCH 48/50] Preserve Agent check failures during config cleanup --- ddev/src/ddev/cli/env/agent.py | 16 ++++++++++++++-- ddev/tests/cli/env/test_agent.py | 25 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/ddev/src/ddev/cli/env/agent.py b/ddev/src/ddev/cli/env/agent.py index c6ef4e08dbf1c..864c2e758ec56 100644 --- a/ddev/src/ddev/cli/env/agent.py +++ b/ddev/src/ddev/cli/env/agent.py @@ -49,6 +49,18 @@ def _validate_env_vars(ctx: click.Context, param: click.Parameter, value: tuple[ return env_vars or None +def _sync_restored_config(app: Application, agent: AgentInterface) -> None: + import sys + + original_error = sys.exception() + try: + agent.sync_config() + except Exception as e: + if original_error is None: + raise + app.display_warning(f'Unable to restore the Agent configuration: {e}') + + @click.command( short_help='Invoke the Agent', context_settings={'help_option_names': [], 'ignore_unknown_options': True} ) @@ -129,7 +141,7 @@ def agent( app.abort(str(e)) finally: env_data.config_file.unlink() - agent.sync_config() + _sync_restored_config(app, agent) else: temp_config_file = env_data.config_file.parent / f'{env_data.config_file.name}.bak.example' env_data.config_file.replace(temp_config_file) @@ -140,4 +152,4 @@ def agent( app.abort(str(e)) finally: temp_config_file.replace(env_data.config_file) - agent.sync_config() + _sync_restored_config(app, agent) diff --git a/ddev/tests/cli/env/test_agent.py b/ddev/tests/cli/env/test_agent.py index f00baab58477a..4c2ad7d06d5c5 100644 --- a/ddev/tests/cli/env/test_agent.py +++ b/ddev/tests/cli/env/test_agent.py @@ -1,8 +1,11 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import subprocess + import pytest +from ddev.cli.env.agent import _sync_restored_config from ddev.e2e.config import EnvDataStorage @@ -202,6 +205,28 @@ def test_temporary_config_is_restored_and_synchronized(ddev, data_dir, temp_dir, sync_config.assert_called_once_with() +def test_sync_failure_does_not_mask_check_failure(mocker): + check_error = subprocess.CalledProcessError(7, ['agent', 'check']) + sync_error = subprocess.CalledProcessError(8, ['kubectl', 'cp']) + app = mocker.Mock() + agent = mocker.Mock() + agent.sync_config.side_effect = sync_error + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + try: + raise check_error + finally: + _sync_restored_config(app, agent) + + assert exc_info.value is check_error + app.display_warning.assert_called_once() + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + _sync_restored_config(app, agent) + + assert exc_info.value is sync_error + + def test_temporary_config_absence_is_restored_and_synchronized(ddev, data_dir, temp_dir, mocker): invoke = mocker.patch('ddev.e2e.agent.docker.DockerAgent.invoke') sync_config = mocker.patch('ddev.e2e.agent.docker.DockerAgent.sync_config') From 76ed941082c3a2235c70809ff0bc8797bdbfc4c3 Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Mon, 3 Aug 2026 09:48:07 +0000 Subject: [PATCH 49/50] Clarify current Kubernetes Agent limitations --- ddev/src/ddev/e2e/agent/kubernetes.py | 6 ++++-- ddev/tests/e2e/agent/test_kubernetes.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index efd1d01cae247..dc5f4925d5a0a 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -348,9 +348,11 @@ def _run_metadata_commands(self, key: str) -> None: @staticmethod def _validate_options(env_vars: dict[str, str]) -> None: if AgentEnvVars.DOGSTATSD_PORT in env_vars: - raise NotImplementedError('Kubernetes Agent does not support DogStatsD') + raise NotImplementedError('The Kubernetes Agent backend does not currently support DogStatsD exposure') if env_vars.get(AgentEnvVars.LOGS_ENABLED, '').lower() == 'true': - raise NotImplementedError('Kubernetes Agent does not support logs') + raise NotImplementedError( + 'The Kubernetes Agent backend does not currently support shared-file log collection' + ) def _require_prepared(self) -> None: process = self._exec(['test', '-f', PREPARED_MARKER], check=False) diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index 9a2ffebc9c0a2..e8f1c2197e481 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -297,8 +297,8 @@ def test_rejects_invalid_wait_timeout_before_creating_resources(agent, metadata, @pytest.mark.parametrize( ('env_vars', 'expected'), [ - ({AgentEnvVars.DOGSTATSD_PORT: '8125'}, 'DogStatsD'), - ({AgentEnvVars.LOGS_ENABLED: 'true'}, 'logs'), + ({AgentEnvVars.DOGSTATSD_PORT: '8125'}, 'DogStatsD exposure'), + ({AgentEnvVars.LOGS_ENABLED: 'true'}, 'shared-file log collection'), ], ) def test_rejects_unsupported_options_before_creating_resources(agent, run_command, env_vars, expected): From 99056e45d4254f6a92468bff18762cf03b84c2bb Mon Sep 17 00:00:00 2001 From: Enrico Donnici Date: Mon, 3 Aug 2026 08:46:45 +0000 Subject: [PATCH 50/50] Keep kubectl warnings out of structured output --- ddev/src/ddev/e2e/agent/kubernetes.py | 27 +++++++++++++++---------- ddev/tests/e2e/agent/test_kubernetes.py | 26 ++++++++++++++++++++---- 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/ddev/src/ddev/e2e/agent/kubernetes.py b/ddev/src/ddev/e2e/agent/kubernetes.py index efd1d01cae247..c56cc40bb93c1 100644 --- a/ddev/src/ddev/e2e/agent/kubernetes.py +++ b/ddev/src/ddev/e2e/agent/kubernetes.py @@ -84,20 +84,21 @@ def _wait_timeout(self) -> int: def _kubectl(self, args: list[str], **kwargs) -> subprocess.CompletedProcess: return self.platform.run_command([*self._kubectl_prefix, *args], **kwargs) - def _captured_kubectl(self, args: list[str], **kwargs) -> subprocess.CompletedProcess: + def _captured_kubectl(self, args: list[str], *, merge_stderr: bool = True, **kwargs) -> subprocess.CompletedProcess: return self._kubectl( args, stdout=self.platform.modules.subprocess.PIPE, - stderr=self.platform.modules.subprocess.STDOUT, + stderr=(self.platform.modules.subprocess.STDOUT if merge_stderr else self.platform.modules.subprocess.PIPE), **kwargs, ) @staticmethod - def _process_output(process: subprocess.CompletedProcess) -> str: - output = process.stdout or b'' - if isinstance(output, bytes): - return output.decode('utf-8', errors='replace') - return output + def _process_output(process: subprocess.CompletedProcess, *, include_stderr: bool = False) -> str: + streams = (process.stdout, process.stderr) if include_stderr else (process.stdout,) + return ''.join( + stream.decode('utf-8', errors='replace') if isinstance(stream, bytes) else stream or '' + for stream in streams + ) def _exec( self, @@ -117,18 +118,22 @@ def _exec( return self._kubectl(args, check=check) def _validate_context(self) -> None: - process = self._captured_kubectl(['config', 'current-context']) + process = self._captured_kubectl(['config', 'current-context'], merge_stderr=False) if process.returncode: - raise RuntimeError(f'Unable to inspect Kubernetes context: {self._process_output(process)}') + raise RuntimeError( + f'Unable to inspect Kubernetes context: {self._process_output(process, include_stderr=True)}' + ) context = self._process_output(process).strip() if not context.startswith('kind-'): raise RuntimeError(f'Refusing to use non-Kind Kubernetes context `{context}`') def _validate_topology(self) -> None: - process = self._captured_kubectl(['get', 'nodes', '-o', 'json']) + process = self._captured_kubectl(['get', 'nodes', '-o', 'json'], merge_stderr=False) if process.returncode: - raise RuntimeError(f'Unable to inspect Kubernetes nodes: {self._process_output(process)}') + raise RuntimeError( + f'Unable to inspect Kubernetes nodes: {self._process_output(process, include_stderr=True)}' + ) try: node_data = json.loads(self._process_output(process)) diff --git a/ddev/tests/e2e/agent/test_kubernetes.py b/ddev/tests/e2e/agent/test_kubernetes.py index 9a2ffebc9c0a2..35f5f881bd89d 100644 --- a/ddev/tests/e2e/agent/test_kubernetes.py +++ b/ddev/tests/e2e/agent/test_kubernetes.py @@ -56,8 +56,8 @@ def agent(app, get_integration, metadata, config_file): return KubernetesAgent(app, get_integration('velero'), 'py3.12', metadata, config_file) -def successful_process(command, *, stdout=b''): - return subprocess.CompletedProcess(command, 0, stdout=stdout) +def successful_process(command, *, stdout=b'', stderr=None): + return subprocess.CompletedProcess(command, 0, stdout=stdout, stderr=stderr) @pytest.fixture @@ -339,7 +339,7 @@ def test_rejects_non_kind_context_before_inspecting_cluster(agent, app, mocker): run_command.assert_called_once_with( ['kubectl', '--kubeconfig', TEST_KUBECONFIG, 'config', 'current-context'], stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, ) @@ -363,7 +363,25 @@ def run(command, **kwargs): assert not any('create' in command for command in calls) for index in (context_index, topology_index): assert run_command.call_args_list[index].kwargs['stdout'] is subprocess.PIPE - assert run_command.call_args_list[index].kwargs['stderr'] is subprocess.STDOUT + assert run_command.call_args_list[index].kwargs['stderr'] is subprocess.PIPE + + +def test_validation_ignores_kubectl_warnings(agent, app, mocker): + warning = b'Warning: deprecated API\n' + nodes = {'items': [{'metadata': {'name': 'kind-control-plane'}, 'spec': {}}]} + run_command = mocker.patch.object( + app.platform, + 'run_command', + side_effect=[ + successful_process([], stdout=b'kind-test\n', stderr=warning), + successful_process([], stdout=json.dumps(nodes).encode(), stderr=warning), + ], + ) + + agent._validate_context() + agent._validate_topology() + + assert all(call.kwargs['stderr'] is subprocess.PIPE for call in run_command.call_args_list) def test_partial_manifest_creation_failure_is_propagated(agent, app, mocker):