diff --git a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py index 093cbeb960e8c..2645e81191951 100644 --- a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py @@ -707,6 +707,97 @@ def _upload_k8s_image(python: str, kubernetes_version: str, output: Output | Non return kind_load_result.returncode, f"Uploaded K8S image to {cluster_name}" +# Test-suite container images that Airflow's K8s system tests pull from Docker +# Hub. Tagged (not `:latest`) so kubelet's default imagePullPolicy is +# IfNotPresent — combined with `kind load` below, this means kubelet uses the +# already-loaded image and never reaches out to Docker Hub. The pin protects +# CI runs from Docker Hub anonymous-pull rate limits, which intermittently +# turn the scheduled K8s test job red. Auto-bumped by +# scripts/ci/prek/upgrade_important_versions.py. +K8S_TEST_IMAGES_TO_PRELOAD: tuple[str, ...] = ( + "alpine:3.23", # xcom_sidecar default in providers/cncf/kubernetes + "busybox:1.37", # busybox-based system tests in kubernetes-tests/ + "ubuntu:24.04", # ubuntu-based system tests in kubernetes-tests/ +) + + +def _docker_pull_with_429_retry(image: str, output: Output | None, max_attempts: int = 5) -> int: + """Run `docker pull ` retrying with exponential backoff on Docker Hub 429s. + + Returns the final docker exit code (0 on success). Non-429 failures fail + fast — only the rate-limit pattern is retried, since for everything else + retrying would just amplify a real error. + """ + import time + + delay = 5 + for attempt in range(1, max_attempts + 1): + result = run_command( + ["docker", "pull", image], + check=False, + output=output, + capture_output=True, + text=True, + ) + if result.returncode == 0: + return 0 + stderr = (result.stderr or "") + (result.stdout or "") + rate_limited = "429" in stderr or "Too Many Requests" in stderr or "toomanyrequests" in stderr + if not rate_limited: + get_console(output=output).print( + f"[error]docker pull {image} failed (non-rate-limit): {stderr.strip()[:500]}" + ) + return result.returncode + if attempt == max_attempts: + get_console(output=output).print( + f"[error]docker pull {image} hit Docker Hub 429 on every {max_attempts} attempts; giving up." + ) + return result.returncode + get_console(output=output).print( + f"[warning]docker pull {image} hit Docker Hub 429 " + f"(attempt {attempt}/{max_attempts}); sleeping {delay}s before retry." + ) + time.sleep(delay) + delay *= 2 + return 1 + + +def _preload_test_images_to_kind( + python: str, + kubernetes_version: str, + output: Output | None, +) -> tuple[int, str]: + """Pre-pull and `kind load` the pinned test-suite images. + + See K8S_TEST_IMAGES_TO_PRELOAD for the list and rationale. Each image is + pulled once on the host (with retry-on-429), then loaded into every kind + node. Pods that reference these images then start without kubelet ever + reaching out to Docker Hub. + """ + cluster_name = get_kind_cluster_name(python=python, kubernetes_version=kubernetes_version) + for image in K8S_TEST_IMAGES_TO_PRELOAD: + get_console(output=output).print( + f"[info]Pre-pulling test image {image} for kind cluster {cluster_name}" + ) + pull_rc = _docker_pull_with_429_retry(image, output=output) + if pull_rc != 0: + return pull_rc, f"docker pull {image} failed" + get_console(output=output).print(f"[info]Loading {image} into kind cluster {cluster_name}") + kind_load_result = run_command_with_k8s_env( + ["kind", "load", "docker-image", "--name", cluster_name, image], + python=python, + output=output, + kubernetes_version=kubernetes_version, + check=False, + ) + if kind_load_result.returncode != 0: + get_console(output=output).print( + f"[error]kind load docker-image {image} into {cluster_name} failed." + ) + return kind_load_result.returncode, f"kind load {image} failed" + return 0, f"Pre-loaded {len(K8S_TEST_IMAGES_TO_PRELOAD)} test images into {cluster_name}" + + @kubernetes_group.command( name="build-k8s-image", help="Build k8s-ready airflow image (optionally all images in parallel).", @@ -2043,6 +2134,16 @@ def _run_complete_tests( returncode, message = _upload_k8s_image( python=python, kubernetes_version=kubernetes_version, output=output ) + if returncode != 0: + _logs(python=python, kubernetes_version=kubernetes_version) + return returncode, message + get_console(output=output).print( + f"\n[info]Pre-loading pinned test images into kind cluster for " + f"Python {python}, Kubernetes {kubernetes_version}\n" + ) + returncode, message = _preload_test_images_to_kind( + python=python, kubernetes_version=kubernetes_version, output=output + ) if returncode != 0: _logs(python=python, kubernetes_version=kubernetes_version) return returncode, message diff --git a/dev/registry/extract_metadata.py b/dev/registry/extract_metadata.py index 463a9c408082b..5d9f635f74ea3 100644 --- a/dev/registry/extract_metadata.py +++ b/dev/registry/extract_metadata.py @@ -46,7 +46,7 @@ try: import tomllib # Python 3.11+ stdlib except ModuleNotFoundError: # pragma: no cover -- Python 3.10 fallback - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] import yaml from registry_contract_models import validate_providers_catalog diff --git a/dev/registry/extract_versions.py b/dev/registry/extract_versions.py index d9dc4e166dcf1..2908b22b32e6a 100644 --- a/dev/registry/extract_versions.py +++ b/dev/registry/extract_versions.py @@ -49,7 +49,7 @@ try: import tomllib # Python 3.11+ stdlib except ModuleNotFoundError: # pragma: no cover -- Python 3.10 fallback - import tomli as tomllib + import tomli as tomllib # type: ignore[no-redef] from registry_contract_models import validate_provider_version_metadata try: diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index a0d9bea9e02c8..8d6413a288594 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -927,6 +927,7 @@ krb Kube kube kubeconfig +kubelet Kubernetes kubernetes KubernetesPodOperator diff --git a/kubernetes-tests/tests/kubernetes_tests/test_kubernetes_pod_operator.py b/kubernetes-tests/tests/kubernetes_tests/test_kubernetes_pod_operator.py index 9cf94c98ac6c5..66825830715d3 100644 --- a/kubernetes-tests/tests/kubernetes_tests/test_kubernetes_pod_operator.py +++ b/kubernetes-tests/tests/kubernetes_tests/test_kubernetes_pod_operator.py @@ -137,7 +137,7 @@ def setup_tests(self, test_label): "affinity": {}, "containers": [ { - "image": "ubuntu", + "image": "ubuntu:24.04", "args": ["echo 10"], "command": ["bash", "-cx"], "env": [], @@ -182,7 +182,7 @@ def test_do_xcom_push_defaults_false(self, kubeconfig_path, tmp_path): shutil.copy(kubeconfig_path, new_config_path) k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -199,7 +199,7 @@ def test_config_path_move(self, kubeconfig_path, tmp_path): k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -218,7 +218,7 @@ def test_config_path_move(self, kubeconfig_path, tmp_path): def test_working_pod(self): k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -235,7 +235,7 @@ def test_working_pod(self): def test_skip_cleanup(self): k = KubernetesPodOperator( namespace="unknown", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -250,7 +250,7 @@ def test_skip_cleanup(self): def test_delete_operator_pod(self): k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -268,7 +268,7 @@ def test_delete_operator_pod(self): def test_skip_on_specified_exit_code(self): k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["exit 42"], task_id=str(uuid4()), @@ -288,7 +288,7 @@ def test_already_checked_on_success(self): """ k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -310,7 +310,7 @@ def test_already_checked_on_failure(self): """ k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["lalala"], labels=self.labels, @@ -331,7 +331,7 @@ def test_already_checked_on_failure(self): def test_pod_hostnetwork(self): k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -351,7 +351,7 @@ def test_pod_dnspolicy(self): dns_policy = "ClusterFirstWithHostNet" k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -373,7 +373,7 @@ def test_pod_schedulername(self): scheduler_name = "default-scheduler" k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -392,7 +392,7 @@ def test_pod_node_selector(self): node_selector = {"beta.kubernetes.io/os": "linux"} k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -414,7 +414,7 @@ def test_pod_resources(self): ) k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -493,7 +493,7 @@ def test_pod_affinity(self, val): } k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -516,7 +516,7 @@ def test_port(self): k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -547,7 +547,7 @@ def test_volume_mount(self): ] k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=args, labels=self.labels, @@ -577,7 +577,7 @@ def test_run_as_user(self, uid): name = str(uuid4()) k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], task_id=name, @@ -602,7 +602,7 @@ def test_fs_group(self, gid): name = str(uuid4()) k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], task_id=name, @@ -626,7 +626,7 @@ def test_disable_privilege_escalation(self): k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -666,7 +666,7 @@ def test_faulty_image(self): def test_faulty_service_account(self): k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -688,7 +688,7 @@ def test_pod_failure(self): bad_internal_command = ["foobar 10 "] k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=bad_internal_command, labels=self.labels, @@ -708,7 +708,7 @@ def test_xcom_push(self, test_label): args = [f"echo '{json.dumps(expected)}' > /airflow/xcom/return.json"] k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=args, labels=self.labels, @@ -733,7 +733,7 @@ def test_env_vars(self): k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], env_vars=env_vars, @@ -852,7 +852,7 @@ def test_full_pod_spec(self, test_label): containers=[ k8s.V1Container( name="base", - image="ubuntu", + image="ubuntu:24.04", command=["/bin/bash"], args=["-c", 'echo {\\"hello\\" : \\"world\\"} | cat > /airflow/xcom/return.json'], env=[k8s.V1EnvVar(name="env_name", value="value")], @@ -901,7 +901,7 @@ def test_init_container(self): init_container = k8s.V1Container( name="init-container", - image="ubuntu", + image="ubuntu:24.04", env=init_environments, volume_mounts=volume_mounts, command=["bash", "-cx"], @@ -914,7 +914,7 @@ def test_init_container(self): ) expected_init_container = { "name": "init-container", - "image": "ubuntu", + "image": "ubuntu:24.04", "command": ["bash", "-cx"], "args": ["echo 10"], "env": [{"name": "key1", "value": "value1"}, {"name": "key2", "value": "value2"}], @@ -923,7 +923,7 @@ def test_init_container(self): k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -1037,7 +1037,7 @@ def test_pod_template_file( }, { "command": ["sh", "-c", 'trap "exit 0" INT; while true; do sleep 1; done;'], - "image": "alpine", + "image": "alpine:3.23", "name": "airflow-xcom-sidecar", "resources": { "requests": {"cpu": "1m", "memory": "10Mi"}, @@ -1077,7 +1077,7 @@ def test_pod_priority_class_name(self, hook_mock, await_pod_completion_mock): priority_class_name = "medium-test" k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -1100,7 +1100,7 @@ def test_pod_name(self): pod_name_too_long = "a" * 221 k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -1122,7 +1122,7 @@ def test_on_kill(self): namespace = "default" k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["sleep 1000"], labels=self.labels, @@ -1164,7 +1164,7 @@ def test_reattach_failing_pod_once(self): def get_op(): return KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["exit 1"], labels=self.labels, @@ -1225,7 +1225,7 @@ def get_op(): def test_changing_base_container_name_with_get_logs(self): k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -1255,7 +1255,7 @@ def test_changing_base_container_name_no_logs(self): """ k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels=self.labels, @@ -1285,7 +1285,7 @@ def test_changing_base_container_name_no_logs_long(self): """ k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["bash", "-cx"], arguments=["sleep 3"], labels=self.labels, @@ -1311,7 +1311,7 @@ def test_changing_base_container_name_no_logs_long(self): def test_changing_base_container_name_failure(self): k = KubernetesPodOperator( namespace="default", - image="ubuntu", + image="ubuntu:24.04", cmds=["exit"], arguments=["1"], labels=self.labels, @@ -1360,13 +1360,13 @@ def test_init_container_logs(self): callback = MagicMock() init_container = k8s.V1Container( name="init-container", - image="busybox", + image="busybox:1.37", command=["sh", "-cx"], args=[f"echo {marker_from_init_container}"], ) k = KubernetesPodOperator( namespace="default", - image="busybox", + image="busybox:1.37", cmds=["sh", "-cx"], arguments=[f"echo {marker_from_main_container}"], labels=self.labels, @@ -1393,25 +1393,25 @@ def test_init_container_logs_filtered(self): callback = MagicMock() init_container_to_log_1 = k8s.V1Container( name="init-container-to-log-1", - image="busybox", + image="busybox:1.37", command=["sh", "-cx"], args=[f"echo {marker_from_init_container_to_log_1}"], ) init_container_to_log_2 = k8s.V1Container( name="init-container-to-log-2", - image="busybox", + image="busybox:1.37", command=["sh", "-cx"], args=[f"echo {marker_from_init_container_to_log_2}"], ) init_container_to_ignore = k8s.V1Container( name="init-container-to-ignore", - image="busybox", + image="busybox:1.37", command=["sh", "-cx"], args=[f"echo {marker_from_init_container_to_ignore}"], ) k = KubernetesPodOperator( namespace="default", - image="busybox", + image="busybox:1.37", cmds=["sh", "-cx"], arguments=[f"echo {marker_from_main_container}"], labels=self.labels, @@ -1477,7 +1477,7 @@ def test_log_output_configurations(self, log_prefix_enabled, log_formatter, expe marker = f"test_log_{uuid4()}" k = KubernetesPodOperator( namespace="default", - image="busybox", + image="busybox:1.37", cmds=["sh", "-cx"], arguments=[f"echo {marker}"], labels={"test_label": "test"}, @@ -1559,7 +1559,7 @@ def test_kubernetes_pod_operator_active_deadline_seconds(self, active_deadline_s k = KubernetesPodOperator( task_id=f"test_task_{active_deadline_seconds}", active_deadline_seconds=active_deadline_seconds, - image="busybox", + image="busybox:1.37", cmds=["sh", "-c", echo], namespace=ns, on_finish_action="keep_pod", diff --git a/providers/cncf/kubernetes/docs/changelog.rst b/providers/cncf/kubernetes/docs/changelog.rst index d5672698561dd..73400e734e27d 100644 --- a/providers/cncf/kubernetes/docs/changelog.rst +++ b/providers/cncf/kubernetes/docs/changelog.rst @@ -27,6 +27,20 @@ Changelog --------- +**Default xcom-sidecar image is now pinned to** ``alpine:3.23``. +The default container image for the xcom sidecar (used by ``KubernetesPodOperator`` +when ``do_xcom_push=True``) has changed from the unpinned ``alpine`` (which resolves +to ``alpine:latest``) to the pinned ``alpine:3.23``. The pin makes the kubelet's +default ``imagePullPolicy`` ``IfNotPresent`` instead of ``Always``, so a node with +the image cached does not re-pull on every task — protecting deployments and CI +from Docker Hub anonymous-pull rate limits. + +Deployments that override the image via ``xcom_sidecar_container_image`` (or the +``[kubernetes] xcom_sidecar_container_image`` config) are unaffected. Deployments +that relied on the unpinned default will now be pinned to ``alpine:3.23`` until +the next Airflow upgrade. Set ``xcom_sidecar_container_image`` explicitly if you +need a different alpine version, a private mirror, or another base image. + 10.17.0 ....... diff --git a/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py index 6cdc9febb0252..0931df7e04eff 100644 --- a/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py @@ -22,6 +22,14 @@ from kubernetes.client import models as k8s +# Pinned alpine version for the xcom sidecar default. Pinning (rather than +# using the implicit `:latest`) makes kubelet's default imagePullPolicy +# `IfNotPresent` instead of `Always`, so a node that has the image cached +# does not re-pull on every task — protecting CI and disconnected +# deployments from Docker Hub anonymous-pull rate limits. Tracked by +# scripts/ci/prek/upgrade_important_versions.py. +XCOM_SIDECAR_IMAGE = "alpine:3.23" + class PodDefaults: """Static defaults for Pods.""" @@ -34,7 +42,7 @@ class PodDefaults: SIDECAR_CONTAINER = k8s.V1Container( name=SIDECAR_CONTAINER_NAME, command=["sh", "-c", XCOM_CMD], - image="alpine", + image=XCOM_SIDECAR_IMAGE, volume_mounts=[VOLUME_MOUNT], resources=k8s.V1ResourceRequirements( requests={ diff --git a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes.py index 732831e0e6413..060d03fcb86e7 100644 --- a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes.py +++ b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes.py @@ -149,7 +149,7 @@ # [START howto_operator_k8s_write_xcom] write_xcom = KubernetesPodOperator( namespace="default", - image="alpine", + image="alpine:3.23", cmds=["sh", "-c", "mkdir -p /airflow/xcom/;echo '[1,2,3,4]' > /airflow/xcom/return.json"], name="write-xcom", do_xcom_push=True, diff --git a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_async.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_async.py index e245314386c60..3754c98ccf1fa 100644 --- a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_async.py +++ b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_async.py @@ -137,7 +137,7 @@ namespace="kubernetes_task_async_log", in_cluster=False, name="astro_k8s_test_pod", - image="ubuntu", + image="ubuntu:24.04", cmds=[ "bash", "-cx", @@ -180,7 +180,7 @@ write_xcom_async = KubernetesPodOperator( task_id="kubernetes_write_xcom_task_async", namespace="default", - image="alpine", + image="alpine:3.23", cmds=["sh", "-c", "mkdir -p /airflow/xcom/;echo '[1,2,3,4]' > /airflow/xcom/return.json"], name="write-xcom", do_xcom_push=True, diff --git a/providers/cncf/kubernetes/tests/unit/cncf/kubernetes/operators/test_pod.py b/providers/cncf/kubernetes/tests/unit/cncf/kubernetes/operators/test_pod.py index e81f248bb50b6..e7b1a27e37cf0 100644 --- a/providers/cncf/kubernetes/tests/unit/cncf/kubernetes/operators/test_pod.py +++ b/providers/cncf/kubernetes/tests/unit/cncf/kubernetes/operators/test_pod.py @@ -892,7 +892,7 @@ def test_xcom_sidecar_container_image_default(self): do_xcom_push=True, ) pod = k.build_pod_request_obj(create_context(k)) - assert pod.spec.containers[1].image == "alpine" + assert pod.spec.containers[1].image == "alpine:3.23" def test_xcom_sidecar_container_resources_default(self): k = KubernetesPodOperator( @@ -2741,7 +2741,7 @@ def test_async_xcom_sidecar_container_image_default_should_execute_successfully( deferrable=True, ) pod = k.build_pod_request_obj(create_context(k)) - assert pod.spec.containers[1].image == "alpine" + assert pod.spec.containers[1].image == "alpine:3.23" def test_async_xcom_sidecar_container_resources_default_should_execute_successfully(self): k = KubernetesPodOperator( diff --git a/scripts/ci/prek/upgrade_important_versions.py b/scripts/ci/prek/upgrade_important_versions.py index 8b751e7d77fed..64c3151cb4f7c 100755 --- a/scripts/ci/prek/upgrade_important_versions.py +++ b/scripts/ci/prek/upgrade_important_versions.py @@ -89,6 +89,79 @@ (AIRFLOW_ROOT_PATH / "dev" / "provider_db_inventory.py", False), (AIRFLOW_ROOT_PATH / "dev" / "pyproject.toml", False), (AIRFLOW_ROOT_PATH / "go-sdk" / ".pre-commit-config.yaml", False), + # Files that pin Docker Hub `alpine:` / `busybox:` tags and should be + # auto-bumped alongside the rest of the "important versions". Adding new + # call sites? Add them here too — the regex in SIMPLE_VERSION_PATTERNS + # only mutates files in this list. + ( + AIRFLOW_ROOT_PATH + / "providers" + / "cncf" + / "kubernetes" + / "src" + / "airflow" + / "providers" + / "cncf" + / "kubernetes" + / "utils" + / "xcom_sidecar.py", + False, + ), + ( + AIRFLOW_ROOT_PATH + / "providers" + / "cncf" + / "kubernetes" + / "tests" + / "system" + / "cncf" + / "kubernetes" + / "example_kubernetes.py", + False, + ), + ( + AIRFLOW_ROOT_PATH + / "providers" + / "cncf" + / "kubernetes" + / "tests" + / "system" + / "cncf" + / "kubernetes" + / "example_kubernetes_async.py", + False, + ), + ( + AIRFLOW_ROOT_PATH + / "providers" + / "cncf" + / "kubernetes" + / "tests" + / "unit" + / "cncf" + / "kubernetes" + / "operators" + / "test_pod.py", + False, + ), + ( + AIRFLOW_ROOT_PATH + / "kubernetes-tests" + / "tests" + / "kubernetes_tests" + / "test_kubernetes_pod_operator.py", + False, + ), + ( + AIRFLOW_ROOT_PATH + / "dev" + / "breeze" + / "src" + / "airflow_breeze" + / "commands" + / "kubernetes_commands.py", + False, + ), ] for file in DOCKER_IMAGES_EXAMPLE_DIR_PATH.rglob("*.sh"): FILES_TO_UPDATE.append((file, False)) @@ -477,6 +550,8 @@ def get_env_bool(name: str, default: bool = True) -> bool: console.print("[bright_blue]Upgrading all important versions") # Package upgrade flags +UPGRADE_ALPINE: bool = get_env_bool("UPGRADE_ALPINE") +UPGRADE_BUSYBOX: bool = get_env_bool("UPGRADE_BUSYBOX") UPGRADE_FLIT_CORE: bool = get_env_bool("UPGRADE_FLIT_CORE") UPGRADE_GITPYTHON: bool = get_env_bool("UPGRADE_GITPYTHON") UPGRADE_GOLANG: bool = get_env_bool("UPGRADE_GOLANG") @@ -604,6 +679,19 @@ def apply_pattern_replacements( "openapi_generator": [ (r"(OPENAPI_GENERATOR_CLI_VER = )(\"[0-9.]+\")", 'OPENAPI_GENERATOR_CLI_VER = "{version}"'), ], + # Pinning Docker Hub base-image tags used by Airflow's K8s system tests + # protects CI from anonymous-pull rate limits — the kind cluster + # `kind load`s the pre-pulled image so kubelet (default + # imagePullPolicy=IfNotPresent for tagged images) never reaches Docker + # Hub. Pattern matches both `alpine:X.Y[.Z]` literals in code and + # `ARG ALPINE_VERSION="X.Y[.Z]"` in chart Dockerfiles. + "alpine": [ + (r"(alpine:)([0-9]+\.[0-9]+(?:\.[0-9]+)?)", "alpine:{version}"), + (r'(ALPINE_VERSION=")([0-9.]+)(")', 'ALPINE_VERSION="{version}"'), + ], + "busybox": [ + (r"(busybox:)([0-9]+\.[0-9]+(?:\.[0-9]+)?)", "busybox:{version}"), + ], "sphinx_airflow_theme": [ ( r"(sphinx-airflow-theme@https://airflow\.apache\.org/sphinx-airflow-theme/sphinx_airflow_theme-)([0-9.]+)(-py3-none-any\.whl)", @@ -640,6 +728,8 @@ def fetch_all_package_versions() -> dict[str, str]: "mypy": get_latest_pypi_version("mypy", UPGRADE_MYPY), "node_lts": get_latest_lts_node_version() if UPGRADE_NODE_LTS else "", "protoc": get_latest_image_version("rvolosatovs/protoc") if UPGRADE_PROTOC else "", + "alpine": get_latest_image_version("alpine") if UPGRADE_ALPINE else "", + "busybox": get_latest_image_version("busybox") if UPGRADE_BUSYBOX else "", "mprocs": get_latest_github_release_version("pvolok/mprocs") if UPGRADE_MPROCS else "", "openapi_generator": get_latest_openapi_generator_version() if UPGRADE_OPENAPI_GENERATOR else "", "sphinx_airflow_theme": get_latest_sphinx_airflow_theme_version()