diff --git a/chart/docs/production-guide.rst b/chart/docs/production-guide.rst index 3f45a3632c66b..f916852d1f837 100644 --- a/chart/docs/production-guide.rst +++ b/chart/docs/production-guide.rst @@ -318,6 +318,39 @@ See :ref:`Extending Airflow Image ` and/or `Building the image `_ for more details on how you can extend, customize and test the modifications of Airflow image. +Per-component image overrides +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can set a different image for the API server, scheduler, DAG processor, and triggerer +main containers under ``images.airflow``. This is useful when you build component-specific +images (for example slimmer images per role) while keeping a shared default for other +Airflow containers. + +Omitted fields in a component block inherit from ``images.airflow`` and then from +``defaultAirflowRepository``, ``defaultAirflowTag``, and ``defaultAirflowDigest``. +When both tag and digest are set for a component, digest takes precedence. + +.. code-block:: yaml + :caption: values.yaml + + images: + airflow: + repository: my-registry/airflow + tag: "3.1.0" + scheduler: + repository: my-registry/airflow-scheduler + tag: "3.1.0-slim" + apiServer: + tag: "3.1.0-api" + +.. code-block:: bash + + helm upgrade --install my-release apache-airflow/airflow \ + --set images.airflow.scheduler.repository=my-registry/airflow-scheduler \ + --set images.airflow.scheduler.tag=3.1.0-slim + +See :doc:`parameters-ref` for the full list of ``images.airflow`` parameters. + Managing Dag Files ------------------ diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 2fefc497905d1..b10abc7ed9b6e 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -1167,3 +1167,25 @@ Usage: path: namespace {{- end }} {{- end -}} + +{{/* +Configure the image for the different components. + +Usage: + {{ include "airflow_component_image" (merge (dict "component" "apiServer") .) }} +*/}} +{{- define "airflow_component_image" -}} + {{- $override := index .Values.images.airflow .component | default dict -}} + {{- $repository := $override.repository | default .Values.images.airflow.repository | default .Values.defaultAirflowRepository -}} + {{- $defaultTag := .Values.images.airflow.tag | default .Values.defaultAirflowTag -}} + {{- $defaultDigest := .Values.images.airflow.digest | default .Values.defaultAirflowDigest -}} + {{- if $override.digest }} + {{- printf "%s@%s" $repository $override.digest -}} + {{- else if $override.tag }} + {{- printf "%s:%s" $repository $override.tag -}} + {{- else if $defaultDigest }} + {{- printf "%s@%s" $repository $defaultDigest -}} + {{- else }} + {{- printf "%s:%s" $repository $defaultTag -}} + {{- end }} +{{- end }} diff --git a/chart/templates/api-server/api-server-deployment.yaml b/chart/templates/api-server/api-server-deployment.yaml index bc02135ae2b04..f507ed4b12942 100644 --- a/chart/templates/api-server/api-server-deployment.yaml +++ b/chart/templates/api-server/api-server-deployment.yaml @@ -161,7 +161,7 @@ spec: {{- end }} containers: - name: api-server - image: {{ template "airflow_image" . }} + image: {{ include "airflow_component_image" (merge (dict "component" "apiServer") .) }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} securityContext: {{ $containerSecurityContext | nindent 12 }} {{- if $containerLifecycleHooks }} diff --git a/chart/templates/dag-processor/dag-processor-deployment.yaml b/chart/templates/dag-processor/dag-processor-deployment.yaml index d3596527c7765..77fc59dced9da 100644 --- a/chart/templates/dag-processor/dag-processor-deployment.yaml +++ b/chart/templates/dag-processor/dag-processor-deployment.yaml @@ -145,7 +145,7 @@ spec: {{- end }} containers: - name: dag-processor - image: {{ template "airflow_image" . }} + image: {{ include "airflow_component_image" (merge (dict "component" "dagProcessor") .) }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} securityContext: {{ $containerSecurityContext | nindent 12 }} {{- if $containerLifecycleHooks }} diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index 35749fdbc5ba2..baa1606d09a69 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -177,7 +177,7 @@ spec: {{- end }} containers: - name: scheduler - image: {{ template "airflow_image" . }} + image: {{ include "airflow_component_image" (merge (dict "component" "scheduler") .) }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} securityContext: {{ $containerSecurityContext | nindent 12 }} {{- if $containerLifecycleHooks }} diff --git a/chart/templates/triggerer/triggerer-deployment.yaml b/chart/templates/triggerer/triggerer-deployment.yaml index 591153d3e054c..ec3f7eaec29b9 100644 --- a/chart/templates/triggerer/triggerer-deployment.yaml +++ b/chart/templates/triggerer/triggerer-deployment.yaml @@ -168,7 +168,7 @@ spec: {{- end }} containers: - name: triggerer - image: {{ template "airflow_image" . }} + image: {{ include "airflow_component_image" (merge (dict "component" "triggerer") .) }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} securityContext: {{ $containerSecurityContext | nindent 12 }} {{- if $containerLifecycleHooks }} diff --git a/chart/tests/helm_tests/airflow_aux/test_airflow_common.py b/chart/tests/helm_tests/airflow_aux/test_airflow_common.py index a86ab39a3b573..57f65a5ef4c42 100644 --- a/chart/tests/helm_tests/airflow_aux/test_airflow_common.py +++ b/chart/tests/helm_tests/airflow_aux/test_airflow_common.py @@ -319,6 +319,39 @@ def test_should_use_correct_default_image(self, expected_image, tag, digest): for doc in docs: assert expected_image == jmespath.search("spec.template.spec.initContainers[0].image", doc) + @pytest.mark.parametrize( + ("component", "component_key", "expected_image", "digest"), + [ + ("api-server", "apiServer", "apache/airflow-api-server@api-server-digest", "api-server-digest"), + ("scheduler", "scheduler", "apache/airflow-scheduler:scheduler-tag", None), + ("dag-processor", "dagProcessor", "apache/airflow-dag-processor@test-digest", "test-digest"), + ("triggerer", "triggerer", "apache/airflow-triggerer:triggerer-tag", None), + ], + ) + def test_should_use_correct_component_image(self, component, component_key, expected_image, digest): + docs = render_chart( + values={ + "defaultAirflowRepository": "apache/airflow", + "defaultAirflowTag": "default-tag", + "defaultAirflowDigest": "default-digest", + "images": { + "airflow": { + component_key: { + "repository": f"apache/airflow-{component}", + "tag": f"{component}-tag", + "digest": digest, + }, + }, + }, + }, + show_only=[ + f"templates/{component}/{component}-deployment.yaml", + ], + ) + + for doc in docs: + assert expected_image == jmespath.search("spec.template.spec.containers[0].image", doc) + def test_should_set_correct_helm_hooks_weight(self): docs = render_chart( show_only=["templates/secrets/fernetkey-secret.yaml"], diff --git a/chart/values.schema.json b/chart/values.schema.json index d753a7348feec..5166af73c7cfe 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -865,6 +865,133 @@ "IfNotPresent" ], "default": "IfNotPresent" + }, + "apiServer": { + "description": "Optional image override for the API server deployment main container. Omitted fields inherit from ``images.airflow`` and then ``defaultAirflowRepository`` / ``defaultAirflowTag`` / ``defaultAirflowDigest``.", + "type": "object", + "additionalProperties": false, + "properties": { + "repository": { + "description": "The image repository for the API server main container. If omitted, inherits from ``images.airflow.repository`` or ``defaultAirflowRepository``.", + "type": [ + "string", + "null" + ], + "default": null + }, + "tag": { + "description": "The image tag for the API server main container. If omitted, inherits from ``images.airflow.tag`` or ``defaultAirflowTag``.", + "type": [ + "string", + "null" + ], + "default": null + }, + "digest": { + "description": "The image digest for the API server main container. If set, it overrides the tag. If omitted, inherits from ``images.airflow.digest`` or ``defaultAirflowDigest``.", + "type": [ + "string", + "null" + ], + "default": null + } + } + }, + "scheduler": { + "description": "Optional image override for the scheduler deployment main container. Omitted fields inherit from ``images.airflow`` and then ``defaultAirflowRepository`` / ``defaultAirflowTag`` / ``defaultAirflowDigest``.", + "type": "object", + "additionalProperties": false, + "properties": { + "repository": { + "description": "The image repository for the scheduler main container. If omitted, inherits from ``images.airflow.repository`` or ``defaultAirflowRepository``.", + "type": [ + "string", + "null" + ], + "default": null, + "examples": [ + "my-registry/airflow-scheduler" + ] + }, + "tag": { + "description": "The image tag for the scheduler main container. If omitted, inherits from ``images.airflow.tag`` or ``defaultAirflowTag``.", + "type": [ + "string", + "null" + ], + "default": null + }, + "digest": { + "description": "The image digest for the scheduler main container. If set, it overrides the tag. If omitted, inherits from ``images.airflow.digest`` or ``defaultAirflowDigest``.", + "type": [ + "string", + "null" + ], + "default": null + } + } + }, + "triggerer": { + "description": "Optional image override for the triggerer deployment main container. Omitted fields inherit from ``images.airflow`` and then ``defaultAirflowRepository`` / ``defaultAirflowTag`` / ``defaultAirflowDigest``.", + "type": "object", + "additionalProperties": false, + "properties": { + "repository": { + "description": "The image repository for the triggerer main container. If omitted, inherits from ``images.airflow.repository`` or ``defaultAirflowRepository``.", + "type": [ + "string", + "null" + ], + "default": null + }, + "tag": { + "description": "The image tag for the triggerer main container. If omitted, inherits from ``images.airflow.tag`` or ``defaultAirflowTag``.", + "type": [ + "string", + "null" + ], + "default": null + }, + "digest": { + "description": "The image digest for the triggerer main container. If set, it overrides the tag. If omitted, inherits from ``images.airflow.digest`` or ``defaultAirflowDigest``.", + "type": [ + "string", + "null" + ], + "default": null + } + } + }, + "dagProcessor": { + "description": "Optional image override for the DAG processor deployment main container. Omitted fields inherit from ``images.airflow`` and then ``defaultAirflowRepository`` / ``defaultAirflowTag`` / ``defaultAirflowDigest``.", + "type": "object", + "additionalProperties": false, + "properties": { + "repository": { + "description": "The image repository for the DAG processor main container. If omitted, inherits from ``images.airflow.repository`` or ``defaultAirflowRepository``.", + "type": [ + "string", + "null" + ], + "default": null + }, + "tag": { + "description": "The image tag for the DAG processor main container. If omitted, inherits from ``images.airflow.tag`` or ``defaultAirflowTag``.", + "type": [ + "string", + "null" + ], + "default": null + }, + "digest": { + "description": "The image digest for the DAG processor main container. If set, it overrides the tag. If omitted, inherits from ``images.airflow.digest`` or ``defaultAirflowDigest``.", + "type": [ + "string", + "null" + ], + "default": null + } + } } } }, diff --git a/chart/values.yaml b/chart/values.yaml index 7e8ca0919e547..0d6b9584a331f 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -75,6 +75,12 @@ images: # Specifying digest takes precedence over tag. digest: ~ pullPolicy: IfNotPresent + # Optional per-component image (repository, tag, digest). Omitted fields inherit from + # this images.airflow block and defaultAirflowRepository / defaultAirflowTag / defaultAirflowDigest. + apiServer: {} + scheduler: {} + triggerer: {} + dagProcessor: {} # To avoid images with user code, you can turn this to 'true' and # all the 'run-airflow-migrations' and 'wait-for-airflow-migrations' jobs/containers # will use the images from 'defaultAirflowRepository:defaultAirflowTag' values