You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2021/05/26 21:55:42 UTC
[airflow] branch master updated: Chart: Add ``extraInitContainers``
to scheduler/webserver/workers (#16098)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 2c17257 Chart: Add ``extraInitContainers`` to scheduler/webserver/workers (#16098)
2c17257 is described below
commit 2c1725709ff499e78723cb05820f82d3560fe971
Author: Jed Cunningham <66...@users.noreply.github.com>
AuthorDate: Wed May 26 15:55:19 2021 -0600
Chart: Add ``extraInitContainers`` to scheduler/webserver/workers (#16098)
Allow users to specify custom init containers on the core airflow
components.
---
chart/files/pod-template-file.kubernetes-helm-yaml | 11 +++++--
.../templates/scheduler/scheduler-deployment.yaml | 3 ++
.../templates/webserver/webserver-deployment.yaml | 3 ++
chart/templates/workers/worker-deployment.yaml | 3 ++
chart/tests/test_pod_template_file.py | 18 ++++++++++
chart/tests/test_scheduler.py | 28 ++++++++++++----
chart/tests/test_webserver.py | 28 ++++++++++++----
chart/tests/test_worker.py | 28 ++++++++++++----
chart/values.schema.json | 15 +++++++++
chart/values.yaml | 6 ++++
docs/helm-chart/using-additional-containers.rst | 38 +++++++++++++++++++---
11 files changed, 155 insertions(+), 26 deletions(-)
diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml b/chart/files/pod-template-file.kubernetes-helm-yaml
index 4415372..f4f0efd 100644
--- a/chart/files/pod-template-file.kubernetes-helm-yaml
+++ b/chart/files/pod-template-file.kubernetes-helm-yaml
@@ -24,10 +24,15 @@ metadata:
{{- toYaml .Values.airflowPodAnnotations | nindent 4 }}
{{- end }}
spec:
-{{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }}
+ {{- if or (and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled)) .Values.workers.extraInitContainers }}
initContainers:
-{{- include "git_sync_container" (dict "Values" .Values "is_init" "true") | indent 4 }}
-{{- end }}
+ {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }}
+ {{- include "git_sync_container" (dict "Values" .Values "is_init" "true") | nindent 4 }}
+ {{- end }}
+ {{- if .Values.workers.extraInitContainers }}
+ {{- toYaml .Values.workers.extraInitContainers | nindent 4 }}
+ {{- end }}
+ {{- end }}
containers:
- args: []
command: []
diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml
index 8137c84..0065eca 100644
--- a/chart/templates/scheduler/scheduler-deployment.yaml
+++ b/chart/templates/scheduler/scheduler-deployment.yaml
@@ -113,6 +113,9 @@ spec:
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
+ {{- if .Values.scheduler.extraInitContainers }}
+ {{- toYaml .Values.scheduler.extraInitContainers | nindent 8 }}
+ {{- end }}
containers:
# Always run the main scheduler container.
- name: scheduler
diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml
index f17679f..385270c 100644
--- a/chart/templates/webserver/webserver-deployment.yaml
+++ b/chart/templates/webserver/webserver-deployment.yaml
@@ -107,6 +107,9 @@ spec:
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
+ {{- if .Values.webserver.extraInitContainers }}
+ {{- toYaml .Values.webserver.extraInitContainers | nindent 8 }}
+ {{- end }}
containers:
- name: webserver
image: {{ template "airflow_image" . }}
diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml
index b4897fd..cab0de3 100644
--- a/chart/templates/workers/worker-deployment.yaml
+++ b/chart/templates/workers/worker-deployment.yaml
@@ -123,6 +123,9 @@ spec:
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
+ {{- if .Values.workers.extraInitContainers }}
+ {{- toYaml .Values.workers.extraInitContainers | nindent 8 }}
+ {{- end }}
containers:
- name: worker
image: {{ template "airflow_image" . }}
diff --git a/chart/tests/test_pod_template_file.py b/chart/tests/test_pod_template_file.py
index c6d57bf..0b46f66 100644
--- a/chart/tests/test_pod_template_file.py
+++ b/chart/tests/test_pod_template_file.py
@@ -420,3 +420,21 @@ class PodTemplateFileTest(unittest.TestCase):
annotations = jmespath.search("metadata.annotations", docs[0])
assert "my_annotation" in annotations
assert "annotated!" in annotations["my_annotation"]
+
+ def test_should_add_extra_init_containers(self):
+ docs = render_chart(
+ values={
+ "workers": {
+ "extraInitContainers": [
+ {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"}
+ ],
+ },
+ },
+ show_only=["templates/pod-template-file.yaml"],
+ chart_dir=self.temp_chart_dir,
+ )
+
+ assert {
+ "name": "test-init-container",
+ "image": "test-registry/test-repo:test-tag",
+ } == jmespath.search("spec.initContainers[-1]", docs[0])
diff --git a/chart/tests/test_scheduler.py b/chart/tests/test_scheduler.py
index fd3e198..446cbf4 100644
--- a/chart/tests/test_scheduler.py
+++ b/chart/tests/test_scheduler.py
@@ -55,18 +55,34 @@ class SchedulerTest(unittest.TestCase):
"executor": "CeleryExecutor",
"scheduler": {
"extraContainers": [
- {
- "name": "test-container",
- "image": "test-registry/test-repo:test-tag",
- "imagePullPolicy": "Always",
- }
+ {"name": "test-container", "image": "test-registry/test-repo:test-tag"}
],
},
},
show_only=["templates/scheduler/scheduler-deployment.yaml"],
)
- assert "test-container" == jmespath.search("spec.template.spec.containers[-1].name", docs[0])
+ assert {
+ "name": "test-container",
+ "image": "test-registry/test-repo:test-tag",
+ } == jmespath.search("spec.template.spec.containers[-1]", docs[0])
+
+ def test_should_add_extra_init_containers(self):
+ docs = render_chart(
+ values={
+ "scheduler": {
+ "extraInitContainers": [
+ {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"}
+ ],
+ },
+ },
+ show_only=["templates/scheduler/scheduler-deployment.yaml"],
+ )
+
+ assert {
+ "name": "test-init-container",
+ "image": "test-registry/test-repo:test-tag",
+ } == jmespath.search("spec.template.spec.initContainers[-1]", docs[0])
def test_should_add_extra_volume_and_extra_volume_mount(self):
docs = render_chart(
diff --git a/chart/tests/test_webserver.py b/chart/tests/test_webserver.py
index b40e75c..d173ca5 100644
--- a/chart/tests/test_webserver.py
+++ b/chart/tests/test_webserver.py
@@ -132,18 +132,34 @@ class WebserverDeploymentTest(unittest.TestCase):
"executor": "CeleryExecutor",
"webserver": {
"extraContainers": [
- {
- "name": "test-container",
- "image": "test-registry/test-repo:test-tag",
- "imagePullPolicy": "Always",
- }
+ {"name": "test-container", "image": "test-registry/test-repo:test-tag"}
],
},
},
show_only=["templates/webserver/webserver-deployment.yaml"],
)
- assert "test-container" == jmespath.search("spec.template.spec.containers[-1].name", docs[0])
+ assert {
+ "name": "test-container",
+ "image": "test-registry/test-repo:test-tag",
+ } == jmespath.search("spec.template.spec.containers[-1]", docs[0])
+
+ def test_should_add_extra_init_containers(self):
+ docs = render_chart(
+ values={
+ "webserver": {
+ "extraInitContainers": [
+ {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"}
+ ],
+ },
+ },
+ show_only=["templates/webserver/webserver-deployment.yaml"],
+ )
+
+ assert {
+ "name": "test-init-container",
+ "image": "test-registry/test-repo:test-tag",
+ } == jmespath.search("spec.template.spec.initContainers[-1]", docs[0])
def test_should_create_valid_affinity_tolerations_and_node_selector(self):
docs = render_chart(
diff --git a/chart/tests/test_worker.py b/chart/tests/test_worker.py
index eaa772c..f2b825f 100644
--- a/chart/tests/test_worker.py
+++ b/chart/tests/test_worker.py
@@ -52,18 +52,34 @@ class WorkerTest(unittest.TestCase):
"executor": "CeleryExecutor",
"workers": {
"extraContainers": [
- {
- "name": "test-container",
- "image": "test-registry/test-repo:test-tag",
- "imagePullPolicy": "Always",
- }
+ {"name": "test-container", "image": "test-registry/test-repo:test-tag"}
],
},
},
show_only=["templates/workers/worker-deployment.yaml"],
)
- assert "test-container" == jmespath.search("spec.template.spec.containers[-1].name", docs[0])
+ assert {
+ "name": "test-container",
+ "image": "test-registry/test-repo:test-tag",
+ } == jmespath.search("spec.template.spec.containers[-1]", docs[0])
+
+ def test_should_add_extra_init_containers(self):
+ docs = render_chart(
+ values={
+ "workers": {
+ "extraInitContainers": [
+ {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"}
+ ],
+ },
+ },
+ show_only=["templates/workers/worker-deployment.yaml"],
+ )
+
+ assert {
+ "name": "test-init-container",
+ "image": "test-registry/test-repo:test-tag",
+ } == jmespath.search("spec.template.spec.initContainers[-1]", docs[0])
def test_should_add_extra_volume_and_extra_volume_mount(self):
docs = render_chart(
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 9f94ae0..73e05dd 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -1001,6 +1001,11 @@
"type": "array",
"default": []
},
+ "extraInitContainers": {
+ "description": "Add additional init containers into workers.",
+ "type": "array",
+ "default": []
+ },
"extraVolumes": {
"description": "Mount additional volumes into workers.",
"type": "array",
@@ -1179,6 +1184,11 @@
"type": "array",
"default": []
},
+ "extraInitContainers": {
+ "description": "Add additional init containers into scheduler.",
+ "type": "array",
+ "default": []
+ },
"extraVolumes": {
"description": "Mount additional volumes into scheduler.",
"type": "array",
@@ -1455,6 +1465,11 @@
"type": "array",
"default": []
},
+ "extraInitContainers": {
+ "description": "Add additional init containers into webserver.",
+ "type": "array",
+ "default": []
+ },
"extraVolumes": {
"description": "Mount additional volumes into webserver.",
"type": "array",
diff --git a/chart/values.yaml b/chart/values.yaml
index d498b5d..cda9e1e 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -397,6 +397,8 @@ workers:
# Launch additional containers into worker.
extraContainers: []
+ # Add additional init containers into workers.
+ extraInitContainers: []
# Mount additional volumes into worker.
extraVolumes: []
@@ -469,6 +471,8 @@ scheduler:
# Launch additional containers into scheduler.
extraContainers: []
+ # Add additional init containers into scheduler.
+ extraInitContainers: []
# Mount additional volumes into scheduler.
extraVolumes: []
@@ -566,6 +570,8 @@ webserver:
# Launch additional containers into webserver.
extraContainers: []
+ # Add additional init containers into webserver.
+ extraInitContainers: []
# Mount additional volumes into webserver.
extraVolumes: []
diff --git a/docs/helm-chart/using-additional-containers.rst b/docs/helm-chart/using-additional-containers.rst
index a3c44fd..d75214e 100644
--- a/docs/helm-chart/using-additional-containers.rst
+++ b/docs/helm-chart/using-additional-containers.rst
@@ -16,20 +16,48 @@
under the License.
Using additional containers
-----------------------------
+===========================
-If you are using your own sidecar container, you can add it through the ``extraContainers`` value.
-You can define different containers for scheduler, webserver and worker pods.
+Sidecar Containers
+------------------
-For example, a sidecar that syncs DAGs from object storage.
+If you want to deploy your own sidecar container, you can add it through the ``extraContainers`` parameter.
+You can define different containers for the scheduler, webserver and worker pods.
+
+For example, sidecars that sync DAGs from object storage.
.. note::
- ``extraContainers`` value supports CeleryExecutor only.
+ ``workers.extraContainers`` is only functional with ``CeleryExecutor``.
.. code-block:: yaml
+ scheduler:
+ extraContainers:
+ - name: s3-sync
+ image: my-company/s3-sync:latest
+ imagePullPolicy: Always
+ workers:
extraContainers:
- name: s3-sync
image: my-company/s3-sync:latest
imagePullPolicy: Always
+
+
+Init Containers
+---------------
+
+You can also deploy extra init containers through the ``extraInitContainers`` parameter.
+You can define different containers for the scheduler, webserver and worker pods.
+
+For example, an init container that just says hello:
+
+.. code-block:: yaml
+
+ scheduler:
+ extraInitContainers:
+ - name: hello
+ image: debian
+ args:
+ - echo
+ - hello