You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/11/15 02:51:53 UTC
[airflow] 10/32: Mount volumes and volumemounts into scheduler and
workers (#11426)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit f3d5fe2b76958ce780a644be58c278fd0d6c8798
Author: Daniel Imberman <da...@gmail.com>
AuthorDate: Mon Oct 12 11:32:05 2020 -0700
Mount volumes and volumemounts into scheduler and workers (#11426)
* Mount arbitrary volumes and volumeMounts to scheduler and worker
Allows users to mount volumes to scheduler and workers
* tested
(cherry picked from commit 045d68da7fccccdf5b8209a4fd3bf138b9ca9f96)
---
.../templates/scheduler/scheduler-deployment.yaml | 6 ++++
chart/templates/workers/worker-deployment.yaml | 6 ++++
chart/tests/scheduler_test.yaml | 38 ++++++++++++++++++++++
chart/tests/worker_test.yaml | 38 ++++++++++++++++++++++
chart/values.schema.json | 20 ++++++++++--
chart/values.yaml | 7 ++++
6 files changed, 113 insertions(+), 2 deletions(-)
diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml
index 794a473..b6878a7 100644
--- a/chart/templates/scheduler/scheduler-deployment.yaml
+++ b/chart/templates/scheduler/scheduler-deployment.yaml
@@ -152,6 +152,9 @@ spec:
mountPath: {{ template "airflow_dags_mount_path" . }}
{{- include "git_sync_container" . | indent 8 }}
{{- end }}
+{{- if .Values.scheduler.extraVolumeMounts }}
+{{ toYaml .Values.scheduler.extraVolumeMounts | indent 12 }}
+{{- end }}
# Always start the garbage collector sidecar.
- name: scheduler-gc
image: {{ template "airflow_image" . }}
@@ -195,6 +198,9 @@ spec:
{{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret }}
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
{{- end }}
+{{- if .Values.scheduler.extraVolumes }}
+{{ toYaml .Values.scheduler.extraVolumes | indent 8 }}
+{{- end }}
{{- if not $stateful }}
- name: logs
emptyDir: {}
diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml
index 38b3a9a..47aa18e 100644
--- a/chart/templates/workers/worker-deployment.yaml
+++ b/chart/templates/workers/worker-deployment.yaml
@@ -118,6 +118,9 @@ spec:
- name: worker-logs
containerPort: {{ .Values.ports.workerLogs }}
volumeMounts:
+{{- if .Values.workers.extraVolumeMounts }}
+{{ toYaml .Values.workers.extraVolumeMounts | indent 12 }}
+{{- end }}
- name: logs
mountPath: {{ template "airflow_logs" . }}
- name: config
@@ -200,6 +203,9 @@ spec:
{{- include "standard_airflow_environment" . | indent 10 }}
{{- end }}
volumes:
+{{- if .Values.workers.extraVolumes }}
+{{ toYaml .Values.workers.extraVolumes | indent 8 }}
+{{- end }}
- name: kerberos-keytab
secret:
secretName: {{ include "kerberos_keytab_secret" . | quote }}
diff --git a/chart/tests/scheduler_test.yaml b/chart/tests/scheduler_test.yaml
new file mode 100644
index 0000000..327b6e3
--- /dev/null
+++ b/chart/tests/scheduler_test.yaml
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+templates:
+ - scheduler/scheduler-deployment.yaml
+tests:
+ - it: should add extraVolume and extraVolumeMount
+ set:
+ executor: CeleryExecutor
+ scheduler:
+ extraVolumes:
+ - name: test-volume
+ emptyDir: {}
+ extraVolumeMounts:
+ - name: test-volume
+ mountPath: /opt/test
+ asserts:
+ - equal:
+ path: spec.template.spec.volumes[1].name
+ value: test-volume
+ - equal:
+ path: spec.template.spec.containers[0].volumeMounts[3].name
+ value: test-volume
diff --git a/chart/tests/worker_test.yaml b/chart/tests/worker_test.yaml
new file mode 100644
index 0000000..29c4b97
--- /dev/null
+++ b/chart/tests/worker_test.yaml
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+templates:
+ - workers/worker-deployment.yaml
+tests:
+ - it: should add extraVolume and extraVolumeMount
+ set:
+ executor: CeleryExecutor
+ workers:
+ extraVolumes:
+ - name: test-volume
+ emptyDir: {}
+ extraVolumeMounts:
+ - name: test-volume
+ mountPath: /opt/test
+ asserts:
+ - equal:
+ path: spec.template.spec.volumes[0].name
+ value: test-volume
+ - equal:
+ path: spec.template.spec.containers[0].volumeMounts[0].name
+ value: test-volume
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 2fc01a9..9370c80 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -472,7 +472,15 @@
"description": "Annotations to add to the worker kubernetes service account.",
"type": "object"
}
- }
+ },
+ "extraVolumes": {
+ "description": "Mount additional volumes into workers.",
+ "type": "array"
+ },
+ "extraVolumeMounts": {
+ "description": "Mount additional volumes into workers.",
+ "type": "array"
+ }
},
"scheduler": {
"description": "Airflow scheduler settings.",
@@ -516,7 +524,15 @@
"description": "Annotations to add to the scheduler kubernetes service account.",
"type": "object"
}
- }
+ },
+ "extraVolumes": {
+ "description": "Mount additional volumes into scheduler.",
+ "type": "array"
+ },
+ "extraVolumeMounts": {
+ "description": "Mount additional volumes into scheduler.",
+ "type": "array"
+ }
},
"webserver": {
"description": "Airflow webserver settings.",
diff --git a/chart/values.yaml b/chart/values.yaml
index 707a34b..1370088 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -286,6 +286,9 @@ workers:
safeToEvict: true
# Annotations to add to worker kubernetes service account.
serviceAccountAnnotations: {}
+ # Mount additional volumes into worker.
+ extraVolumes: []
+ extraVolumeMounts: []
# Airflow scheduler settings
scheduler:
@@ -316,6 +319,10 @@ scheduler:
# Annotations to add to scheduler kubernetes service account.
serviceAccountAnnotations: {}
+ # Mount additional volumes into scheduler.
+ extraVolumes: []
+ extraVolumeMounts: []
+
# Airflow webserver settings
webserver:
livenessProbe: