You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/11/15 02:51:49 UTC
[airflow] 06/32: Add CeleryKubernetesExecutor to helm chart (#11288)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 4c2cb60e6294475f252979832e3842a7f3500667
Author: Daniel Imberman <da...@gmail.com>
AuthorDate: Mon Oct 5 15:46:10 2020 -0700
Add CeleryKubernetesExecutor to helm chart (#11288)
Users of the CeleryKubernetesExecutor will require both
Celery and Kubernetes features to launch tasks.
This PR will also serve as the basis for integration tests for this
executor
Co-authored-by: Daniel Imberman <da...@astronomer.io>
(cherry picked from commit 48474600ccd7c586eb5dd8ea9aa2d818db7de505)
---
chart/templates/NOTES.txt | 16 ++++++--
chart/templates/_helpers.yaml | 6 +--
chart/templates/configmap.yaml | 2 +-
chart/templates/flower/flower-deployment.yaml | 2 +-
.../flower-ingress.yaml} | 43 ++++++++++++----------
chart/templates/flower/flower-service.yaml | 2 +-
chart/templates/rbac/pod-launcher-rolebinding.yaml | 4 +-
chart/templates/redis/redis-networkpolicy.yaml | 2 +-
chart/templates/redis/redis-service.yaml | 2 +-
chart/templates/redis/redis-statefulset.yaml | 2 +-
.../templates/scheduler/scheduler-deployment.yaml | 2 -
chart/templates/workers/worker-deployment.yaml | 2 +-
chart/templates/workers/worker-kedaautoscaler.yaml | 2 +-
chart/templates/workers/worker-networkpolicy.yaml | 2 +-
chart/templates/workers/worker-service.yaml | 2 +-
15 files changed, 51 insertions(+), 40 deletions(-)
diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt
index 3df1e3b..afca72a 100644
--- a/chart/templates/NOTES.txt
+++ b/chart/templates/NOTES.txt
@@ -20,10 +20,18 @@ under the License.
Your release is named {{ .Release.Name }}.
-You can now access your dashboard(s) by executing the following command(s) and visiting the corresponding port at localhost in your browser:
+{{- if .Values.ingress.enabled }}
+You can now access your dashboard(s) by following defined Ingress urls:
-Airflow dashboard: kubectl port-forward svc/{{ .Release.Name }}-webserver {{ .Values.ports.airflowUI }}:{{ .Values.ports.airflowUI }}
-{{- if eq .Values.executor "CeleryExecutor"}}
-Flower dashboard: kubectl port-forward svc/{{ .Release.Name }}-flower {{ .Values.ports.flowerUI }}:{{ .Values.ports.flowerUI }}
+Airflow dashboard: http{{ if .Values.ingress.web.tls.enabled }}s{{ end }}://{{ .Values.ingress.web.host }}{{ .Values.ingress.web.path }}/
+{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
+Flower dashboard: http{{ if .Values.ingress.flower.tls.enabled }}s{{ end }}://{{ .Values.ingress.flower.host }}{{ .Values.ingress.flower.path }}/
+{{- end }}
+{{- else }}
+You can now access your dashboard(s) by executing the following command(s) and visiting the corresponding port at localhost in your browser:
+Airflow dashboard: kubectl port-forward svc/{{ .Release.Name }}-webserver {{ .Values.ports.airflowUI }}:{{ .Values.ports.airflowUI }} --namespace {{ .Release.Namespace }}
+{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")}}
+Flower dashboard: kubectl port-forward svc/{{ .Release.Name }}-flower {{ .Values.ports.flowerUI }}:{{ .Values.ports.flowerUI }} --namespace {{ .Release.Namespace }}
+{{- end }}
{{- end }}
diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml
index 5d3ae73..02b255a 100644
--- a/chart/templates/_helpers.yaml
+++ b/chart/templates/_helpers.yaml
@@ -33,7 +33,7 @@
secretKeyRef:
name: {{ template "airflow_metadata_secret" . }}
key: connection
- {{- if eq .Values.executor "CeleryExecutor" }}
+ {{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
- name: AIRFLOW__CELERY__CELERY_RESULT_BACKEND
valueFrom:
secretKeyRef:
@@ -72,7 +72,7 @@
{{- range $i, $config := .Values.env }}
- name: {{ $config.name }}
value: {{ $config.value | quote }}
- {{- if eq $.Values.executor "KubernetesExecutor" }}
+ {{- if or (eq $.Values.executor "KubernetesExecutor") (eq $.Values.executor "CeleryKubernetesExecutor") }}
- name: AIRFLOW__KUBERNETES_ENVIRONMENT_VARIABLES__{{ $config.name }}
value: {{ $config.value | quote }}
{{- end }}
@@ -85,7 +85,7 @@
name: {{ $config.secretName }}
key: {{ default "value" $config.secretKey }}
{{- end }}
- {{- if eq .Values.executor "KubernetesExecutor" }}
+ {{- if or (eq $.Values.executor "KubernetesExecutor") (eq $.Values.executor "CeleryKubernetesExecutor") }}
{{- range $i, $config := .Values.secret }}
- name: AIRFLOW__KUBERNETES_SECRETS__{{ $config.envName }}
value: {{ printf "%s=%s" $config.secretName $config.secretKey }}
diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml
index b5bc656..2cf4502 100644
--- a/chart/templates/configmap.yaml
+++ b/chart/templates/configmap.yaml
@@ -55,7 +55,7 @@ data:
known_hosts: |
{{ .Values.dags.gitSync.knownHosts | nindent 4 }}
{{- end }}
-{{- if eq .Values.executor "KubernetesExecutor" }}
+{{- if or (eq $.Values.executor "KubernetesExecutor") (eq $.Values.executor "CeleryKubernetesExecutor") }}
pod_template_file.yaml: |-
{{- if .Values.podTemplate }}
{{ .Values.podTemplate | nindent 4 }}
diff --git a/chart/templates/flower/flower-deployment.yaml b/chart/templates/flower/flower-deployment.yaml
index 145c80b..887b70d 100644
--- a/chart/templates/flower/flower-deployment.yaml
+++ b/chart/templates/flower/flower-deployment.yaml
@@ -18,7 +18,7 @@
################################
## Airflow Flower Deployment
#################################
-{{- if eq .Values.executor "CeleryExecutor" }}
+{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
kind: Deployment
apiVersion: apps/v1
metadata:
diff --git a/chart/templates/workers/worker-service.yaml b/chart/templates/flower/flower-ingress.yaml
similarity index 54%
copy from chart/templates/workers/worker-service.yaml
copy to chart/templates/flower/flower-ingress.yaml
index 2c7768f..228631b 100644
--- a/chart/templates/workers/worker-service.yaml
+++ b/chart/templates/flower/flower-ingress.yaml
@@ -16,31 +16,36 @@
# under the License.
################################
-## Airflow Worker Service
+## Airflow Flower Ingress
#################################
-{{- if eq .Values.executor "CeleryExecutor" }}
-kind: Service
-apiVersion: v1
+{{- if and .Values.ingress.enabled (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }}
+apiVersion: networking.k8s.io/v1beta1
+kind: Ingress
metadata:
- name: {{ .Release.Name }}-worker
+ name: {{ .Release.Name }}-flower-ingress
labels:
tier: airflow
- component: worker
+ component: flower-ingress
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
-{{- with .Values.labels }}
-{{ toYaml . | indent 4 }}
-{{- end }}
+ annotations:
+ {{- range $key, $value := .Values.ingress.flower.annotations }}
+ {{ $key }}: {{ $value | quote }}
+ {{- end }}
spec:
- clusterIP: None
- selector:
- tier: airflow
- component: worker
- release: {{ .Release.Name }}
- ports:
- - name: worker-logs
- protocol: TCP
- port: {{ .Values.ports.workerLogs }}
- targetPort: {{ .Values.ports.workerLogs }}
+ {{- if .Values.ingress.flower.tls.enabled }}
+ tls:
+ - hosts:
+ - {{ .Values.ingress.flower.host }}
+ secretName: {{ .Values.ingress.flower.tls.secretName }}
+ {{- end }}
+ rules:
+ - http:
+ paths:
+ - path: {{ .Values.ingress.flower.path }}
+ backend:
+ serviceName: {{ .Release.Name }}-flower
+ servicePort: flower-ui
+ host: {{ .Values.ingress.flower.host }}
{{- end }}
diff --git a/chart/templates/flower/flower-service.yaml b/chart/templates/flower/flower-service.yaml
index 187046b8f..b723a12 100644
--- a/chart/templates/flower/flower-service.yaml
+++ b/chart/templates/flower/flower-service.yaml
@@ -18,7 +18,7 @@
################################
## Airflow Flower Service Component
#################################
-{{- if eq .Values.executor "CeleryExecutor" }}
+{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
kind: Service
apiVersion: v1
metadata:
diff --git a/chart/templates/rbac/pod-launcher-rolebinding.yaml b/chart/templates/rbac/pod-launcher-rolebinding.yaml
index 6a1a5be..a3eab66 100644
--- a/chart/templates/rbac/pod-launcher-rolebinding.yaml
+++ b/chart/templates/rbac/pod-launcher-rolebinding.yaml
@@ -19,8 +19,8 @@
## Airflow Pod Launcher Role Binding
#################################
{{- if and .Values.rbacEnabled .Values.allowPodLaunching }}
-{{- $grantScheduler := or (eq .Values.executor "LocalExecutor") (eq .Values.executor "SequentialExecutor") (eq .Values.executor "KubernetesExecutor") }}
-{{- $grantWorker := or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "KubernetesExecutor") }}
+{{- $grantScheduler := or (eq .Values.executor "LocalExecutor") (eq .Values.executor "SequentialExecutor") (eq .Values.executor "KubernetesExecutor") (eq .Values.executor "CeleryKubernetesExecutor")}}
+{{- $grantWorker := or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "KubernetesExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
{{- if .Values.multiNamespaceMode }}
kind: ClusterRoleBinding
{{- else }}
diff --git a/chart/templates/redis/redis-networkpolicy.yaml b/chart/templates/redis/redis-networkpolicy.yaml
index b0a9e49..a7a7950 100644
--- a/chart/templates/redis/redis-networkpolicy.yaml
+++ b/chart/templates/redis/redis-networkpolicy.yaml
@@ -18,7 +18,7 @@
################################
## Airflow Redis NetworkPolicy
#################################
-{{- if (and .Values.networkPolicies.enabled (eq .Values.executor "CeleryExecutor")) }}
+{{- if (and .Values.networkPolicies.enabled (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor"))) }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
diff --git a/chart/templates/redis/redis-service.yaml b/chart/templates/redis/redis-service.yaml
index ac4bb21..b6e79db 100644
--- a/chart/templates/redis/redis-service.yaml
+++ b/chart/templates/redis/redis-service.yaml
@@ -18,7 +18,7 @@
################################
## Airflow Redis Service
#################################
-{{- if eq .Values.executor "CeleryExecutor" }}
+{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
kind: Service
apiVersion: v1
metadata:
diff --git a/chart/templates/redis/redis-statefulset.yaml b/chart/templates/redis/redis-statefulset.yaml
index e1eff0c..6cd7361 100644
--- a/chart/templates/redis/redis-statefulset.yaml
+++ b/chart/templates/redis/redis-statefulset.yaml
@@ -18,7 +18,7 @@
################################
## Airflow Redis StatefulSet
#################################
-{{- if eq .Values.executor "CeleryExecutor" }}
+{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
kind: StatefulSet
apiVersion: apps/v1
metadata:
diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml
index f2b4a99..794a473 100644
--- a/chart/templates/scheduler/scheduler-deployment.yaml
+++ b/chart/templates/scheduler/scheduler-deployment.yaml
@@ -21,8 +21,6 @@
# Are we using a local/sequenial executor?
{{- $local := or (eq .Values.executor "LocalExecutor") (eq .Values.executor "SequentialExecutor") }}
-# Are we using the kubernetes executor?
-{{- $kube := eq .Values.executor "KubernetesExecutor" }}
# Is persistence enabled on the _workers_?
# This is important because in $local mode, the scheduler assumes the role of the worker
{{- $persistence := .Values.workers.persistence.enabled }}
diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml
index fe07e20..38b3a9a 100644
--- a/chart/templates/workers/worker-deployment.yaml
+++ b/chart/templates/workers/worker-deployment.yaml
@@ -19,7 +19,7 @@
## Airflow Worker Deployment
#################################
{{- $persistence := .Values.workers.persistence.enabled }}
-{{- if eq .Values.executor "CeleryExecutor" }}
+{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
kind: {{ if $persistence }}StatefulSet{{ else }}Deployment{{ end }}
apiVersion: apps/v1
metadata:
diff --git a/chart/templates/workers/worker-kedaautoscaler.yaml b/chart/templates/workers/worker-kedaautoscaler.yaml
index 891afbf..c733618 100644
--- a/chart/templates/workers/worker-kedaautoscaler.yaml
+++ b/chart/templates/workers/worker-kedaautoscaler.yaml
@@ -18,7 +18,7 @@
################################
## Airflow Worker KEDA Scaler
#################################
-{{- if (and .Values.workers.keda.enabled (eq .Values.executor "CeleryExecutor")) }}
+{{- if (and .Values.workers.keda.enabled ( or (eq .Values.executor "CeleryExecutor")) (eq .Values.executor "CeleryKubernetesExecutor")) }}
apiVersion: keda.k8s.io/v1alpha1
kind: ScaledObject
metadata:
diff --git a/chart/templates/workers/worker-networkpolicy.yaml b/chart/templates/workers/worker-networkpolicy.yaml
index f6420e3..2c5e123 100644
--- a/chart/templates/workers/worker-networkpolicy.yaml
+++ b/chart/templates/workers/worker-networkpolicy.yaml
@@ -18,7 +18,7 @@
################################
## Airflow Worker NetworkPolicy
#################################
-{{- if (and .Values.networkPolicies.enabled (eq .Values.executor "CeleryExecutor")) }}
+{{- if (and .Values.networkPolicies.enabled ( or (eq .Values.executor "CeleryExecutor")) (eq .Values.executor "CeleryKubernetesExecutor")) }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
diff --git a/chart/templates/workers/worker-service.yaml b/chart/templates/workers/worker-service.yaml
index 2c7768f..8923f53 100644
--- a/chart/templates/workers/worker-service.yaml
+++ b/chart/templates/workers/worker-service.yaml
@@ -18,7 +18,7 @@
################################
## Airflow Worker Service
#################################
-{{- if eq .Values.executor "CeleryExecutor" }}
+{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
kind: Service
apiVersion: v1
metadata: