You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/11/15 02:51:48 UTC
[airflow] 05/32: Single/Multi-Namespace mode for helm chart (#11034)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit ae681ba4b0c2c83c4d4533f500b539ee64ba0af2
Author: Daniel Imberman <da...@gmail.com>
AuthorDate: Mon Oct 5 10:38:52 2020 -0700
Single/Multi-Namespace mode for helm chart (#11034)
* Multi-Namespace mode for helm chart
Users should not REQUIRE a ClusterRole/ClusterRolebinding
to run airflow via helm. This change will allow "single" and "multi"
namespace modes so users can add airflow to managed kubernetes clusters
* add namespace to role
* add rolebinding too
* add docs
* add values.schema.json change
(cherry picked from commit 93475e9f4dc92d5cb5efea3310e7ddc468c0b991)
---
chart/README.md | 2 +-
chart/templates/rbac/pod-launcher-role.yaml | 7 +
chart/templates/rbac/pod-launcher-rolebinding.yaml | 7 +
chart/values.schema.json | 1059 ++++++++++++++++++++
chart/values.yaml | 145 ++-
5 files changed, 1169 insertions(+), 51 deletions(-)
diff --git a/chart/README.md b/chart/README.md
index 8372bb4..ef1caa3 100644
--- a/chart/README.md
+++ b/chart/README.md
@@ -206,7 +206,7 @@ The following tables lists the configurable parameters of the Airflow chart and
| `webserver.defaultUser` | Optional default airflow user information | `{}` |
| `dags.persistence.*` | Dag persistence configuration | Please refer to `values.yaml` |
| `dags.gitSync.*` | Git sync configuration | Please refer to `values.yaml` |
-
+| `multiNamespaceMode` | Whether the KubernetesExecutor can launch pods in multiple namespaces | `False` |
Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example,
diff --git a/chart/templates/rbac/pod-launcher-role.yaml b/chart/templates/rbac/pod-launcher-role.yaml
index 7e10122..dc56c92 100644
--- a/chart/templates/rbac/pod-launcher-role.yaml
+++ b/chart/templates/rbac/pod-launcher-role.yaml
@@ -19,10 +19,17 @@
## Airflow Pod Launcher Role
#################################
{{- if and .Values.rbacEnabled .Values.allowPodLaunching }}
+{{- if .Values.multiNamespaceMode }}
kind: ClusterRole
+{{- else }}
+kind: Role
+{{- end }}
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ .Release.Name }}-pod-launcher-role
+{{- if not .Values.multiNamespaceMode }}
+ namespace: {{ .Release.Namespace }}
+{{- end }}
labels:
tier: airflow
release: {{ .Release.Name }}
diff --git a/chart/templates/rbac/pod-launcher-rolebinding.yaml b/chart/templates/rbac/pod-launcher-rolebinding.yaml
index 4dba494..6a1a5be 100644
--- a/chart/templates/rbac/pod-launcher-rolebinding.yaml
+++ b/chart/templates/rbac/pod-launcher-rolebinding.yaml
@@ -21,9 +21,16 @@
{{- if and .Values.rbacEnabled .Values.allowPodLaunching }}
{{- $grantScheduler := or (eq .Values.executor "LocalExecutor") (eq .Values.executor "SequentialExecutor") (eq .Values.executor "KubernetesExecutor") }}
{{- $grantWorker := or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "KubernetesExecutor") }}
+{{- if .Values.multiNamespaceMode }}
kind: ClusterRoleBinding
+{{- else }}
+kind: RoleBinding
+{{- end }}
apiVersion: rbac.authorization.k8s.io/v1
metadata:
+{{- if not .Values.multiNamespaceMode }}
+ namespace: {{ .Release.Namespace }}
+{{- end }}
name: {{ .Release.Name }}-pod-launcher-rolebinding
labels:
tier: airflow
diff --git a/chart/values.schema.json b/chart/values.schema.json
new file mode 100644
index 0000000..9776116
--- /dev/null
+++ b/chart/values.schema.json
@@ -0,0 +1,1059 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema",
+ "description": "Default values for airflow. Declare variables to be passed into your templates.",
+ "type": "object",
+ "properties": {
+ "uid": {
+ "description": "User of airflow user.",
+ "type": "integer"
+ },
+ "gid": {
+ "description": "Group of airflow user.",
+ "type": "integer"
+ },
+ "airflowHome": {
+ "description": "Airflow home directory. Used for mount paths.",
+ "type": "string"
+ },
+ "defaultAirflowRepository": {
+ "description": "Default airflow repository. Overrides all the specific images below.",
+ "type": "string"
+ },
+ "defaultAirflowTag": {
+ "description": "Default airflow tag to deploy.",
+ "type": "string"
+ },
+ "multi_namespaceMode": {
+ "description": "Whether the KubernetesExecutor can launch workers in multiple namespaces",
+ "type": "boolean"
+ },
+ "nodeSelector": {
+ "description": "Select certain nodes for airflow pods.",
+ "type": "object",
+ "additionalProperties": {
+ "type": "string"
+ }
+ },
+ "affinity": {
+ "description": "Select certain nodes for airflow pods.",
+ "type": "object"
+ },
+ "tolerations": {
+ "description": "Select certain nodes for airflow pods.",
+ "type": "array"
+ },
+ "labels": {
+ "description": "Add common labels to all objects and pods defined in this chart.",
+ "type": "object",
+ "additionalProperties": {
+ "type": "string"
+ }
+ },
+ "ingress": {
+ "description": "Ingress configuration.",
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "description": "Enable ingress resource.",
+ "type": "boolean"
+ },
+ "web": {
+ "description": "Configuration for the Ingress of the web Service.",
+ "type": "object",
+ "properties": {
+ "annotations": {
+ "description": "Annotations for the web Ingress.",
+ "type": "object"
+ },
+ "path": {
+ "description": "The path for the web Ingress.",
+ "type": "string"
+ },
+ "host": {
+ "description": "The hostname for the web Ingress.",
+ "type": "string"
+ },
+ "tls": {
+ "description": "Configuration for web Ingress TLS.",
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "description": "Enable TLS termination for the web Ingress.",
+ "type": "boolean"
+ },
+ "secretName": {
+ "description": "The name of a pre-created Secret containing a TLS private key and certificate.",
+ "type": "string"
+ }
+ }
+ },
+ "precedingPaths": {
+ "description": "HTTP paths to add to the web Ingress before the default path.",
+ "type": "array"
+ },
+ "succeedingPaths": {
+ "description": "HTTP paths to add to the web Ingress after the default path.",
+ "type": "array"
+ }
+ }
+ },
+ "flower": {
+ "description": "Configuration for the Ingress of the flower Service.",
+ "type": "object",
+ "properties": {
+ "annotations": {
+ "description": "Annotations for the flower Ingress.",
+ "type": "object"
+ },
+ "path": {
+ "description": "The path for the flower Ingress.",
+ "type": "string"
+ },
+ "host": {
+ "description": "The hostname for the flower Ingress.",
+ "type": "string"
+ },
+ "tls": {
+ "description": "Configuration for flower Ingress TLS.",
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "description": "Enable TLS termination for the flower Ingress.",
+ "type": "boolean"
+ },
+ "secretName": {
+ "description": "The name of a pre-created Secret containing a TLS private key and certificate.",
+ "type": "string"
+ }
+ }
+ },
+ "precedingPaths": {
+ "description": "HTTP paths to add to the flower Ingress before the default path.",
+ "type": "array"
+ },
+ "succeedingPaths": {
+ "description": "HTTP paths to add to the flower Ingress after the default path.",
+ "type": "array"
+ }
+ }
+ }
+ }
+ },
+ "networkPolicies": {
+ "description": "Network policy configuration.",
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "description": "Enabled network policies.",
+ "type": "boolean"
+ }
+ }
+ },
+ "airflowPodAnnotations": {
+ "description": "Extra annotations to apply to all Airflow pods.",
+ "type": "object"
+ },
+ "rbacEnabled": {
+ "description": "Enable RBAC (default on most clusters these days).",
+ "type": "boolean"
+ },
+ "executor": {
+ "description": "Airflow executor.",
+ "type": "string"
+ },
+ "allowPodLaunching": {
+ "description": "If this is true and using LocalExecutor/SequentialExecutor/KubernetesExecutor, the scheduler's service account will have access to communicate with the api-server and launch pods. If this is true and using the CeleryExecutor, the workers will be able to launch pods.",
+ "type": "boolean"
+ },
+ "images": {
+ "description": "Images.",
+ "type": "object",
+ "properties": {
+ "airflow": {
+ "description": "Configuration of the airflow image.",
+ "type": "object",
+ "properties": {
+ "repository": {
+ "description": "The airflow image repository.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "tag": {
+ "description": "The airflow image tag.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "pullPolicy": {
+ "description": "The airflow image pull policy.",
+ "type": "string"
+ }
+ }
+ },
+ "flower": {
+ "description": "Configuration of the flower image.",
+ "type": "object",
+ "properties": {
+ "repository": {
+ "description": "The flower image repository.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "tag": {
+ "description": "The flower image tag.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "pullPolicy": {
+ "description": "The flower image pull policy.",
+ "type": "string"
+ }
+ }
+ },
+ "statsd": {
+ "description": "Configuration of the statsd image.",
+ "type": "object",
+ "properties": {
+ "repository": {
+ "description": "The statsd image repository.",
+ "type": "string"
+ },
+ "tag": {
+ "description": "The statsd image tag.",
+ "type": "string"
+ },
+ "pullPolicy": {
+ "description": "The statsd image pull policy.",
+ "type": "string"
+ }
+ }
+ },
+ "redis": {
+ "description": "Configuration of the redis image.",
+ "type": "object",
+ "properties": {
+ "repository": {
+ "description": "The redis image repository.",
+ "type": "string"
+ },
+ "tag": {
+ "description": "The redis image tag.",
+ "type": "string"
+ },
+ "pullPolicy": {
+ "description": "The redis image pull policy.",
+ "type": "string"
+ }
+ }
+ },
+ "pgbouncer": {
+ "description": "Configuration of the pgbouncer image.",
+ "type": "object",
+ "properties": {
+ "repository": {
+ "description": "The pgbouncer image repository.",
+ "type": "string"
+ },
+ "tag": {
+ "description": "The pgbouncer image tag.",
+ "type": "string"
+ },
+ "pullPolicy": {
+ "description": "The pgbouncer image pull policy.",
+ "type": "string"
+ }
+ }
+ },
+ "pgbouncerExporter": {
+ "description": "Configuration of the pgbouncerExporter image.",
+ "type": "object",
+ "properties": {
+ "repository": {
+ "description": "The pgbouncerExporter image repository.",
+ "type": "string"
+ },
+ "tag": {
+ "description": "The pgbouncerExporter image tag.",
+ "type": "string"
+ },
+ "pullPolicy": {
+ "description": "The pgbouncerExporter image pull policy.",
+ "type": "string"
+ }
+ }
+ }
+ }
+ },
+ "env": {
+ "description": "Environment variables for all airflow containers.",
+ "type": "array"
+ },
+ "secret": {
+ "description": "Secrets for all airflow containers.",
+ "type": "array"
+ },
+ "data": {
+ "description": "Airflow database configuration.",
+ "type": "object",
+ "properties": {
+ "metadataSecretName": {
+ "description": "Metadata connection string secret.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "resultBackendSecretName": {
+ "description": "Result backend connection string secret.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "metadataConnection": {
+ "description": "Metadata connection configuration.",
+ "type": "object",
+ "properties": {
+ "user": {
+ "description": "The database user.",
+ "type": "string"
+ },
+ "pass": {
+ "description": "The user's password.",
+ "type": "string"
+ },
+ "host": {
+ "description": "The database host.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "port": {
+ "description": "The database port.",
+ "type": "integer"
+ },
+ "db": {
+ "description": "The name of the database.",
+ "type": "string"
+ },
+ "sslmode": {
+ "description": "The database SSL parameter.",
+ "type": "string"
+ }
+ }
+ },
+ "resultBackendConnection": {
+ "description": "Result backend connection configuration.",
+ "type": "object",
+ "properties": {
+ "user": {
+ "description": "The database user.",
+ "type": "string"
+ },
+ "pass": {
+ "description": "The database password.",
+ "type": "string"
+ },
+ "host": {
+ "description": "The database host.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "port": {
+ "description": "The database port.",
+ "type": "integer"
+ },
+ "db": {
+ "description": "The name of the database.",
+ "type": "string"
+ },
+ "sslmode": {
+ "description": "The database SSL parameter.",
+ "type": "string"
+ }
+ }
+ }
+ }
+ },
+ "fernetKey": {
+ "description": "The Fernet key used to encrypt passwords.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "fernetKeySecretName": {
+ "description": "The Fernet key secret name.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "workers": {
+ "description": "Airflow Worker configuration.",
+ "type": "object",
+ "properties": {
+ "replicas": {
+ "description": "Number of airflow celery workers in StatefulSet.",
+ "type": "integer"
+ },
+ "keda": {
+ "description": "KEDA configuration.",
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "description": "Allow KEDA autoscaling. `Persistence.enabled` must be set to false to use KEDA.",
+ "type": "boolean"
+ },
+ "namespaceLabels": {
+ "type": "object"
+ },
+ "pollingInterval": {
+ "description": "How often KEDA polls the airflow DB to report new scale requests to the HPA.",
+ "type": "integer"
+ },
+ "cooldownPeriod": {
+ "description": "How many seconds KEDA will wait before scaling to zero.",
+ "type": "integer"
+ },
+ "maxReplicaCount": {
+ "description": "Maximum number of workers created by KEDA.",
+ "type": "integer"
+ }
+ }
+ },
+ "persistence": {
+ "description": "Persistence configuration.",
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "description": "Enable persistent volumes.",
+ "type": "boolean"
+ },
+ "size": {
+ "description": "Volume size for worker StatefulSet.",
+ "type": "string"
+ },
+ "storageClassName": {
+ "description": "If using a custom storageClass, pass name ref to all StatefulSets here.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "fixPermissions": {
+ "description": "Execute init container to chown log directory. This is currently only needed in KinD, due to usage of local-path provisioner.",
+ "type": "boolean"
+ }
+ }
+ },
+ "resources": {
+ "type": "object"
+ },
+ "terminationGracePeriodSeconds": {
+ "description": "Grace period for tasks to finish after SIGTERM is sent from Kubernetes.",
+ "type": "integer"
+ },
+ "safeToEvict": {
+ "description": "This setting tells Kubernetes that it's ok to evict when it wants to scale a node down.",
+ "type": "boolean"
+ }
+ }
+ },
+ "scheduler": {
+ "description": "Airflow scheduler settings.",
+ "type": "object",
+ "properties": {
+ "podDisruptionBudget": {
+ "description": "Scheduler pod disruption budget.",
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "description": "Enable pod disruption budget.",
+ "type": "boolean"
+ },
+ "config": {
+ "description": "Disruption budget configuration.",
+ "type": "object",
+ "properties": {
+ "maxUnavailable": {
+ "description": "Max unavailable pods for scheduler.",
+ "type": "integer"
+ }
+ }
+ }
+ }
+ },
+ "resources": {
+ "type": "object"
+ },
+ "airflowLocalSettings": {
+ "description": "This setting can overwrite podMutation setting.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "safeToEvict": {
+ "description": "This setting tells Kubernetes that its ok to evict when it wants to scale a node down.",
+ "type": "boolean"
+ }
+ }
+ },
+ "webserver": {
+ "description": "Airflow webserver settings.",
+ "type": "object",
+ "properties": {
+ "livenessProbe": {
+ "description": "Liveness probe configuration.",
+ "type": "object",
+ "properties": {
+ "initialDelaySeconds": {
+ "description": "Webserver Liveness probe initial delay.",
+ "type": "integer"
+ },
+ "timeoutSeconds": {
+ "description": "Webserver Liveness probe timeout seconds.",
+ "type": "integer"
+ },
+ "failureThreshold": {
+ "description": "Webserver Liveness probe failure threshold.",
+ "type": "integer"
+ },
+ "periodSeconds": {
+ "description": "Webserver Liveness probe period seconds.",
+ "type": "integer"
+ }
+ }
+ },
+ "readinessProbe": {
+ "description": "Readiness probe configuration.",
+ "type": "object",
+ "properties": {
+ "initialDelaySeconds": {
+ "description": "Webserver Readiness probe initial delay.",
+ "type": "integer"
+ },
+ "timeoutSeconds": {
+ "description": "Webserver Readiness probe timeout seconds.",
+ "type": "integer"
+ },
+ "failureThreshold": {
+ "description": "Webserver Readiness probe failure threshold.",
+ "type": "integer"
+ },
+ "periodSeconds": {
+ "description": "Webserver Readiness probe period seconds.",
+ "type": "integer"
+ }
+ }
+ },
+ "replicas": {
+ "description": "How many Airflow webserver replicas should run.",
+ "type": "integer"
+ },
+ "extraNetworkPolicies": {
+ "description": "Additional network policies as needed.",
+ "type": "array"
+ },
+ "resources": {
+ "type": "object"
+ },
+ "defaultUser": {
+ "description": "Optional default airflow user information",
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "description": "Enable default user creation.",
+ "type": "boolean"
+ },
+ "role": {
+ "description": "Default user role.",
+ "type": "string"
+ },
+ "username": {
+ "description": "Default user username.",
+ "type": "string"
+ },
+ "email": {
+ "description": "Default user email address.",
+ "type": "string"
+ },
+ "firstName": {
+ "description": "Default user firstname.",
+ "type": "string"
+ },
+ "lastName": {
+ "description": "Default user lastname.",
+ "type": "string"
+ },
+ "password": {
+ "description": "Default user password.",
+ "type": "string"
+ }
+ }
+ },
+ "extraVolumes": {
+ "description": "Mount additional volumes into webserver.",
+ "type": "array"
+ },
+ "extraVolumeMounts": {
+ "description": "Mount additional volumes into webserver.",
+ "type": "array"
+ },
+ "webserverConfig": {
+ "description": "This will be mounted into the Airflow Webserver as a custom `webserver_config.py`. You can bake a `webserver_config.py` in to your image instead.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "service": {
+ "description": "Webserver service configuration.",
+ "type": "object",
+ "properties": {
+ "type": {
+ "description": "Webserver service type.",
+ "type": "string"
+ },
+ "annotations": {
+ "description": "Annotations for the webserver service.",
+ "type": "object"
+ }
+ }
+ }
+ }
+ },
+ "flower": {
+ "description": "Flower settings.",
+ "type": "object",
+ "properties": {
+ "extraNetworkPolicies": {
+ "description": "Additional network policies as needed.",
+ "type": "array"
+ },
+ "resources": {
+ "type": "object"
+ },
+ "service": {
+ "description": "Flower service configuration.",
+ "type": "object",
+ "properties": {
+ "type": {
+ "description": "Flower service type.",
+ "type": "string"
+ }
+ }
+ }
+ }
+ },
+ "statsd": {
+ "description": "Statsd settings.",
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "description": "Enable statsd.",
+ "type": "boolean"
+ },
+ "extraNetworkPolicies": {
+ "description": "Additional network policies as needed.",
+ "type": "array"
+ },
+ "resources": {
+ "type": "object"
+ },
+ "service": {
+ "description": "Statsd service configuration.",
+ "type": "object",
+ "properties": {
+ "extraAnnotations": {
+ "description": "Extra annotations for the statsd service.",
+ "type": "object"
+ }
+ }
+ }
+ }
+ },
+ "pgbouncer": {
+ "description": "Pgbouncer settings.",
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "description": "Enable pgbouncer.",
+ "type": "boolean"
+ },
+ "extraNetworkPolicies": {
+ "description": "Additional network policies as needed.",
+ "type": "array"
+ },
+ "metadataPoolSize": {
+ "description": "Metadata pool size.",
+ "type": "integer"
+ },
+ "resultBackendPoolSize": {
+ "description": "Result backend pool size.",
+ "type": "integer"
+ },
+ "maxClientConn": {
+ "description": "Maximum clients that can connect to pgbouncer (higher = more file descriptors).",
+ "type": "integer"
+ },
+ "podDisruptionBudget": {
+ "description": "Pgbouner pod disruption budget.",
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "description": "Enabled pod distribution budget.",
+ "type": "boolean"
+ },
+ "config": {
+ "description": "Pod distribution configuration.",
+ "type": "object",
+ "properties": {
+ "maxUnavailable": {
+ "description": "Max unavailable pods for pgbouncer.",
+ "type": "integer"
+ }
+ }
+ }
+ }
+ },
+ "resources": {
+ "type": "object"
+ },
+ "service": {
+ "description": "Pgbouncer service configuration.",
+ "type": "object",
+ "properties": {
+ "extraAnnotations": {
+ "description": "Extra annotations for the pgbouncer service.",
+ "type": "object"
+ }
+ }
+ },
+ "verbose": {
+ "description": "Increase pgbouncer verbosity.",
+ "type": "integer"
+ },
+ "logDisconnections": {
+ "description": "Log disconnections with reasons.",
+ "type": "integer"
+ },
+ "logConnections": {
+ "description": "Log successful logins.",
+ "type": "integer"
+ }
+ }
+ },
+ "redis": {
+ "description": "",
+ "type": "object",
+ "properties": {
+ "terminationGracePeriodSeconds": {
+ "description": "Grace period for tasks to finish after SIGTERM is sent from Kubernetes.",
+ "type": "integer"
+ },
+ "persistence": {
+ "description": "Persistence configuration.",
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "description": "Enable persistent volumes.",
+ "type": "boolean"
+ },
+ "size": {
+ "description": "Volume size for worker StatefulSet.",
+ "type": "string"
+ },
+ "storageClassName": {
+ "description": "If using a custom storageClass, pass name ref to all StatefulSets here.",
+ "type": [
+ "string",
+ "null"
+ ]
+ }
+ }
+ },
+ "resources": {
+ "type": "object"
+ },
+ "passwordSecretName": {
+ "description": "Redis password secret.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "brokerURLSecretName": {
+ "description": "Redis broker URL secret.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "password": {
+ "description": "If password is set, create secret with it, else generate a new one on install.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "safeToEvict": {
+ "description": "This setting tells Kubernetes that its ok to evict when it wants to scale a node down.",
+ "type": "boolean"
+ }
+ }
+ },
+ "registry": {
+ "description": "Auth secret for a private registry. This is used if pulling airflow images from a private registry.",
+ "type": "object",
+ "properties": {
+ "secretName": {
+ "description": "Registry connection string secret.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "connection": {
+ "description": "Registry connection configuration.",
+ "type": "object"
+ }
+ }
+ },
+ "elasticsearch": {
+ "description": "Elasticsearch logging configuration.",
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "description": "Enable elasticsearch task logging.",
+ "type": "boolean"
+ },
+ "secretName": {
+ "description": "A secret containing the connection string.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "connection": {
+ "description": "Elasticsearch connection configuration.",
+ "type": "object"
+ }
+ }
+ },
+ "ports": {
+ "description": "All ports used by chart.",
+ "type": "object",
+ "properties": {
+ "flowerUI": {
+ "description": "Flower UI port.",
+ "type": "integer"
+ },
+ "airflowUI": {
+ "description": "Airflow UI port.",
+ "type": "integer"
+ },
+ "workerLogs": {
+ "description": "Worker logs port.",
+ "type": "integer"
+ },
+ "redisDB": {
+ "description": "Redis port.",
+ "type": "integer"
+ },
+ "statsdIngest": {
+ "description": "Statsd ingest port.",
+ "type": "integer"
+ },
+ "statsdScrape": {
+ "description": "Statsd scrape port.",
+ "type": "integer"
+ },
+ "pgbouncer": {
+ "description": "Pgbouncer port.",
+ "type": "integer"
+ },
+ "pgbouncerScrape": {
+ "description": "Pgbouncer scrape port.",
+ "type": "integer"
+ }
+ }
+ },
+ "quotas": {
+ "description": "Define any ResourceQuotas for namespace.",
+ "type": "object"
+ },
+ "limits": {
+ "description": "Define default/max/min values for pods and containers in namespace.",
+ "type": "array"
+ },
+ "podMutation": {
+ "description": "Settings for pod_mutation_hook",
+ "type": "object",
+ "properties": {
+ "tolerations": {
+ "description": "Tolerations provided here would be applied using pod_mutation_hook. So any pods spun up using KubernetesExecutor or KubernetesPodOperator will contain these tolerations.",
+ "type": "array"
+ },
+ "affinity": {
+ "description": "Pods spun up would land in the node that matches the affinity.",
+ "type": "object"
+ }
+ }
+ },
+ "cleanup": {
+ "description": "This runs as a CronJob to cleanup old pods.",
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "description": "Enable cleanup.",
+ "type": "boolean"
+ },
+ "schedule": {
+ "description": "Cleanup schedule.",
+ "type": "string"
+ }
+ }
+ },
+ "postgresql": {
+ "description": "Configuration for postgresql subchart.",
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "description": "Enable postgresql subchart.",
+ "type": "boolean"
+ },
+ "postgresqlPassword": {
+ "description": "Postgresql password.",
+ "type": "string"
+ },
+ "postgresqlUsername": {
+ "description": "Postgresql username.",
+ "type": "string"
+ }
+ }
+ },
+ "config": {
+ "description": "Settings to go into the mounted airflow.cfg",
+ "type": "object",
+ "additionalProperties": {
+ "type": "object",
+ "additionalProperties": {
+ "type": [
+ "boolean",
+ "integer",
+ "number",
+ "string"
+ ]
+ }
+ }
+ },
+ "dags": {
+ "description": "DAGs settings.",
+ "type": "object",
+ "properties": {
+ "persistence": {
+ "description": "Persistence configuration.",
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "description": "Enable persistent volume for storing dags.",
+ "type": "boolean"
+ },
+ "size": {
+ "description": "Volume size for dags.",
+ "type": "string"
+ },
+ "storageClassName": {
+ "description": "If using a custom storageClass, pass name here.",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "accessMode": {
+ "description": "Access mode of the persistent volume.",
+ "type": "string"
+ },
+ "existingClaim": {
+ "description": "The name of an existing PVC to use.",
+ "type": [
+ "string",
+ "null"
+ ]
+ }
+ }
+ },
+ "gitSync": {
+ "description": "Git sync settings.",
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "description": "Enable Git sync.",
+ "type": "boolean"
+ },
+ "repo": {
+ "description": "Git repository.",
+ "type": "string"
+ },
+ "branch": {
+ "description": "Git branch",
+ "type": "string"
+ },
+ "rev": {
+ "description": "Git revision.",
+ "type": "string"
+ },
+ "root": {
+ "description": "Root directory.",
+ "type": "string"
+ },
+ "dest": {
+ "description": "Destination folder.",
+ "type": "string"
+ },
+ "depth": {
+ "description": "Repository depth.",
+ "type": "integer"
+ },
+ "maxFailures": {
+ "description": "The number of consecutive failures allowed before aborting.",
+ "type": "integer"
+ },
+ "subPath": {
+ "description": "Subpath within the repo where dags are located.",
+ "type": "string"
+ },
+ "wait": {
+ "description": "Interval between git sync attempts in seconds.",
+ "type": "integer"
+ },
+ "containerRepository": {
+ "description": "Git sync image repository.",
+ "type": "string"
+ },
+ "containerTag": {
+ "description": "Git sync image tag.",
+ "type": "string"
+ },
+ "containerName": {
+ "description": "Git sync container name.",
+ "type": "string"
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/chart/values.yaml b/chart/values.yaml
index 513dc47..f7c5e67 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -31,7 +31,7 @@ airflowHome: "/opt/airflow"
defaultAirflowRepository: apache/airflow
# Default airflow tag to deploy
-defaultAirflowTag: 1.10.10.1-alpha2-python3.6
+defaultAirflowTag: 1.10.12
# Select certain nodes for airflow pods.
@@ -42,6 +42,59 @@ tolerations: []
# Add common labels to all objects and pods defined in this chart.
labels: {}
+# Ingress configuration
+ingress:
+ # Enable ingress resource
+ enabled: false
+
+ # Configs for the Ingress of the web Service
+ web:
+ # Annotations for the web Ingress
+ annotations: {}
+
+ # The path for the web Ingress
+ path: ""
+
+ # The hostname for the web Ingress
+ host: ""
+
+ # configs for web Ingress TLS
+ tls:
+ # Enable TLS termination for the web Ingress
+ enabled: false
+ # the name of a pre-created Secret containing a TLS private key and certificate
+ secretName: ""
+
+ # HTTP paths to add to the web Ingress before the default path
+ precedingPaths: []
+
+ # Http paths to add to the web Ingress after the default path
+ succeedingPaths: []
+
+ # Configs for the Ingress of the flower Service
+ flower:
+ # Annotations for the flower Ingress
+ annotations: {}
+
+ # The path for the flower Ingress
+ path: ""
+
+ # The hostname for the flower Ingress
+ host: ""
+
+ # configs for web Ingress TLS
+ tls:
+ # Enable TLS termination for the flower Ingress
+ enabled: false
+ # the name of a pre-created Secret containing a TLS private key and certificate
+ secretName: ""
+
+ # HTTP paths to add to the flower Ingress before the default path
+ precedingPaths: []
+
+ # Http paths to add to the flower Ingress after the default path
+ succeedingPaths: []
+
# Network policy configuration
networkPolicies:
# Enabled network policies
@@ -196,7 +249,7 @@ workers:
pollingInterval: 5
# How many seconds KEDA will wait before scaling to zero.
- # Note that HPA has a seperate cooldwon period for scale-downs
+ # Note that HPA has a separate cooldown period for scale-downs
cooldownPeriod: 30
# Maximum number of workers created by keda
@@ -280,12 +333,12 @@ webserver:
extraNetworkPolicies: []
resources: {}
- # limits:
- # cpu: 100m
- # memory: 128Mi
- # requests:
- # cpu: 100m
- # memory: 128Mi
+ # limits:
+ # cpu: 100m
+ # memory: 128Mi
+ # requests:
+ # cpu: 100m
+ # memory: 128Mi
# Create initial user.
defaultUser:
@@ -316,18 +369,20 @@ webserver:
service:
type: ClusterIP
+ ## service annotations
+ annotations: {}
# Flower settings
flower:
# Additional network policies as needed
extraNetworkPolicies: []
resources: {}
- # limits:
- # cpu: 100m
- # memory: 128Mi
- # requests:
- # cpu: 100m
- # memory: 128Mi
+ # limits:
+ # cpu: 100m
+ # memory: 128Mi
+ # requests:
+ # cpu: 100m
+ # memory: 128Mi
service:
type: ClusterIP
@@ -338,12 +393,12 @@ statsd:
# Additional network policies as needed
extraNetworkPolicies: []
resources: {}
- # limits:
- # cpu: 100m
- # memory: 128Mi
- # requests:
- # cpu: 100m
- # memory: 128Mi
+ # limits:
+ # cpu: 100m
+ # memory: 128Mi
+ # requests:
+ # cpu: 100m
+ # memory: 128Mi
service:
extraAnnotations: {}
@@ -370,21 +425,13 @@ pgbouncer:
config:
maxUnavailable: 1
- # Limit the resources to pgbouncerExported.
- # When you specify the resource request the scheduler uses this information to decide which node to place
- # the Pod on. When you specify a resource limit for a Container, the kubelet enforces those limits so
- # that the running container is not allowed to use more of that resource than the limit you set.
- # See: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
- # Example:
- #
- # resource:
- # limits:
- # cpu: 100m
- # memory: 128Mi
- # requests:
- # cpu: 100m
- # memory: 128Mi
resources: {}
+ # limits:
+ # cpu: 100m
+ # memory: 128Mi
+ # requests:
+ # cpu: 100m
+ # memory: 128Mi
service:
extraAnnotations: {}
@@ -430,13 +477,11 @@ redis:
registry:
secretName: ~
- # Example:
- # connection:
- # user: ~
- # pass: ~
- # host: ~
- # email: ~
connection: {}
+ # user: ~
+ # pass: ~
+ # host: ~
+ # email: ~
# Elasticsearch logging configuration
elasticsearch:
@@ -445,14 +490,11 @@ elasticsearch:
# A secret containing the connection
secretName: ~
# Or an object representing the connection
- # Example:
- # connection:
- # user: ~
- # pass: ~
- # host: ~
- # port: ~
connection: {}
-
+ # user: ~
+ # pass: ~
+ # host: ~
+ # port: ~
# All ports used by chart
ports:
@@ -562,6 +604,7 @@ config:
namespace: '{{ .Release.Namespace }}'
airflow_configmap: '{{ include "airflow_config" . }}'
airflow_local_settings_configmap: '{{ include "airflow_config" . }}'
+ pod_template_file: '{{ include "airflow_pod_template_file" . }}/pod_template_file.yaml'
worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}'
worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}'
worker_container_image_pull_policy: '{{ .Values.images.airflow.pullPolicy }}'
@@ -569,6 +612,7 @@ config:
image_pull_secrets: '{{ template "registry_secret" . }}'
dags_in_image: '{{ ternary "False" "True" (or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled) }}'
delete_worker_pods: 'True'
+ multi_namespace_mode: '{{ .Values.multiNamespaceMode }}'
delete_worker_pods_on_failure: 'False'
run_as_user: '{{ .Values.uid }}'
fs_group: '{{ .Values.gid }}'
@@ -594,6 +638,10 @@ config:
AIRFLOW__CORE__FERNET_KEY: '{{ printf "%s=fernet-key" (include "fernet_key_secret" .) }}'
# yamllint enable rule:line-length
+multiNamespaceMode: 'False'
+
+podTemplate: ~
+
# Git sync
dags:
persistence:
@@ -635,7 +683,6 @@ dags:
# GIT_SYNC_USERNAME: <base64_encoded_git_username>
# GIT_SYNC_PASSWORD: <base64_encoded_git_password>
# and specify the name of the secret below
- #
# credentialsSecret: git-credentials
#
#
@@ -651,10 +698,8 @@ dags:
# gitSshKey: <base64_encoded_data>
# and specify the name of the secret below
# sshKeySecret: airflow-ssh-secret
- #
# If you are using an ssh private key, you can additionally
# specify the content of your known_hosts file, example:
- #
# knownHosts: |
# <host1>,<ip1> <key1>
# <host2>,<ip2> <key2>