You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by je...@apache.org on 2021/12/17 18:44:30 UTC
[airflow] branch main updated: Chart: Add support for securityContext (#18249)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e936e0f Chart: Add support for securityContext (#18249)
e936e0f is described below
commit e936e0f7ede58d46e47ced7b39cd79f2130fb73e
Author: Ney Walens De Mesquita <wa...@gmail.com>
AuthorDate: Fri Dec 17 19:43:55 2021 +0100
Chart: Add support for securityContext (#18249)
This adds the ability to set both a global `securityContext` and `securityContext` by deployment,
allowing for greater flexibility in configuring how Airflow is run.
---
chart/files/pod-template-file.kubernetes-helm-yaml | 5 +-
chart/templates/_helpers.yaml | 84 ++++++++-
chart/templates/cleanup/cleanup-cronjob.yaml | 2 +
chart/templates/flower/flower-deployment.yaml | 4 +-
chart/templates/jobs/create-user-job.yaml | 4 +-
chart/templates/jobs/migrate-database-job.yaml | 4 +-
.../templates/scheduler/scheduler-deployment.yaml | 5 +-
chart/templates/statsd/statsd-deployment.yaml | 4 +-
.../templates/triggerer/triggerer-deployment.yaml | 5 +-
.../templates/webserver/webserver-deployment.yaml | 5 +-
chart/templates/workers/worker-deployment.yaml | 7 +-
chart/tests/test_security_context.py | 200 +++++++++++++++++++++
chart/values.schema.json | 143 +++++++++++++++
chart/values.yaml | 65 +++++++
docs/helm-chart/production-guide.rst | 127 +++++++++++++
15 files changed, 638 insertions(+), 26 deletions(-)
diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml b/chart/files/pod-template-file.kubernetes-helm-yaml
index cc04d57..57386c6 100644
--- a/chart/files/pod-template-file.kubernetes-helm-yaml
+++ b/chart/files/pod-template-file.kubernetes-helm-yaml
@@ -18,6 +18,7 @@
{{- $nodeSelector := or .Values.nodeSelector .Values.workers.nodeSelector }}
{{- $affinity := or .Values.affinity .Values.workers.affinity }}
{{- $tolerations := or .Values.tolerations .Values.workers.tolerations }}
+{{- $securityContext := include "airflowSecurityContext" (list . .Values.workers) }}
apiVersion: v1
kind: Pod
metadata:
@@ -83,9 +84,7 @@ spec:
- name: {{ template "registry_secret" . }}
{{- end }}
restartPolicy: Never
- securityContext:
- runAsUser: {{ .Values.uid }}
- fsGroup: {{ .Values.gid }}
+ securityContext: {{ $securityContext | nindent 4 }}
nodeSelector: {{ toYaml $nodeSelector | nindent 4 }}
affinity: {{ toYaml $affinity | nindent 4 }}
tolerations: {{ toYaml $tolerations | nindent 4 }}
diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml
index 52eca78..424c6ab 100644
--- a/chart/templates/_helpers.yaml
+++ b/chart/templates/_helpers.yaml
@@ -143,8 +143,7 @@ If release name contains chart name it will be used as a full name.
- name: {{ .Values.dags.gitSync.containerName }}{{ if .is_init }}-init{{ end }}
image: {{ template "git_sync_image" . }}
imagePullPolicy: {{ .Values.images.gitSync.pullPolicy }}
- securityContext:
- runAsUser: {{ .Values.dags.gitSync.uid }}
+ securityContext: {{ include "localSecurityContext" .Values.dags.gitSync | nindent 4 }}
env:
{{- if .Values.dags.gitSync.sshKeySecret }}
- name: GIT_SSH_KEY_FILE
@@ -616,3 +615,84 @@ Create the name of the cleanup service account to use
{{- end -}}
{{- $kubeVersion -}}
{{- end -}}
+
+{{/*
+Set the default value for securityContext
+If no value is passed for securityContext or <node>.securityContext, defaults to global uid and gid.
+
+ +------------------------+ +-----------------+ +-------------------------+
+ | <node>.securityContext | -> | securityContext | -> | Values.uid + Values.gid |
+ +------------------------+ +-----------------+ +-------------------------+
+
+Values are not accumulated meaning that if runAsUser is set to 10 in <node>.securityContext,
+any extra values set to securityContext or uid+gid will be ignored.
+
+The template can be called like so:
+ include "airflowSecurityContext" (list . .Values.webserver)
+
+Where `.` is the global variables scope and `.Values.webserver` the local variables scope for the webserver template.
+*/}}
+{{- define "airflowSecurityContext" -}}
+ {{- $ := index . 0 -}}
+ {{- with index . 1 }}
+ {{- if .securityContext -}}
+{{ toYaml .securityContext | print }}
+ {{- else if $.Values.securityContext -}}
+{{ toYaml $.Values.securityContext | print }}
+ {{- else -}}
+runAsUser: {{ $.Values.uid }}
+fsGroup: {{ $.Values.gid }}
+ {{- end -}}
+ {{- end -}}
+{{- end -}}
+
+{{/*
+Set the default value for securityContext
+If no value is passed for securityContext or <node>.securityContext, defaults to UID in the local node.
+
+ +------------------------+ +-------------+
+ | <node>.securityContext | > | <node>.uid |
+ +------------------------+ +-------------+
+
+The template can be called like so:
+ include "localSecurityContext" .Values.statsd
+
+It is important to pass the local variables scope to this template as it is used to determine the local node value for uid.
+*/}}
+{{- define "localSecurityContext" -}}
+ {{- if .securityContext -}}
+{{ toYaml .securityContext | print }}
+ {{- else -}}
+runAsUser: {{ .uid }}
+ {{- end -}}
+{{- end -}}
+
+{{/*
+Set the default value for workers chown for persistent storage
+If no value is passed for securityContext or <node>.securityContext, defaults to global uid and gid.
+The template looks for `runAsUser` and `fsGroup` specifically, any other parameter will be ignored.
+
+ +------------------------+ +-----------------+ +-------------------------+
+ | <node>.securityContext | -> | securityContext | -> | Values.uid + Values.gid |
+ +------------------------+ +-----------------+ +-------------------------+
+
+Values are not accumulated meaning that if runAsUser is set to 10 in <node>.securityContext,
+any extra values set to securityContext or uid+gid will be ignored.
+
+The template can be called like so:
+ include "airflowSecurityContextIds" (list . .Values.workers)
+
+Where `.` is the global variables scope and `.Values.workers` the local variables scope for the workers template.
+*/}}
+{{- define "airflowSecurityContextIds" -}}
+ {{- $ := index . 0 -}}
+ {{- with index . 1 }}
+ {{- if .securityContext -}}
+{{ pluck "runAsUser" .securityContext | first | default $.Values.uid }}:{{ pluck "fsGroup" .securityContext | first | default $.Values.gid }}
+ {{- else if $.Values.securityContext -}}
+{{ pluck "runAsUser" $.Values.securityContext | first | default $.Values.uid }}:{{ pluck "fsGroup" $.Values.securityContext | first | default $.Values.gid }}
+ {{- else -}}
+{{ $.Values.uid }}:{{ $.Values.gid }}
+ {{- end -}}
+ {{- end -}}
+{{- end -}}
diff --git a/chart/templates/cleanup/cleanup-cronjob.yaml b/chart/templates/cleanup/cleanup-cronjob.yaml
index c9bbbfd..402932d 100644
--- a/chart/templates/cleanup/cleanup-cronjob.yaml
+++ b/chart/templates/cleanup/cleanup-cronjob.yaml
@@ -22,6 +22,7 @@
{{- $nodeSelector := or .Values.cleanup.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.cleanup.affinity .Values.affinity }}
{{- $tolerations := or .Values.cleanup.tolerations .Values.tolerations }}
+{{- $securityContext := include "airflowSecurityContext" (list . .Values.cleanup) }}
{{- if semverCompare ">= 1.21.x" (include "kubeVersion" .) }}
apiVersion: batch/v1
{{- else }}
@@ -70,6 +71,7 @@ spec:
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
+ securityContext: {{ $securityContext | nindent 12 }}
containers:
- name: airflow-cleanup-pods
image: {{ template "airflow_image" . }}
diff --git a/chart/templates/flower/flower-deployment.yaml b/chart/templates/flower/flower-deployment.yaml
index 2c40ee1..9e9244d 100644
--- a/chart/templates/flower/flower-deployment.yaml
+++ b/chart/templates/flower/flower-deployment.yaml
@@ -23,6 +23,7 @@
{{- $nodeSelector := or .Values.flower.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.flower.affinity .Values.affinity }}
{{- $tolerations := or .Values.flower.tolerations .Values.tolerations }}
+{{- $securityContext := include "airflowSecurityContext" (list . .Values.flower) }}
kind: Deployment
apiVersion: apps/v1
metadata:
@@ -67,8 +68,7 @@ spec:
{{ toYaml $tolerations | indent 8 }}
serviceAccountName: {{ include "flower.serviceAccountName" . }}
restartPolicy: Always
- securityContext:
- runAsUser: {{ .Values.uid }}
+ securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
diff --git a/chart/templates/jobs/create-user-job.yaml b/chart/templates/jobs/create-user-job.yaml
index a5e5df0..9a2582a 100644
--- a/chart/templates/jobs/create-user-job.yaml
+++ b/chart/templates/jobs/create-user-job.yaml
@@ -22,6 +22,7 @@
{{- $nodeSelector := or .Values.createUserJob.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.createUserJob.affinity .Values.affinity }}
{{- $tolerations := or .Values.createUserJob.tolerations .Values.tolerations }}
+{{- $securityContext := include "airflowSecurityContext" (list . .Values.createUserJob) }}
apiVersion: batch/v1
kind: Job
metadata:
@@ -65,8 +66,7 @@ spec:
{{- end }}
{{- end }}
spec:
- securityContext:
- runAsUser: {{ .Values.uid }}
+ securityContext: {{ $securityContext | nindent 8 }}
restartPolicy: OnFailure
nodeSelector:
{{ toYaml $nodeSelector | indent 8 }}
diff --git a/chart/templates/jobs/migrate-database-job.yaml b/chart/templates/jobs/migrate-database-job.yaml
index 859b045..0362f20 100644
--- a/chart/templates/jobs/migrate-database-job.yaml
+++ b/chart/templates/jobs/migrate-database-job.yaml
@@ -21,6 +21,7 @@
{{- $nodeSelector := or .Values.migrateDatabaseJob.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.migrateDatabaseJob.affinity .Values.affinity }}
{{- $tolerations := or .Values.migrateDatabaseJob.tolerations .Values.tolerations }}
+{{- $securityContext := include "airflowSecurityContext" (list . .Values.migrateDatabaseJob) }}
apiVersion: batch/v1
kind: Job
metadata:
@@ -64,8 +65,7 @@ spec:
{{- end }}
{{- end }}
spec:
- securityContext:
- runAsUser: {{ .Values.uid }}
+ securityContext: {{ $securityContext | nindent 8 }}
restartPolicy: OnFailure
nodeSelector:
{{ toYaml $nodeSelector | indent 8 }}
diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml
index a73c6b9..d7a5e1a 100644
--- a/chart/templates/scheduler/scheduler-deployment.yaml
+++ b/chart/templates/scheduler/scheduler-deployment.yaml
@@ -31,6 +31,7 @@
{{- $nodeSelector := or .Values.scheduler.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.scheduler.affinity .Values.affinity }}
{{- $tolerations := or .Values.scheduler.tolerations .Values.tolerations }}
+{{- $securityContext := include "airflowSecurityContext" (list . .Values.scheduler) }}
kind: {{ if $stateful }}StatefulSet{{ else }}Deployment{{ end }}
apiVersion: apps/v1
@@ -98,9 +99,7 @@ spec:
restartPolicy: Always
terminationGracePeriodSeconds: 10
serviceAccountName: {{ include "scheduler.serviceAccountName" . }}
- securityContext:
- runAsUser: {{ .Values.uid }}
- fsGroup: {{ .Values.gid }}
+ securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
diff --git a/chart/templates/statsd/statsd-deployment.yaml b/chart/templates/statsd/statsd-deployment.yaml
index e14e224..670dc53 100644
--- a/chart/templates/statsd/statsd-deployment.yaml
+++ b/chart/templates/statsd/statsd-deployment.yaml
@@ -22,6 +22,7 @@
{{- $nodeSelector := or .Values.statsd.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.statsd.affinity .Values.affinity }}
{{- $tolerations := or .Values.statsd.tolerations .Values.tolerations }}
+{{- $securityContext := include "localSecurityContext" .Values.statsd }}
kind: Deployment
apiVersion: apps/v1
metadata:
@@ -63,8 +64,7 @@ spec:
tolerations:
{{ toYaml $tolerations | indent 8 }}
serviceAccountName: {{ include "statsd.serviceAccountName" . }}
- securityContext:
- runAsUser: {{ .Values.statsd.uid }}
+ securityContext: {{ $securityContext | nindent 8 }}
restartPolicy: Always
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
diff --git a/chart/templates/triggerer/triggerer-deployment.yaml b/chart/templates/triggerer/triggerer-deployment.yaml
index 2591640..11ccdc6 100644
--- a/chart/templates/triggerer/triggerer-deployment.yaml
+++ b/chart/templates/triggerer/triggerer-deployment.yaml
@@ -23,6 +23,7 @@
{{- $nodeSelector := or .Values.nodeSelector .Values.triggerer.nodeSelector }}
{{- $affinity := or .Values.affinity .Values.triggerer.affinity }}
{{- $tolerations := or .Values.tolerations .Values.triggerer.tolerations }}
+{{- $securityContext := include "airflowSecurityContext" (list . .Values.triggerer) }}
kind: Deployment
apiVersion: apps/v1
metadata:
@@ -81,9 +82,7 @@ spec:
terminationGracePeriodSeconds: {{ .Values.triggerer.terminationGracePeriodSeconds }}
restartPolicy: Always
serviceAccountName: {{ include "triggerer.serviceAccountName" . }}
- securityContext:
- runAsUser: {{ .Values.uid }}
- fsGroup: {{ .Values.gid }}
+ securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml
index 4d76cd5..8aeb91b 100644
--- a/chart/templates/webserver/webserver-deployment.yaml
+++ b/chart/templates/webserver/webserver-deployment.yaml
@@ -21,6 +21,7 @@
{{- $nodeSelector := or .Values.webserver.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.webserver.affinity .Values.affinity }}
{{- $tolerations := or .Values.webserver.tolerations .Values.tolerations }}
+{{- $securityContext := include "airflowSecurityContext" (list . .Values.webserver) }}
kind: Deployment
apiVersion: apps/v1
metadata:
@@ -93,9 +94,7 @@ spec:
tolerations:
{{ toYaml $tolerations | indent 8 }}
restartPolicy: Always
- securityContext:
- runAsUser: {{ .Values.uid }}
- fsGroup: {{ .Values.gid }}
+ securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml
index e00ff7c..38c50ce 100644
--- a/chart/templates/workers/worker-deployment.yaml
+++ b/chart/templates/workers/worker-deployment.yaml
@@ -23,6 +23,7 @@
{{- $nodeSelector := or .Values.nodeSelector .Values.workers.nodeSelector }}
{{- $affinity := or .Values.affinity .Values.workers.affinity }}
{{- $tolerations := or .Values.tolerations .Values.workers.tolerations }}
+{{- $securityContext := include "airflowSecurityContext" (list . .Values.workers) }}
kind: {{ if $persistence }}StatefulSet{{ else }}Deployment{{ end }}
apiVersion: apps/v1
metadata:
@@ -95,9 +96,7 @@ spec:
terminationGracePeriodSeconds: {{ .Values.workers.terminationGracePeriodSeconds }}
restartPolicy: Always
serviceAccountName: {{ include "worker.serviceAccountName" . }}
- securityContext:
- runAsUser: {{ .Values.uid }}
- fsGroup: {{ .Values.gid }}
+ securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
@@ -112,7 +111,7 @@ spec:
command:
- chown
- -R
- - "{{ .Values.uid }}:{{ .Values.gid }}"
+ - "{{ include "airflowSecurityContextIds" (list . .Values.workers) }}"
- {{ template "airflow_logs" . }}
securityContext:
runAsUser: 0
diff --git a/chart/tests/test_security_context.py b/chart/tests/test_security_context.py
new file mode 100644
index 0000000..d40e856
--- /dev/null
+++ b/chart/tests/test_security_context.py
@@ -0,0 +1,200 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import jmespath
+
+from tests.helm_template_generator import render_chart
+
+
+class TestSCBackwardsCompatibility:
+ def test_check_deployments_and_jobs(self):
+ docs = render_chart(
+ values={
+ "uid": 3000,
+ "gid": 30,
+ "webserver": {"defaultUser": {"enabled": True}},
+ "flower": {"enabled": True},
+ "airflowVersion": "2.2.0",
+ "executor": "CeleryKubernetesExecutor",
+ },
+ show_only=[
+ "templates/flower/flower-deployment.yaml",
+ "templates/scheduler/scheduler-deployment.yaml",
+ "templates/triggerer/triggerer-deployment.yaml",
+ "templates/webserver/webserver-deployment.yaml",
+ "templates/workers/worker-deployment.yaml",
+ "templates/jobs/create-user-job.yaml",
+ "templates/jobs/migrate-database-job.yaml",
+ ],
+ )
+
+ for index in range(len(docs)):
+ assert 3000 == jmespath.search("spec.template.spec.securityContext.runAsUser", docs[index])
+ assert 30 == jmespath.search("spec.template.spec.securityContext.fsGroup", docs[index])
+
+ def test_check_statsd_uid(self):
+ docs = render_chart(
+ values={"statsd": {"enabled": True, "uid": 3000}},
+ show_only=["templates/statsd/statsd-deployment.yaml"],
+ )
+
+ assert 3000 == jmespath.search("spec.template.spec.securityContext.runAsUser", docs[0])
+
+ def test_check_cleanup_job(self):
+ docs = render_chart(
+ values={"uid": 3000, "gid": 30, "cleanup": {"enabled": True}},
+ show_only=["templates/cleanup/cleanup-cronjob.yaml"],
+ )
+
+ assert 3000 == jmespath.search(
+ "spec.jobTemplate.spec.template.spec.securityContext.runAsUser", docs[0]
+ )
+ assert 30 == jmespath.search("spec.jobTemplate.spec.template.spec.securityContext.fsGroup", docs[0])
+
+ def test_gitsync_sidecar_and_init_container(self):
+ docs = render_chart(
+ values={
+ "dags": {"gitSync": {"enabled": True, "uid": 3000}},
+ "airflowVersion": "1.10.15",
+ },
+ show_only=[
+ "templates/workers/worker-deployment.yaml",
+ "templates/webserver/webserver-deployment.yaml",
+ "templates/scheduler/scheduler-deployment.yaml",
+ ],
+ )
+
+ for index in range(len(docs)):
+ assert "git-sync" in [
+ c["name"] for c in jmespath.search("spec.template.spec.containers", docs[index])
+ ]
+ assert "git-sync-init" in [
+ c["name"] for c in jmespath.search("spec.template.spec.initContainers", docs[index])
+ ]
+ assert 3000 == jmespath.search(
+ "spec.template.spec.initContainers[?name=='git-sync-init'].securityContext.runAsUser | [0]",
+ docs[index],
+ )
+ assert 3000 == jmespath.search(
+ "spec.template.spec.containers[?name=='git-sync'].securityContext.runAsUser | [0]",
+ docs[index],
+ )
+
+
+class TestSecurityContext:
+ # Test securityContext setting for Pods and Containers
+ def test_check_default_setting(self):
+ docs = render_chart(
+ values={
+ "securityContext": {"runAsUser": 6000, "fsGroup": 60},
+ "webserver": {"defaultUser": {"enabled": True}},
+ "flower": {"enabled": True},
+ "statsd": {"enabled": False},
+ "airflowVersion": "2.2.0",
+ "executor": "CeleryKubernetesExecutor",
+ },
+ show_only=[
+ "templates/flower/flower-deployment.yaml",
+ "templates/scheduler/scheduler-deployment.yaml",
+ "templates/triggerer/triggerer-deployment.yaml",
+ "templates/webserver/webserver-deployment.yaml",
+ "templates/workers/worker-deployment.yaml",
+ "templates/jobs/create-user-job.yaml",
+ "templates/jobs/migrate-database-job.yaml",
+ ],
+ )
+
+ for index in range(len(docs)):
+ print(docs[index])
+ assert 6000 == jmespath.search("spec.template.spec.securityContext.runAsUser", docs[index])
+ assert 60 == jmespath.search("spec.template.spec.securityContext.fsGroup", docs[index])
+
+ # Test priority:
+ # <local>.securityContext > securityContext > uid + gid
+ def test_check_local_setting(self):
+ component_contexts = {"securityContext": {"runAsUser": 9000, "fsGroup": 90}}
+ docs = render_chart(
+ values={
+ "uid": 3000,
+ "gid": 30,
+ "securityContext": {"runAsUser": 6000, "fsGroup": 60},
+ "webserver": {"defaultUser": {"enabled": True}, **component_contexts},
+ "workers": {**component_contexts},
+ "flower": {"enabled": True, **component_contexts},
+ "scheduler": {**component_contexts},
+ "createUserJob": {**component_contexts},
+ "migrateDatabaseJob": {**component_contexts},
+ "triggerer": {**component_contexts},
+ "statsd": {"enabled": True, **component_contexts},
+ "airflowVersion": "2.2.0",
+ "executor": "CeleryKubernetesExecutor",
+ },
+ show_only=[
+ "templates/flower/flower-deployment.yaml",
+ "templates/scheduler/scheduler-deployment.yaml",
+ "templates/triggerer/triggerer-deployment.yaml",
+ "templates/webserver/webserver-deployment.yaml",
+ "templates/workers/worker-deployment.yaml",
+ "templates/jobs/create-user-job.yaml",
+ "templates/jobs/migrate-database-job.yaml",
+ "templates/statsd/statsd-deployment.yaml",
+ ],
+ )
+
+ for index in range(len(docs)):
+ print(docs[index])
+ assert 9000 == jmespath.search("spec.template.spec.securityContext.runAsUser", docs[index])
+ assert 90 == jmespath.search("spec.template.spec.securityContext.fsGroup", docs[index])
+
+ # Test containerSecurity priority over uid under statsd
+ def test_check_statsd_uid(self):
+ docs = render_chart(
+ values={"statsd": {"enabled": True, "uid": 3000, "securityContext": {"runAsUser": 7000}}},
+ show_only=["templates/statsd/statsd-deployment.yaml"],
+ )
+
+ assert 7000 == jmespath.search("spec.template.spec.securityContext.runAsUser", docs[0])
+
+ # Test containerSecurity priority over uid under dags.gitSync
+ def test_gitsync_sidecar_and_init_container(self):
+ docs = render_chart(
+ values={
+ "dags": {"gitSync": {"enabled": True, "uid": 9000, "securityContext": {"runAsUser": 8000}}},
+ "airflowVersion": "1.10.15",
+ },
+ show_only=[
+ "templates/workers/worker-deployment.yaml",
+ "templates/webserver/webserver-deployment.yaml",
+ "templates/scheduler/scheduler-deployment.yaml",
+ ],
+ )
+
+ for index in range(len(docs)):
+ assert "git-sync" in [
+ c["name"] for c in jmespath.search("spec.template.spec.containers", docs[index])
+ ]
+ assert "git-sync-init" in [
+ c["name"] for c in jmespath.search("spec.template.spec.initContainers", docs[index])
+ ]
+ assert 8000 == jmespath.search(
+ "spec.template.spec.initContainers[?name=='git-sync-init'].securityContext.runAsUser | [0]",
+ docs[index],
+ )
+ assert 8000 == jmespath.search(
+ "spec.template.spec.containers[?name=='git-sync'].securityContext.runAsUser | [0]",
+ docs[index],
+ )
diff --git a/chart/values.schema.json b/chart/values.schema.json
index aefa363..72f37f2 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -76,6 +76,20 @@
"default": "2.2.2",
"x-docsSection": "Common"
},
+ "securityContext": {
+ "description": "Pod security context definition. The values in this parameter will be used when `securityContext` is not defined for specific Pods",
+ "type": "object",
+ "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.22.0/_definitions.json#/definitions/io.k8s.api.core.v1.PodSecurityContext",
+ "default": {},
+ "x-docsSection": "Kubernetes",
+ "examples": [
+ {
+ "runAsUser": 50000,
+ "runAsGroup": 0,
+ "fsGroup": 0
+ }
+ ]
+ },
"nodeSelector": {
"description": "Select certain nodes for all pods.",
"type": "object",
@@ -1379,6 +1393,19 @@
"$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.22.0-standalone-strict/resourcerequirements-v1.json"
}
}
+ },
+ "securityContext": {
+ "description": "Security context for the worker pod. If not set, the values from `securityContext` will be used.",
+ "type": "object",
+ "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.22.0/_definitions.json#/definitions/io.k8s.api.core.v1.PodSecurityContext",
+ "default": {},
+ "examples": [
+ {
+ "runAsUser": 50000,
+ "runAsGroup": 0,
+ "fsGroup": 0
+ }
+ ]
}
}
},
@@ -1657,6 +1684,19 @@
"$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.22.0-standalone-strict/resourcerequirements-v1.json"
}
}
+ },
+ "securityContext": {
+ "description": "Security context for the scheduler pod. If not set, the values from `securityContext` will be used.",
+ "type": "object",
+ "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.22.0/_definitions.json#/definitions/io.k8s.api.core.v1.PodSecurityContext",
+ "default": {},
+ "examples": [
+ {
+ "runAsUser": 50000,
+ "runAsGroup": 0,
+ "fsGroup": 0
+ }
+ ]
}
}
},
@@ -1860,6 +1900,19 @@
"additionalProperties": {
"type": "string"
}
+ },
+ "securityContext": {
+ "description": "Security context for the triggerer pod. If not set, the values from `securityContext` will be used.",
+ "type": "object",
+ "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.22.0/_definitions.json#/definitions/io.k8s.api.core.v1.PodSecurityContext",
+ "default": {},
+ "examples": [
+ {
+ "runAsUser": 50000,
+ "runAsGroup": 0,
+ "fsGroup": 0
+ }
+ ]
}
}
},
@@ -1960,6 +2013,19 @@
"$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.22.0-standalone-strict/toleration-v1.json"
}
},
+ "securityContext": {
+ "description": "Security context for the create user job pod. If not set, the values from `securityContext` will be used.",
+ "type": "object",
+ "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.22.0/_definitions.json#/definitions/io.k8s.api.core.v1.PodSecurityContext",
+ "default": {},
+ "examples": [
+ {
+ "runAsUser": 50000,
+ "runAsGroup": 0,
+ "fsGroup": 0
+ }
+ ]
+ },
"resources": {
"description": "Resources for the create user job pod",
"type": "object",
@@ -2099,6 +2165,19 @@
"$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.22.0-standalone-strict/toleration-v1.json"
}
},
+ "securityContext": {
+ "description": "Security context for the migrate database job pod. If not set, the values from `securityContext` will be used.",
+ "type": "object",
+ "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.22.0/_definitions.json#/definitions/io.k8s.api.core.v1.PodSecurityContext",
+ "default": {},
+ "examples": [
+ {
+ "runAsUser": 50000,
+ "runAsGroup": 0,
+ "fsGroup": 0
+ }
+ ]
+ },
"useHelmHooks": {
"description": "Specify if you want to use the default Helm Hook annotations",
"type": "boolean",
@@ -2285,6 +2364,19 @@
}
}
},
+ "securityContext": {
+ "description": "Security context for the webserver job pod. If not set, the values from `securityContext` will be used.",
+ "type": "object",
+ "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.22.0/_definitions.json#/definitions/io.k8s.api.core.v1.PodSecurityContext",
+ "default": {},
+ "examples": [
+ {
+ "runAsUser": 50000,
+ "runAsGroup": 0,
+ "fsGroup": 0
+ }
+ ]
+ },
"resources": {
"description": "Resources for webserver pods.",
"type": "object",
@@ -2788,6 +2880,19 @@
"additionalProperties": {
"type": "string"
}
+ },
+ "securityContext": {
+ "description": "Security context for the flower pod. If not set, the values from `securityContext` will be used.",
+ "type": "object",
+ "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.22.0/_definitions.json#/definitions/io.k8s.api.core.v1.PodSecurityContext",
+ "default": {},
+ "examples": [
+ {
+ "runAsUser": 50000,
+ "runAsGroup": 0,
+ "fsGroup": 0
+ }
+ ]
}
}
},
@@ -2903,6 +3008,19 @@
"description": "Additional mappings for StatsD exporter.",
"type": "array",
"default": []
+ },
+ "securityContext": {
+ "description": "Security context for the statsd pod. If not set, the values from `securityContext` will be used.",
+ "type": "object",
+ "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.22.0/_definitions.json#/definitions/io.k8s.api.core.v1.PodSecurityContext",
+ "default": {},
+ "examples": [
+ {
+ "runAsUser": 50000,
+ "runAsGroup": 0,
+ "fsGroup": 0
+ }
+ ]
}
}
},
@@ -3654,6 +3772,19 @@
}
}
}
+ },
+ "securityContext": {
+ "description": "Security context for the cleanup job pod. If not set, the values from `securityContext` will be used.",
+ "type": "object",
+ "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.22.0/_definitions.json#/definitions/io.k8s.api.core.v1.PodSecurityContext",
+ "default": {},
+ "examples": [
+ {
+ "runAsUser": 50000,
+ "runAsGroup": 0,
+ "fsGroup": 0
+ }
+ ]
}
}
},
@@ -3813,6 +3944,18 @@
"type": "string",
"default": "git-sync"
},
+ "securityContext": {
+ "description": "Security context for the gitSync container. If not set, the values from `securityContext` will be used.",
+ "type": "object",
+ "$ref": "https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/v1.22.0/_definitions.json#/definitions/io.k8s.api.core.v1.SecurityContext",
+ "default": {},
+ "examples": [
+ {
+ "runAsUser": 50000,
+ "runAsGroup": 0
+ }
+ ]
+ },
"uid": {
"description": "Git sync container run as user parameter.",
"type": "integer",
diff --git a/chart/values.yaml b/chart/values.yaml
index 6681b56..e4880ea 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -32,6 +32,12 @@ kubeVersionOverride: ""
uid: 50000
gid: 0
+# Default security context for airflow
+securityContext: {}
+# runAsUser: 50000
+# fsGroup: 0
+# runAsGroup: 0
+
# Airflow home directory
# Used for mount paths
airflowHome: /opt/airflow
@@ -405,6 +411,12 @@ workers:
maxSurge: "100%"
maxUnavailable: "50%"
+ # When not set, the values defined in the global securityContext will be used
+ securityContext: {}
+ # runAsUser: 50000
+ # fsGroup: 0
+ # runAsGroup: 0
+
# Create ServiceAccount
serviceAccount:
# Specifies whether a ServiceAccount should be created
@@ -549,6 +561,12 @@ scheduler:
# (when not using LocalExecutor and workers.persistence)
strategy: ~
+ # When not set, the values defined in the global securityContext will be used
+ securityContext: {}
+ # runAsUser: 50000
+ # fsGroup: 0
+ # runAsGroup: 0
+
# Create ServiceAccount
serviceAccount:
# Specifies whether a ServiceAccount should be created
@@ -628,6 +646,12 @@ createUserJob:
# jobAnnotations are annotations on the create user job
jobAnnotations: {}
+ # When not set, the values defined in the global securityContext will be used
+ securityContext: {}
+ # runAsUser: 50000
+ # fsGroup: 0
+ # runAsGroup: 0
+
# Create ServiceAccount
serviceAccount:
# Specifies whether a ServiceAccount should be created
@@ -668,6 +692,12 @@ migrateDatabaseJob:
# jobAnnotations are annotations on the database migration job
jobAnnotations: {}
+ # When not set, the values defined in the global securityContext will be used
+ securityContext: {}
+ # runAsUser: 50000
+ # fsGroup: 0
+ # runAsGroup: 0
+
# Create ServiceAccount
serviceAccount:
# Specifies whether a ServiceAccount should be created
@@ -738,6 +768,12 @@ webserver:
# Allow overriding Update Strategy for Webserver
strategy: ~
+ # When not set, the values defined in the global securityContext will be used
+ securityContext: {}
+ # runAsUser: 50000
+ # fsGroup: 0
+ # runAsGroup: 0
+
# Additional network policies as needed (Deprecated - renamed to `webserver.networkPolicy.ingress.from`)
extraNetworkPolicies: []
networkPolicy:
@@ -861,6 +897,12 @@ triggerer:
# Annotations to add to triggerer kubernetes service account.
annotations: {}
+ # When not set, the values defined in the global securityContext will be used
+ securityContext: {}
+ # runAsUser: 50000
+ # fsGroup: 0
+ # runAsGroup: 0
+
resources: {}
# limits:
# cpu: 100m
@@ -935,6 +977,12 @@ flower:
# cpu: 100m
# memory: 128Mi
+ # When not set, the values defined in the global securityContext will be used
+ securityContext: {}
+ # runAsUser: 50000
+ # fsGroup: 0
+ # runAsGroup: 0
+
# Create ServiceAccount
serviceAccount:
# Specifies whether a ServiceAccount should be created
@@ -998,6 +1046,12 @@ statsd:
# Annotations to add to worker kubernetes service account.
annotations: {}
+ # When not set, the values defined in the global securityContext will be used
+ securityContext: {}
+ # runAsUser: 65534
+ # fsGroup: 0
+ # runAsGroup: 0
+
# Additional network policies as needed
extraNetworkPolicies: []
resources: {}
@@ -1263,6 +1317,11 @@ cleanup:
# Annotations to add to cleanup cronjob kubernetes service account.
annotations: {}
+ # When not set, the values defined in the global securityContext will be used
+ securityContext: {}
+ # runAsUser: 50000
+ # runAsGroup: 0
+
# Configuration for postgresql subchart
# Not recommended for production
postgresql:
@@ -1434,6 +1493,12 @@ dags:
wait: 60
containerName: git-sync
uid: 65533
+
+ # When not set, the values defined in the global securityContext will be used
+ securityContext: {}
+ # runAsUser: 65533
+ # runAsGroup: 0
+
extraVolumeMounts: []
env: []
resources: {}
diff --git a/docs/helm-chart/production-guide.rst b/docs/helm-chart/production-guide.rst
index fbb70d3..04149e7 100644
--- a/docs/helm-chart/production-guide.rst
+++ b/docs/helm-chart/production-guide.rst
@@ -283,3 +283,130 @@ In order to enable the usage of SCCs, one must set the parameter :ref:`rbac.crea
In this chart, SCCs are bound to the Pods via RoleBindings meaning that the option ``rbac.create`` must also be set to ``true`` in order to fully enable the SCC usage.
For more information about SCCs and what can be achieved with this construct, please refer to `Managing security context constraints <https://docs.openshift.com/container-platform/latest/authentication/managing-security-context-constraints.html#scc-prioritization_configuring-internal-oauth/>`_.
+
+Security Context
+----------------
+
+In Kubernetes a ``securityContext`` can be used to define user ids, group ids and capabilities such as running a container in privileged mode.
+
+When deploying an application to Kubernetes, it is recommended to give the least privilege to containers so as
+to reduce access and protect the host where the container is running.
+
+In the Airflow Helm chart, the ``securityContext`` can be configured in several ways:
+
+ * :ref:`uid <parameters:Airflow>` (configures the global uid or RunAsUser)
+ * :ref:`gid <parameters:Airflow>` (configures the global gid or fsGroup)
+ * :ref:`securityContext <parameters:Kubernetes>` (same as ``uid`` but allows for setting all `Pod securityContext options <https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#podsecuritycontext-v1-core>`_)
+
+The same way one can configure the global :ref:`securityContext <parameters:Kubernetes>`, it is also possible to configure different values for specific workloads by setting their local ``securityContext`` as follows:
+
+.. code-block:: yaml
+
+ workers:
+ securityContext:
+ runAsUser: 5000
+ fsGroup: 0
+
+In the example above, the workers Pod ``securityContext`` will be set to ``runAsUser: 5000`` and ``runAsGroup: 0``.
+
+As one can see, the local setting will take precedence over the global setting when defined. The following explains the precedence rule for ``securityContext`` options in this chart:
+
+.. code-block:: yaml
+
+ uid: 40000
+ gid: 0
+
+ securityContext:
+ runAsUser: 50000
+ fsGroup: 0
+
+ workers:
+ securityContext:
+ runAsUser: 1001
+ fsGroup: 0
+
+This will generate the following worker deployment:
+
+.. code-block:: yaml
+
+ kind: StatefulSet
+ apiVersion: apps/v1
+ metadata:
+ name: airflow-worker
+ spec:
+ serviceName: airflow-worker
+ template:
+ spec:
+ securityContext: # As the securityContext was defined in ``workers``, its value will take priority
+ runAsUser: 1001
+ fsGroup: 0
+
+If we remove both the ``securityContext`` and ``workers.securityContext`` from the example above, the output will be the following:
+
+.. code-block:: yaml
+
+ uid: 40000
+ gid: 0
+
+ securityContext: {}
+
+ workers:
+ securityContext: {}
+
+This will generate the following worker deployment:
+
+.. code-block:: yaml
+
+ kind: StatefulSet
+ apiVersion: apps/v1
+ metadata:
+ name: airflow-worker
+ spec:
+ serviceName: airflow-worker
+ template:
+ spec:
+ securityContext:
+ runAsUser: 40000 # As the securityContext was not defined in ``workers`` or ``podSecurity``, the value from uid will be used
+ fsGroup: 0 # As the securityContext was not defined in ``workers`` or ``podSecurity``, the value from gid will be used
+ initContainers:
+ - name: wait-for-airflow-migrations
+ ...
+ containers:
+ - name: worker
+ ...
+
+And finally if we set ``securityContext`` but not ``workers.securityContext``:
+
+.. code-block:: yaml
+
+ uid: 40000
+ gid: 0
+
+ securityContext:
+ runAsUser: 50000
+ fsGroup: 0
+
+ workers:
+ securityContext: {}
+
+This will generate the following worker deployment:
+
+.. code-block:: yaml
+
+ kind: StatefulSet
+ apiVersion: apps/v1
+ metadata:
+ name: airflow-worker
+ spec:
+ serviceName: airflow-worker
+ template:
+ spec:
+ securityContext: # As the securityContext was not defined in ``workers``, the values from securityContext will take priority
+ runAsUser: 50000
+ fsGroup: 0
+ initContainers:
+ - name: wait-for-airflow-migrations
+ ...
+ containers:
+ - name: worker
+ ...