You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/12/23 19:04:11 UTC
[airflow] branch v1-10-test updated (70036e7 -> 05b6464)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git.
from 70036e7 Default python version is used when building image
new 21ac6c1 fixup! Include airflow/contrib/executors in the dist package
new bb5c8e5 The default value in chart should be 2.0.0 (#13125)
new 1fd654a Use new logging options on values.yaml (#13173)
new 74ff3e8 Fix parenthesis preventing Keda ScaledObject creation (#13183)
new 9fefd7c Update chart readme to remove astronomer references (#13210)
new 05b6464 Allow webserver to read pod logs directly (#12598)
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
chart/README.md | 39 +++++++++------
...launcher-role.yaml => pod-log-reader-role.yaml} | 16 ++----
...inding.yaml => pod-log-reader-rolebinding.yaml} | 21 ++++++--
chart/templates/workers/worker-kedaautoscaler.yaml | 2 +-
chart/tests/helm_template_generator.py | 7 +--
chart/tests/test_basic_helm_chart.py | 4 +-
chart/tests/test_keda.py | 57 ++++++++++++++++++++++
chart/values.schema.json | 4 ++
chart/values.yaml | 13 +++--
docs/conf.py | 1 +
10 files changed, 118 insertions(+), 46 deletions(-)
copy chart/templates/rbac/{pod-launcher-role.yaml => pod-log-reader-role.yaml} (84%)
copy chart/templates/rbac/{pod-cleanup-rolebinding.yaml => pod-log-reader-rolebinding.yaml} (71%)
create mode 100644 chart/tests/test_keda.py
[airflow] 02/06: The default value in chart should be 2.0.0 (#13125)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit bb5c8e5a9bffe25efd5b0b75efcfe71d952cf937
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Fri Dec 18 11:50:23 2020 +0100
The default value in chart should be 2.0.0 (#13125)
(cherry picked from commit f5c4b2442d096ccb873d18f50bc2c5d89d780b03)
---
chart/values.yaml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/chart/values.yaml b/chart/values.yaml
index 091a0c9..cb605c3 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -31,7 +31,7 @@ airflowHome: "/opt/airflow"
defaultAirflowRepository: apache/airflow
# Default airflow tag to deploy
-defaultAirflowTag: 1.10.12
+defaultAirflowTag: 2.0.0
# Select certain nodes for airflow pods.
[airflow] 04/06: Fix parenthesis preventing Keda ScaledObject
creation (#13183)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 74ff3e81002c2f8d57239b1a8be06a6dbbfe425b
Author: dstandish <ds...@users.noreply.github.com>
AuthorDate: Mon Dec 21 02:19:26 2020 -0800
Fix parenthesis preventing Keda ScaledObject creation (#13183)
(cherry picked from commit a9d562e1c3c16c98750c9e3be74347f882acb97a)
---
chart/templates/workers/worker-kedaautoscaler.yaml | 2 +-
chart/tests/helm_template_generator.py | 7 +--
chart/tests/test_keda.py | 57 ++++++++++++++++++++++
3 files changed, 62 insertions(+), 4 deletions(-)
diff --git a/chart/templates/workers/worker-kedaautoscaler.yaml b/chart/templates/workers/worker-kedaautoscaler.yaml
index e135076..1493133 100644
--- a/chart/templates/workers/worker-kedaautoscaler.yaml
+++ b/chart/templates/workers/worker-kedaautoscaler.yaml
@@ -18,7 +18,7 @@
################################
## Airflow Worker KEDA Scaler
#################################
-{{- if (and .Values.workers.keda.enabled ( or (eq .Values.executor "CeleryExecutor")) (eq .Values.executor "CeleryKubernetesExecutor")) }}
+{{- if (and .Values.workers.keda.enabled ( or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor"))) }}
apiVersion: keda.k8s.io/v1alpha1
kind: ScaledObject
metadata:
diff --git a/chart/tests/helm_template_generator.py b/chart/tests/helm_template_generator.py
index d8e3f49..8b9fdb2 100644
--- a/chart/tests/helm_template_generator.py
+++ b/chart/tests/helm_template_generator.py
@@ -61,7 +61,7 @@ def validate_k8s_object(instance):
validate.validate(instance)
-def render_chart(name="RELEASE-NAME", values=None, show_only=None):
+def render_chart(name="RELEASE-NAME", values=None, show_only=None, validate_schema=True):
"""
Function that renders a helm chart into dictionaries. For helm chart testing only
"""
@@ -77,8 +77,9 @@ def render_chart(name="RELEASE-NAME", values=None, show_only=None):
templates = subprocess.check_output(command)
k8s_objects = yaml.load_all(templates)
k8s_objects = [k8s_object for k8s_object in k8s_objects if k8s_object] # type: ignore
- for k8s_object in k8s_objects:
- validate_k8s_object(k8s_object)
+ if validate_schema:
+ for k8s_object in k8s_objects:
+ validate_k8s_object(k8s_object)
return k8s_objects
diff --git a/chart/tests/test_keda.py b/chart/tests/test_keda.py
new file mode 100644
index 0000000..57da31a
--- /dev/null
+++ b/chart/tests/test_keda.py
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import unittest
+
+import jmespath
+from parameterized import parameterized
+
+from tests.helm_template_generator import render_chart
+
+
+class KedaTest(unittest.TestCase):
+ def test_keda_disabled_by_default(self):
+ """disabled by default"""
+ docs = render_chart(
+ values={},
+ show_only=["templates/workers/worker-kedaautoscaler.yaml"],
+ validate_schema=False,
+ )
+ self.assertListEqual(docs, [])
+
+ @parameterized.expand(
+ [
+ ('SequentialExecutor', False),
+ ('CeleryExecutor', True),
+ ('CeleryKubernetesExecutor', True),
+ ]
+ )
+ def test_keda_enabled(self, executor, is_created):
+ """
+ ScaledObject should only be created when set to enabled and executor is Celery or CeleryKubernetes
+ """
+ docs = render_chart(
+ values={
+ "workers": {"keda": {"enabled": True}, "persistence": {"enabled": False}},
+ 'executor': executor,
+ },
+ show_only=["templates/workers/worker-kedaautoscaler.yaml"],
+ validate_schema=False,
+ )
+ if is_created:
+ self.assertEqual("RELEASE-NAME-worker", jmespath.search("metadata.name", docs[0]))
+ else:
+ self.assertListEqual(docs, [])
[airflow] 01/06: fixup! Include airflow/contrib/executors in the
dist package
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 21ac6c18eb3e7a563914f18b00a417fc96fb6541
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Wed Dec 23 19:50:02 2020 +0100
fixup! Include airflow/contrib/executors in the dist package
---
docs/conf.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/docs/conf.py b/docs/conf.py
index d70dd69..4d93634 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -192,6 +192,7 @@ exclude_patterns = [
'_api/airflow/configuration',
'_api/airflow/contrib/auth',
'_api/airflow/contrib/example_dags',
+ '_api/airflow/contrib/executors/index.rst',
'_api/airflow/contrib/index.rst',
'_api/airflow/contrib/kubernetes',
'_api/airflow/contrib/task_runner',
[airflow] 05/06: Update chart readme to remove astronomer
references (#13210)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 9fefd7c3e6998c4ec5f7e13549f130bbb980051e
Author: dstandish <ds...@users.noreply.github.com>
AuthorDate: Mon Dec 21 05:12:19 2020 -0800
Update chart readme to remove astronomer references (#13210)
(cherry picked from commit a3cc78dc16bf228901ba64e65d202d4685bf4fc7)
---
chart/README.md | 39 +++++++++++++++++++++++----------------
1 file changed, 23 insertions(+), 16 deletions(-)
diff --git a/chart/README.md b/chart/README.md
index c5106be..7a7c81b 100644
--- a/chart/README.md
+++ b/chart/README.md
@@ -286,18 +286,12 @@ Confirm it's up:
kubectl cluster-info --context kind-kind
```
-**Add Astronomer's Helm repo:**
-
-```
-helm repo add astronomer https://helm.astronomer.io
-helm repo update
-```
**Create namespace + install the chart:**
```
kubectl create namespace airflow
-helm install airflow --n airflow astronomer/airflow
+helm install airflow --n airflow .
```
It may take a few minutes. Confirm the pods are up:
@@ -312,25 +306,38 @@ to port-forward the Airflow UI to http://localhost:8080/ to confirm Airflow is w
**Build a Docker image from your DAGs:**
-1. Start a project using [astro-cli](https://github.com/astronomer/astro-cli), which will generate a Dockerfile, and load your DAGs in. You can test locally before pushing to kind with `astro airflow start`.
+1. Create a project
- mkdir my-airflow-project && cd my-airflow-project
- astro dev init
+ ```shell script
+ mkdir my-airflow-project && cd my-airflow-project
+ mkdir dags # put dags here
+ cat <<EOM > Dockerfile
+ FROM apache/airflow
+ COPY . .
+ EOM
+ ```
2. Then build the image:
- docker build -t my-dags:0.0.1 .
+ ```shell script
+ docker build -t my-dags:0.0.1 .
+ ```
3. Load the image into kind:
- kind load docker-image my-dags:0.0.1
+ ```shell script
+ kind load docker-image my-dags:0.0.1
+ ```
4. Upgrade Helm deployment:
- helm upgrade airflow -n airflow \
- --set images.airflow.repository=my-dags \
- --set images.airflow.tag=0.0.1 \
- astronomer/airflow
+ ```shell script
+ # from airflow chart directory
+ helm upgrade airflow -n airflow \
+ --set images.airflow.repository=my-dags \
+ --set images.airflow.tag=0.0.1 \
+ .
+ ```
## Contributing
[airflow] 03/06: Use new logging options on values.yaml (#13173)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 1fd654a4ff9f5f7d90a2bfa09f186709f27a8794
Author: Flávio de Assis <34...@users.noreply.github.com>
AuthorDate: Sat Dec 19 22:14:35 2020 -0300
Use new logging options on values.yaml (#13173)
(cherry picked from commit 23a47879ababe76f6cf9034a2bae055b2a91bf1f)
---
chart/values.yaml | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/chart/values.yaml b/chart/values.yaml
index cb605c3..d84a785 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -643,13 +643,16 @@ config:
core:
dags_folder: '{{ include "airflow_dags" . }}'
load_examples: 'False'
- colored_console_log: 'False'
executor: '{{ .Values.executor }}'
+ # For Airflow 1.10, backward compatibility
+ colored_console_log: 'False'
remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
# Authentication backend used for the experimental API
api:
auth_backend: airflow.api.auth.backend.deny_all
logging:
+ remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
+ colored_console_log: 'False'
logging_level: DEBUG
metrics:
statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}'
@@ -660,10 +663,8 @@ config:
enable_proxy_fix: 'True'
expose_config: 'True'
rbac: 'True'
-
celery:
default_queue: celery
-
scheduler:
scheduler_heartbeat_sec: 5
# For Airflow 1.10, backward compatibility
@@ -671,7 +672,6 @@ config:
statsd_port: 9125
statsd_prefix: airflow
statsd_host: '{{ printf "%s-statsd" .Release.Name }}'
-
# Restart Scheduler every 41460 seconds (11 hours 31 minutes)
# The odd time is chosen so it is not always restarting on the same "hour" boundary
run_duration: 41460
@@ -682,13 +682,11 @@ config:
max_retries: 3
timeout: 30
retry_timeout: 'True'
-
kerberos:
keytab: '{{ .Values.kerberos.keytabPath }}'
reinit_frequency: '{{ .Values.kerberos.reinitFrequency }}'
principal: '{{ .Values.kerberos.principal }}'
ccache: '{{ .Values.kerberos.ccacheMountPath }}/{{ .Values.kerberos.ccacheFileName }}'
-
kubernetes:
namespace: '{{ .Release.Namespace }}'
airflow_configmap: '{{ include "airflow_config" . }}'
[airflow] 06/06: Allow webserver to read pod logs directly (#12598)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 05b6464b69d9baf787a878c8a850c3217878c2a5
Author: Daniel Imberman <da...@gmail.com>
AuthorDate: Tue Nov 24 15:13:23 2020 -0800
Allow webserver to read pod logs directly (#12598)
* Allow webserver to read pod logs directly
For users who are testing the KubernetesExecutor, allows users to read
pod logs via the Kubernetes API. Worth noting that these logs will only
be accessible while the worker is running.
* fix tests
(cherry picked from commit 9f28e416dbc6374dc9c7115304731a7bc0b4bfa9)
---
chart/templates/rbac/pod-log-reader-role.yaml | 56 ++++++++++++++++++++++
.../templates/rbac/pod-log-reader-rolebinding.yaml | 53 ++++++++++++++++++++
chart/tests/test_basic_helm_chart.py | 4 +-
chart/values.schema.json | 4 ++
chart/values.yaml | 1 +
5 files changed, 117 insertions(+), 1 deletion(-)
diff --git a/chart/templates/rbac/pod-log-reader-role.yaml b/chart/templates/rbac/pod-log-reader-role.yaml
new file mode 100644
index 0000000..72f5e35
--- /dev/null
+++ b/chart/templates/rbac/pod-log-reader-role.yaml
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+################################
+## Airflow Pod Reader Role
+#################################
+{{- if and .Values.rbacEnabled .Values.webserver.allowPodLogReading }}
+{{- if .Values.multiNamespaceMode }}
+kind: ClusterRole
+{{- else }}
+kind: Role
+{{- end }}
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: {{ .Release.Name }}-pod-log-reader-role
+{{- if not .Values.multiNamespaceMode }}
+ namespace: {{ .Release.Namespace }}
+{{- end }}
+ labels:
+ tier: airflow
+ release: {{ .Release.Name }}
+ chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ heritage: {{ .Release.Service }}
+{{- with .Values.labels }}
+{{ toYaml . | indent 4 }}
+{{- end }}
+rules:
+ - apiGroups:
+ - ""
+ resources:
+ - "pods"
+ verbs:
+ - "list"
+ - "get"
+ - "watch"
+ - apiGroups:
+ - ""
+ resources:
+ - "pods/log"
+ verbs:
+ - "get"
+{{- end }}
diff --git a/chart/templates/rbac/pod-log-reader-rolebinding.yaml b/chart/templates/rbac/pod-log-reader-rolebinding.yaml
new file mode 100644
index 0000000..25371eb
--- /dev/null
+++ b/chart/templates/rbac/pod-log-reader-rolebinding.yaml
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+################################
+## Airflow Pod Reader Role Binding
+#################################
+{{- if and .Values.rbacEnabled .Values.webserver.allowPodLogReading }}
+{{- if .Values.multiNamespaceMode }}
+kind: ClusterRoleBinding
+{{- else }}
+kind: RoleBinding
+{{- end }}
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+{{- if not .Values.multiNamespaceMode }}
+ namespace: {{ .Release.Namespace }}
+{{- end }}
+ name: {{ .Release.Name }}-pod-log-reader-rolebinding
+ labels:
+ tier: airflow
+ release: {{ .Release.Name }}
+ chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ heritage: {{ .Release.Service }}
+{{- with .Values.labels }}
+{{ toYaml . | indent 4 }}
+{{- end }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+{{- if .Values.multiNamespaceMode }}
+ kind: ClusterRole
+{{- else }}
+ kind: Role
+{{- end }}
+ name: {{ .Release.Name }}-pod-log-reader-role
+subjects:
+ - kind: ServiceAccount
+ name: {{ .Release.Name }}-webserver
+ namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/chart/tests/test_basic_helm_chart.py b/chart/tests/test_basic_helm_chart.py
index 26ea1c1..af66267 100644
--- a/chart/tests/test_basic_helm_chart.py
+++ b/chart/tests/test_basic_helm_chart.py
@@ -22,7 +22,7 @@ import jmespath
from tests.helm_template_generator import render_chart
-OBJECT_COUNT_IN_BASIC_DEPLOYMENT = 22
+OBJECT_COUNT_IN_BASIC_DEPLOYMENT = 24
class TestBaseChartTest(unittest.TestCase):
@@ -50,7 +50,9 @@ class TestBaseChartTest(unittest.TestCase):
('Secret', 'TEST-BASIC-airflow-result-backend'),
('ConfigMap', 'TEST-BASIC-airflow-config'),
('Role', 'TEST-BASIC-pod-launcher-role'),
+ ('Role', 'TEST-BASIC-pod-log-reader-role'),
('RoleBinding', 'TEST-BASIC-pod-launcher-rolebinding'),
+ ('RoleBinding', 'TEST-BASIC-pod-log-reader-rolebinding'),
('Service', 'TEST-BASIC-postgresql-headless'),
('Service', 'TEST-BASIC-postgresql'),
('Service', 'TEST-BASIC-statsd'),
diff --git a/chart/values.schema.json b/chart/values.schema.json
index f1d8271..3248b66 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -688,6 +688,10 @@
"type": "object",
"additionalProperties": false,
"properties": {
+ "allowPodLogReading": {
+ "description": "Allow webserver to read k8s pod logs. Useful when you don't have an external log store.",
+ "type": "boolean"
+ },
"livenessProbe": {
"description": "Liveness probe configuration.",
"type": "object",
diff --git a/chart/values.yaml b/chart/values.yaml
index d84a785..38f26e5 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -387,6 +387,7 @@ scheduler:
# Airflow webserver settings
webserver:
+ allowPodLogReading: true
livenessProbe:
initialDelaySeconds: 15
timeoutSeconds: 30