You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2021/04/09 12:18:39 UTC
[airflow] branch master updated: Chart: Add tests for tolerations,
affinity & node-selector (#15297)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new cf67bb8 Chart: Add tests for tolerations, affinity & node-selector (#15297)
cf67bb8 is described below
commit cf67bb88bc315a89910f2cee28a869dc53ff36f1
Author: Kaxil Naik <ka...@gmail.com>
AuthorDate: Fri Apr 9 13:18:19 2021 +0100
Chart: Add tests for tolerations, affinity & node-selector (#15297)
This commits adds tests and improve tests coverage for the Helm chart
by testing that tolerations, affinity and node-selector can be modified
for the following components:
- Flower
- Pgbouncer
- Scheduler
- Statsd
- Webserver
- Worker
---
chart/tests/test_flower.py | 45 ++++++++++++++++++
chart/tests/test_pgbouncer.py | 79 ++++++++++++++++++++++++++++++++
chart/tests/test_redis.py | 46 +++++++++++++++++++
chart/tests/test_scheduler.py | 44 ++++++++++++++++++
chart/tests/test_statsd.py | 44 ++++++++++++++++++
chart/tests/test_webserver_deployment.py | 44 ++++++++++++++++++
chart/tests/test_worker.py | 45 ++++++++++++++++++
7 files changed, 347 insertions(+)
diff --git a/chart/tests/test_flower.py b/chart/tests/test_flower.py
index 1a10e10..6d3ce79 100644
--- a/chart/tests/test_flower.py
+++ b/chart/tests/test_flower.py
@@ -82,3 +82,48 @@ class TestFlower:
assert ['curl', 'localhost:7777'] == jmespath.search(
"spec.template.spec.containers[0].readinessProbe.exec.command", docs[0]
)
+
+ def test_should_create_valid_affinity_tolerations_and_node_selector(self):
+ docs = render_chart(
+ values={
+ "executor": "CeleryExecutor",
+ "flower": {
+ "affinity": {
+ "nodeAffinity": {
+ "requiredDuringSchedulingIgnoredDuringExecution": {
+ "nodeSelectorTerms": [
+ {
+ "matchExpressions": [
+ {"key": "foo", "operator": "In", "values": ["true"]},
+ ]
+ }
+ ]
+ }
+ }
+ },
+ "tolerations": [
+ {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"}
+ ],
+ "nodeSelector": {"diskType": "ssd"},
+ },
+ },
+ show_only=["templates/flower/flower-deployment.yaml"],
+ )
+
+ assert "Deployment" == jmespath.search("kind", docs[0])
+ assert "foo" == jmespath.search(
+ "spec.template.spec.affinity.nodeAffinity."
+ "requiredDuringSchedulingIgnoredDuringExecution."
+ "nodeSelectorTerms[0]."
+ "matchExpressions[0]."
+ "key",
+ docs[0],
+ )
+ assert "ssd" == jmespath.search(
+ "spec.template.spec.nodeSelector.diskType",
+ docs[0],
+ )
+ assert "dynamic-pods" == jmespath.search(
+ "spec.template.spec.tolerations[0].key",
+ docs[0],
+ )
diff --git a/chart/tests/test_pgbouncer.py b/chart/tests/test_pgbouncer.py
new file mode 100644
index 0000000..8b550d4
--- /dev/null
+++ b/chart/tests/test_pgbouncer.py
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+import jmespath
+
+from tests.helm_template_generator import render_chart
+
+
+class PgbouncerTest(unittest.TestCase):
+ def test_should_create_pgbouncer(self):
+ docs = render_chart(
+ values={"pgbouncer": {"enabled": True}},
+ show_only=["templates/pgbouncer/pgbouncer-deployment.yaml"],
+ )
+
+ assert "RELEASE-NAME-pgbouncer" == jmespath.search("metadata.name", docs[0])
+
+ assert "pgbouncer" == jmespath.search("spec.template.spec.containers[0].name", docs[0])
+
+ def test_should_create_valid_affinity_tolerations_and_node_selector(self):
+ docs = render_chart(
+ values={
+ "pgbouncer": {
+ "enabled": True,
+ "affinity": {
+ "nodeAffinity": {
+ "requiredDuringSchedulingIgnoredDuringExecution": {
+ "nodeSelectorTerms": [
+ {
+ "matchExpressions": [
+ {"key": "foo", "operator": "In", "values": ["true"]},
+ ]
+ }
+ ]
+ }
+ }
+ },
+ "tolerations": [
+ {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"}
+ ],
+ "nodeSelector": {"diskType": "ssd"},
+ }
+ },
+ show_only=["templates/pgbouncer/pgbouncer-deployment.yaml"],
+ )
+
+ assert "Deployment" == jmespath.search("kind", docs[0])
+ assert "foo" == jmespath.search(
+ "spec.template.spec.affinity.nodeAffinity."
+ "requiredDuringSchedulingIgnoredDuringExecution."
+ "nodeSelectorTerms[0]."
+ "matchExpressions[0]."
+ "key",
+ docs[0],
+ )
+ assert "ssd" == jmespath.search(
+ "spec.template.spec.nodeSelector.diskType",
+ docs[0],
+ )
+ assert "dynamic-pods" == jmespath.search(
+ "spec.template.spec.tolerations[0].key",
+ docs[0],
+ )
diff --git a/chart/tests/test_redis.py b/chart/tests/test_redis.py
index a3e1168..846b7bd 100644
--- a/chart/tests/test_redis.py
+++ b/chart/tests/test_redis.py
@@ -20,6 +20,7 @@ from base64 import b64decode
from subprocess import CalledProcessError
from typing import Optional
+import jmespath
import pytest
from parameterized import parameterized
@@ -229,3 +230,48 @@ class RedisTest(unittest.TestCase):
)
self.assert_broker_url_env(k8s_obj_by_key, expected_broker_url_secret_name)
+
+ def test_should_create_valid_affinity_tolerations_and_node_selector(self):
+ docs = render_chart(
+ values={
+ "executor": "CeleryExecutor",
+ "redis": {
+ "affinity": {
+ "nodeAffinity": {
+ "requiredDuringSchedulingIgnoredDuringExecution": {
+ "nodeSelectorTerms": [
+ {
+ "matchExpressions": [
+ {"key": "foo", "operator": "In", "values": ["true"]},
+ ]
+ }
+ ]
+ }
+ }
+ },
+ "tolerations": [
+ {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"}
+ ],
+ "nodeSelector": {"diskType": "ssd"},
+ },
+ },
+ show_only=["templates/redis/redis-statefulset.yaml"],
+ )
+
+ assert "StatefulSet" == jmespath.search("kind", docs[0])
+ assert "foo" == jmespath.search(
+ "spec.template.spec.affinity.nodeAffinity."
+ "requiredDuringSchedulingIgnoredDuringExecution."
+ "nodeSelectorTerms[0]."
+ "matchExpressions[0]."
+ "key",
+ docs[0],
+ )
+ assert "ssd" == jmespath.search(
+ "spec.template.spec.nodeSelector.diskType",
+ docs[0],
+ )
+ assert "dynamic-pods" == jmespath.search(
+ "spec.template.spec.tolerations[0].key",
+ docs[0],
+ )
diff --git a/chart/tests/test_scheduler.py b/chart/tests/test_scheduler.py
index 269dae7..80ff94f 100644
--- a/chart/tests/test_scheduler.py
+++ b/chart/tests/test_scheduler.py
@@ -58,3 +58,47 @@ class SchedulerTest(unittest.TestCase):
assert "test-volume" == jmespath.search(
"spec.template.spec.containers[0].volumeMounts[3].name", docs[0]
)
+
+ def test_should_create_valid_affinity_tolerations_and_node_selector(self):
+ docs = render_chart(
+ values={
+ "scheduler": {
+ "affinity": {
+ "nodeAffinity": {
+ "requiredDuringSchedulingIgnoredDuringExecution": {
+ "nodeSelectorTerms": [
+ {
+ "matchExpressions": [
+ {"key": "foo", "operator": "In", "values": ["true"]},
+ ]
+ }
+ ]
+ }
+ }
+ },
+ "tolerations": [
+ {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"}
+ ],
+ "nodeSelector": {"diskType": "ssd"},
+ }
+ },
+ show_only=["templates/scheduler/scheduler-deployment.yaml"],
+ )
+
+ assert "Deployment" == jmespath.search("kind", docs[0])
+ assert "foo" == jmespath.search(
+ "spec.template.spec.affinity.nodeAffinity."
+ "requiredDuringSchedulingIgnoredDuringExecution."
+ "nodeSelectorTerms[0]."
+ "matchExpressions[0]."
+ "key",
+ docs[0],
+ )
+ assert "ssd" == jmespath.search(
+ "spec.template.spec.nodeSelector.diskType",
+ docs[0],
+ )
+ assert "dynamic-pods" == jmespath.search(
+ "spec.template.spec.tolerations[0].key",
+ docs[0],
+ )
diff --git a/chart/tests/test_statsd.py b/chart/tests/test_statsd.py
index 9b00524..6381443 100644
--- a/chart/tests/test_statsd.py
+++ b/chart/tests/test_statsd.py
@@ -50,3 +50,47 @@ class StatsdTest(unittest.TestCase):
"mountPath": "/etc/statsd-exporter/mappings.yml",
"subPath": "mappings.yml",
} in jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0])
+
+ def test_should_create_valid_affinity_tolerations_and_node_selector(self):
+ docs = render_chart(
+ values={
+ "statsd": {
+ "affinity": {
+ "nodeAffinity": {
+ "requiredDuringSchedulingIgnoredDuringExecution": {
+ "nodeSelectorTerms": [
+ {
+ "matchExpressions": [
+ {"key": "foo", "operator": "In", "values": ["true"]},
+ ]
+ }
+ ]
+ }
+ }
+ },
+ "tolerations": [
+ {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"}
+ ],
+ "nodeSelector": {"diskType": "ssd"},
+ }
+ },
+ show_only=["templates/statsd/statsd-deployment.yaml"],
+ )
+
+ assert "Deployment" == jmespath.search("kind", docs[0])
+ assert "foo" == jmespath.search(
+ "spec.template.spec.affinity.nodeAffinity."
+ "requiredDuringSchedulingIgnoredDuringExecution."
+ "nodeSelectorTerms[0]."
+ "matchExpressions[0]."
+ "key",
+ docs[0],
+ )
+ assert "ssd" == jmespath.search(
+ "spec.template.spec.nodeSelector.diskType",
+ docs[0],
+ )
+ assert "dynamic-pods" == jmespath.search(
+ "spec.template.spec.tolerations[0].key",
+ docs[0],
+ )
diff --git a/chart/tests/test_webserver_deployment.py b/chart/tests/test_webserver_deployment.py
index bdae7e7..064160a 100644
--- a/chart/tests/test_webserver_deployment.py
+++ b/chart/tests/test_webserver_deployment.py
@@ -143,3 +143,47 @@ class WebserverDeploymentTest(unittest.TestCase):
)
assert "test-container" == jmespath.search("spec.template.spec.containers[-1].name", docs[0])
+
+ def test_should_create_valid_affinity_tolerations_and_node_selector(self):
+ docs = render_chart(
+ values={
+ "webserver": {
+ "affinity": {
+ "nodeAffinity": {
+ "requiredDuringSchedulingIgnoredDuringExecution": {
+ "nodeSelectorTerms": [
+ {
+ "matchExpressions": [
+ {"key": "foo", "operator": "In", "values": ["true"]},
+ ]
+ }
+ ]
+ }
+ }
+ },
+ "tolerations": [
+ {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"}
+ ],
+ "nodeSelector": {"diskType": "ssd"},
+ }
+ },
+ show_only=["templates/webserver/webserver-deployment.yaml"],
+ )
+
+ assert "Deployment" == jmespath.search("kind", docs[0])
+ assert "foo" == jmespath.search(
+ "spec.template.spec.affinity.nodeAffinity."
+ "requiredDuringSchedulingIgnoredDuringExecution."
+ "nodeSelectorTerms[0]."
+ "matchExpressions[0]."
+ "key",
+ docs[0],
+ )
+ assert "ssd" == jmespath.search(
+ "spec.template.spec.nodeSelector.diskType",
+ docs[0],
+ )
+ assert "dynamic-pods" == jmespath.search(
+ "spec.template.spec.tolerations[0].key",
+ docs[0],
+ )
diff --git a/chart/tests/test_worker.py b/chart/tests/test_worker.py
index eb210c8..b0b5f0e 100644
--- a/chart/tests/test_worker.py
+++ b/chart/tests/test_worker.py
@@ -88,3 +88,48 @@ class WorkerTest(unittest.TestCase):
assert "100%" == jmespath.search("spec.strategy.rollingUpdate.maxSurge", docs[0])
assert "50%" == jmespath.search("spec.strategy.rollingUpdate.maxUnavailable", docs[0])
+
+ def test_should_create_valid_affinity_tolerations_and_node_selector(self):
+ docs = render_chart(
+ values={
+ "executor": "CeleryExecutor",
+ "workers": {
+ "affinity": {
+ "nodeAffinity": {
+ "requiredDuringSchedulingIgnoredDuringExecution": {
+ "nodeSelectorTerms": [
+ {
+ "matchExpressions": [
+ {"key": "foo", "operator": "In", "values": ["true"]},
+ ]
+ }
+ ]
+ }
+ }
+ },
+ "tolerations": [
+ {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"}
+ ],
+ "nodeSelector": {"diskType": "ssd"},
+ },
+ },
+ show_only=["templates/workers/worker-deployment.yaml"],
+ )
+
+ assert "StatefulSet" == jmespath.search("kind", docs[0])
+ assert "foo" == jmespath.search(
+ "spec.template.spec.affinity.nodeAffinity."
+ "requiredDuringSchedulingIgnoredDuringExecution."
+ "nodeSelectorTerms[0]."
+ "matchExpressions[0]."
+ "key",
+ docs[0],
+ )
+ assert "ssd" == jmespath.search(
+ "spec.template.spec.nodeSelector.diskType",
+ docs[0],
+ )
+ assert "dynamic-pods" == jmespath.search(
+ "spec.template.spec.tolerations[0].key",
+ docs[0],
+ )