You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by di...@apache.org on 2020/10/28 01:31:03 UTC
[airflow] branch master updated: Add Python Helm testing framework
(#11693)
This is an automated email from the ASF dual-hosted git repository.
dimberman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 0d1ad66 Add Python Helm testing framework (#11693)
0d1ad66 is described below
commit 0d1ad6648ee63311614e043d5893cf36a0cd9aea
Author: Daniel Imberman <da...@gmail.com>
AuthorDate: Tue Oct 27 18:29:47 2020 -0700
Add Python Helm testing framework (#11693)
* Helm Python Testing
* helm change
* add back args
---
.dockerignore | 1 +
.github/workflows/ci.yml | 50 ++++-
.pre-commit-config.yaml | 22 ++-
BREEZE.rst | 15 +-
Dockerfile.ci | 8 +
STATIC_CODE_CHECKS.rst | 2 +
TESTING.rst | 42 +++++
breeze-complete | 3 +-
chart/requirements.lock | 6 +-
chart/tests/__init__.py | 16 ++
chart/tests/conftest.py | 36 ++++
chart/tests/git-sync-scheduler_test.yaml | 154 ---------------
chart/tests/helm_template_generator.py | 51 +++++
chart/tests/test_basic_helm_chart.py | 66 +++++++
chart/tests/test_git_sync_scheduler.py | 226 +++++++++++++++++++++++
scripts/ci/docker-compose/local.yml | 1 +
scripts/ci/libraries/_local_mounts.sh | 1 +
scripts/in_container/entrypoint_ci.sh | 9 +-
setup.cfg | 2 +-
tests/airflow_pylint/disable_checks_for_tests.py | 3 +-
20 files changed, 540 insertions(+), 174 deletions(-)
diff --git a/.dockerignore b/.dockerignore
index 0a1d7d0..d31dd84 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -31,6 +31,7 @@
!common
!dags
!dev
+!chart
!docs
!licenses
!metastore_browser
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8f6a7da..8331386 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -143,7 +143,7 @@ jobs:
helm-tests:
timeout-minutes: 5
- name: "Checks: Helm tests"
+ name: "Checks: Helm tests. Will soon be replaced with python tests"
runs-on: ubuntu-latest
needs: [build-info]
if: >
@@ -432,6 +432,52 @@ jobs:
name: airflow-provider-readmes
path: "./files/airflow-readme-*"
+ helm-python-tests:
+ timeout-minutes: 5
+ name: "Python unit tests for helm chart"
+ runs-on: ubuntu-latest
+ needs: [build-info, ci-images]
+ env:
+ MOUNT_LOCAL_SOURCES: "true"
+ RUN_TESTS: true
+ TEST_TYPES: "Helm"
+ BACKEND: "sqlite"
+ PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
+ if: >
+ needs.build-info.outputs.needs-helm-tests == 'true' &&
+ (github.repository == 'apache/airflow' || github.event_name != 'schedule')
+ steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v2
+ - name: "Setup python"
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
+ - name: "Free space"
+ run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
+ run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
+ - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
+ run: ./scripts/ci/testing/ci_run_airflow_testing.sh
+ - name: "Upload airflow logs"
+ uses: actions/upload-artifact@v2
+ if: failure()
+ with:
+ name: airflow-logs-helm
+ path: "./files/airflow_logs*"
+ - name: "Upload container logs"
+ uses: actions/upload-artifact@v2
+ if: failure()
+ with:
+ name: container-logs-helm
+ path: "./files/container_logs*"
+ - name: "Upload artifact for coverage"
+ uses: actions/upload-artifact@v2
+ with:
+ name: >
+ coverage-helm
+ path: "./files/coverage.xml"
+
tests-postgres:
timeout-minutes: 60
@@ -467,7 +513,7 @@ jobs:
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
+ - name: "Tests: Helm"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
- name: "Upload airflow logs"
uses: actions/upload-artifact@v2
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index bd3f98e..778beda 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -159,7 +159,7 @@ repos:
rev: 20.8b1
hooks:
- id: black
- files: api_connexion/.*\.py|.*providers.*\.py
+ files: api_connexion/.*\.py|.*providers.*\.py|^chart/tests/.*\.py
exclude: .*kubernetes_pod\.py|.*google/common/hooks/base_google\.py$
args: [--config=./pyproject.toml]
- repo: https://github.com/pre-commit/pre-commit-hooks
@@ -207,7 +207,15 @@ repos:
args:
- --convention=pep257
- --add-ignore=D100,D102,D104,D105,D107,D205,D400,D401
- exclude: ^tests/.*\.py$|^scripts/.*\.py$|^dev|^provider_packages|^kubernetes_tests|.*example_dags/.*
+ exclude: |
+ (?x)
+ ^tests/.*\.py$|
+ ^scripts/.*\.py$|
+ ^dev|
+ ^provider_packages|
+ ^kubernetes_tests|
+ .*example_dags/.*|
+ ^chart/.*\.py$
- repo: local
hooks:
- id: lint-openapi
@@ -283,7 +291,7 @@ repos:
entry: "^\\s*from\\s+\\."
pass_filenames: true
files: \.py$
- exclude: ^tests/
+ exclude: ^tests/|^chart/tests/
- id: language-matters
language: pygrep
name: Check for language that we do not accept as community
@@ -301,7 +309,7 @@ repos:
^CHANGELOG.txt$
- id: consistent-pylint
language: pygrep
- name: Check for inconsitent pylint disable/enable without space
+ name: Check for inconsistent pylint disable/enable without space
entry: "pylint:disable|pylint:enable"
pass_filenames: true
files: \.py$
@@ -457,6 +465,12 @@ repos:
language: system
entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh"
files: \.py$
+ exclude: ^dev|^provider_packages|^chart
+ - id: mypy
+ name: Run mypy for helm chart tests
+ language: system
+ entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh"
+ files: ^chart/.*\.py$
exclude: ^dev|^provider_packages
require_serial: true
- id: pylint
diff --git a/BREEZE.rst b/BREEZE.rst
index b92f3ce..4a39f2b 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -2003,11 +2003,12 @@ This is the current syntax for `./breeze <./breeze>`_:
consistent-pylint daysago-import-check debug-statements detect-private-key doctoc
dont-use-safe-filter end-of-file-fixer fix-encoding-pragma flake8 forbid-tabs
helm-lint incorrect-use-of-LoggingMixin insert-license isort language-matters
- lint-dockerfile lint-openapi mermaid mixed-line-ending mypy no-relative-imports
- pre-commit-descriptions provide-create-sessions pydevd pydocstyle pylint
- pylint-tests python-no-log-warn restrict-start_date rst-backticks setup-order
- shellcheck sort-in-the-wild stylelint trailing-whitespace update-breeze-file
- update-extras update-local-yml-file update-setup-cfg-file yamllint
+ lint-dockerfile lint-openapi mermaid mixed-line-ending mypy mypy-helm
+ no-relative-imports pre-commit-descriptions provide-create-sessions pydevd
+ pydocstyle pylint pylint-tests python-no-log-warn restrict-start_date rst-backticks
+ setup-order shellcheck sort-in-the-wild stylelint trailing-whitespace
+ update-breeze-file update-extras update-local-yml-file update-setup-cfg-file
+ yamllint
You can pass extra arguments including options to to the pre-commit framework as
<EXTRA_ARGS> passed after --. For example:
@@ -2042,7 +2043,7 @@ This is the current syntax for `./breeze <./breeze>`_:
--test-type TEST_TYPE
Type of the test to run. One of:
- All,Core,Providers,API,CLI,Integration,Other,WWW,Heisentests,Postgres,MySQL
+ All,Core,Providers,API,CLI,Integration,Other,WWW,Heisentests,Postgres,MySQL,Helm
Default: All
@@ -2419,7 +2420,7 @@ This is the current syntax for `./breeze <./breeze>`_:
--test-type TEST_TYPE
Type of the test to run. One of:
- All,Core,Providers,API,CLI,Integration,Other,WWW,Heisentests,Postgres,MySQL
+ All,Core,Providers,API,CLI,Integration,Other,WWW,Heisentests,Postgres,MySQL,Helm
Default: All
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 98c7dd4..573ad78 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -350,6 +350,14 @@ RUN echo "source /etc/bash_completion" >> ~/.bashrc
WORKDIR ${AIRFLOW_SOURCES}
+# Install Helm
+ARG HELM_VERSION="v3.2.4"
+
+RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \
+ && HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-amd64.tar.gz" \
+ && curl --location "${HELM_URL}" | tar -xvz -O "${SYSTEM}"-amd64/helm > /usr/local/bin/helm \
+ && chmod +x /usr/local/bin/helm
+
# Additional python deps to install
ARG ADDITIONAL_PYTHON_DEPS=""
diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst
index f19cf7a..d3fb5c7 100644
--- a/STATIC_CODE_CHECKS.rst
+++ b/STATIC_CODE_CHECKS.rst
@@ -118,6 +118,8 @@ require Breeze Docker images to be installed locally:
----------------------------------- ---------------------------------------------------------------- ------------
``mypy`` Runs mypy. *
----------------------------------- ---------------------------------------------------------------- ------------
+``mypy-helm`` Runs mypy. *
+----------------------------------- ---------------------------------------------------------------- ------------
``pre-commit-descriptions`` Check if all pre-commits are described in docs.
----------------------------------- ---------------------------------------------------------------- ------------
``provide-create-sessions`` Make sure provide-session and create-session imports are OK.
diff --git a/TESTING.rst b/TESTING.rst
index b42ae98..dca801e 100644
--- a/TESTING.rst
+++ b/TESTING.rst
@@ -196,6 +196,48 @@ Run all Quarantined tests:
./breeze --test-type Quarantined tests --db-reset
+Helm Unit Tests
+===============
+
+On the Airflow Project, we have decided to stick with pythonic testing for our Helm chart. This makes our chart
+easier to test, easier to modify, and able to run with the same testing infrastructure. To add Helm unit tests
+go to the ``chart/tests`` directory and add your unit test by creating a class that extends ``unittest.TestCase``
+
+.. code-block:: python
+
+ class TestBaseChartTest(unittest.TestCase):
+
+To render the chart create a yaml string with the nested dictionary of options you wish to test. you can then
+use our ``render_chart`` function to render the object of interest into a testable python dictionary. Once the chart
+has been rendered, you can use the ``render_k8s_object`` function to create a k8s model object that simultaneously
+ensures that the object created properly conforms to the expected object spec and allows you to use object values
+instead of nested dictionaries.
+
+Example test here:
+
+.. code-block:: python
+ from .helm_template_generator import render_chart, render_k8s_object
+
+
+ git_sync_basic = """
+ dags:
+ gitSync:
+ enabled: true
+ """
+
+ class TestGitSyncScheduler(unittest.TestCase):
+
+ def test_basic(self):
+ helm_settings = yaml.safe_load(git_sync_basic)
+ res = render_chart('GIT-SYNC', helm_settings,
+ show_only=["templates/scheduler/scheduler-deployment.yaml"])
+ dep: k8s.V1Deployment = render_k8s_object(res[0], k8s.V1Deployment)
+ self.assertEqual("dags", dep.spec.template.spec.volumes[1].name)
+
+To run tests using breeze run the following command
+
+.. code-block:: bash
+ ./breeze --test-type Helm tests
Airflow Integration Tests
=========================
diff --git a/breeze-complete b/breeze-complete
index 9862548..6f0cdb4 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -33,7 +33,7 @@ _breeze_allowed_kind_versions="v0.8.0"
_breeze_allowed_mysql_versions="5.7 8"
_breeze_allowed_postgres_versions="9.6 10 11 12 13"
_breeze_allowed_kind_operations="start stop restart status deploy test shell"
-_breeze_allowed_test_types="All Core Providers API CLI Integration Other WWW Heisentests Postgres MySQL"
+_breeze_allowed_test_types="All Core Providers API CLI Integration Other WWW Heisentests Postgres MySQL Helm"
# shellcheck disable=SC2034
{
@@ -100,6 +100,7 @@ lint-openapi
mermaid
mixed-line-ending
mypy
+mypy-helm
no-relative-imports
pre-commit-descriptions
provide-create-sessions
diff --git a/chart/requirements.lock b/chart/requirements.lock
index f86e696..715458e 100644
--- a/chart/requirements.lock
+++ b/chart/requirements.lock
@@ -1,6 +1,6 @@
dependencies:
- name: postgresql
- repository: https://kubernetes-charts.storage.googleapis.com
+ repository: https://kubernetes-charts.storage.googleapis.com/
version: 6.3.12
-digest: sha256:58d88cf56e78b2380091e9e16cc6ccf58b88b3abe4a1886dd47cd9faef5309af
-generated: "2020-06-21T19:11:53.498134738+02:00"
+digest: sha256:e8d53453861c590e6ae176331634c9268a11cf894be17ed580fa2b347101be97
+generated: "2020-10-27T21:16:13.0063538Z"
diff --git a/chart/tests/__init__.py b/chart/tests/__init__.py
new file mode 100644
index 0000000..13a8339
--- /dev/null
+++ b/chart/tests/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/chart/tests/conftest.py b/chart/tests/conftest.py
new file mode 100644
index 0000000..c1a06d8
--- /dev/null
+++ b/chart/tests/conftest.py
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import subprocess
+import sys
+import pytest
+
+# We should set these before loading _any_ of the rest of airflow so that the
+# unit test mode config is set as early as possible.
+tests_directory = os.path.dirname(os.path.realpath(__file__))
+
+
+@pytest.fixture(autouse=True, scope="session")
+def upgrade_helm():
+ """
+ Upgrade Helm repo
+ """
+ subprocess.check_output(
+ ["helm", "repo", "add", "stable", "https://kubernetes-charts.storage.googleapis.com/"]
+ )
+ subprocess.check_output(["helm", "dep", "update", sys.path[0]])
diff --git a/chart/tests/git-sync-scheduler_test.yaml b/chart/tests/git-sync-scheduler_test.yaml
deleted file mode 100644
index 9e8d972..0000000
--- a/chart/tests/git-sync-scheduler_test.yaml
+++ /dev/null
@@ -1,154 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
----
-templates:
- - scheduler/scheduler-deployment.yaml
-tests:
- - it: should add dags volume
- set:
- dags:
- gitSync:
- enabled: true
- asserts:
- - equal:
- path: spec.template.spec.volumes[1].name
- value: dags
- - it: validate the git sync container spec
- set:
- images:
- gitSync:
- repository: test-registry/test-repo
- tag: test-tag
- pullPolicy: Allways
- dags:
- gitSync:
- enabled: true
- containerName: git-sync-test
- wait: 66
- maxFailures: 70
- subPath: "path1/path2"
- dest: "test-dest"
- root: "/git-root"
- rev: HEAD
- depth: 1
- repo: https://github.com/apache/airflow.git
- branch: test-branch
- sshKeySecret: ~
- credentialsSecret: ~
- knownHosts: ~
- persistence:
- enabled: true
- asserts:
- - equal:
- path: spec.template.spec.containers[1]
- value:
- name: git-sync-test
- securityContext:
- runAsUser: 65533
- image: test-registry/test-repo:test-tag
- imagePullPolicy: Allways
- env:
- - name: GIT_SYNC_REV
- value: HEAD
- - name: GIT_SYNC_BRANCH
- value: test-branch
- - name: GIT_SYNC_REPO
- value: https://github.com/apache/airflow.git
- - name: GIT_SYNC_DEPTH
- value: "1"
- - name: GIT_SYNC_ROOT
- value: /git-root
- - name: GIT_SYNC_DEST
- value: test-dest
- - name: GIT_SYNC_ADD_USER
- value: "true"
- - name: GIT_SYNC_WAIT
- value: "66"
- - name: GIT_SYNC_MAX_SYNC_FAILURES
- value: "70"
- volumeMounts:
- - mountPath: /git-root
- name: dags
- - it: validate if ssh params are added
- set:
- dags:
- gitSync:
- enabled: true
- containerName: git-sync-test
- sshKeySecret: ssh-secret
- knownHosts: ~
- branch: test-branch
- asserts:
- - contains:
- path: spec.template.spec.containers[1].env
- content:
- name: GIT_SSH_KEY_FILE
- value: "/etc/git-secret/ssh"
- - contains:
- path: spec.template.spec.containers[1].env
- content:
- name: GIT_SYNC_SSH
- value: "true"
- - contains:
- path: spec.template.spec.containers[1].env
- content:
- name: GIT_KNOWN_HOSTS
- value: "false"
- - contains:
- path: spec.template.spec.volumes
- content:
- name: git-sync-ssh-key
- secret:
- secretName: ssh-secret
- defaultMode: 288
- - it: should set username and pass env variables
- set:
- dags:
- gitSync:
- enabled: true
- credentialsSecret: user-pass-secret
- sshKeySecret: ~
- asserts:
- - contains:
- path: spec.template.spec.containers[1].env
- content:
- name: GIT_SYNC_USERNAME
- valueFrom:
- secretKeyRef:
- name: user-pass-secret
- key: GIT_SYNC_USERNAME
- - contains:
- path: spec.template.spec.containers[1].env
- content:
- name: GIT_SYNC_PASSWORD
- valueFrom:
- secretKeyRef:
- name: user-pass-secret
- key: GIT_SYNC_PASSWORD
- - it: should set the volume claim correctly when using an existing claim
- set:
- dags:
- persistence:
- enabled: true
- existingClaim: test-claim
- asserts:
- - contains:
- path: spec.template.spec.volumes
- content:
- name: dags
- persistentVolumeClaim:
- claimName: test-claim
diff --git a/chart/tests/helm_template_generator.py b/chart/tests/helm_template_generator.py
new file mode 100644
index 0000000..dccbb1e
--- /dev/null
+++ b/chart/tests/helm_template_generator.py
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import subprocess
+import sys
+from tempfile import NamedTemporaryFile
+
+import yaml
+from kubernetes.client.api_client import ApiClient
+
+api_client = ApiClient()
+
+
+def render_chart(name="RELEASE-NAME", values=None, show_only=None):
+ """
+ Function that renders a helm chart into dictionaries. For helm chart testing only
+ """
+ values = values or {}
+ with NamedTemporaryFile() as tmp_file:
+ content = yaml.dump(values)
+ tmp_file.write(content.encode())
+ tmp_file.flush()
+ command = ["helm", "template", name, sys.path[0], '--values', tmp_file.name]
+ if show_only:
+ for i in show_only:
+ command.extend(["--show-only", i])
+ templates = subprocess.check_output(command)
+ k8s_objects = yaml.load_all(templates)
+ k8s_objects = [k8s_object for k8s_object in k8s_objects if k8s_object] # type: ignore
+ return k8s_objects
+
+
+def render_k8s_object(obj, type_to_render):
+ """
+ Function that renders dictionaries into k8s objects. For helm chart testing only.
+ """
+ return api_client._ApiClient__deserialize_model(obj, type_to_render) # pylint: disable=W0212
diff --git a/chart/tests/test_basic_helm_chart.py b/chart/tests/test_basic_helm_chart.py
new file mode 100644
index 0000000..ea02c91
--- /dev/null
+++ b/chart/tests/test_basic_helm_chart.py
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+from .helm_template_generator import render_chart
+
+OBJECT_COUNT_IN_BASIC_DEPLOYMENT = 22
+
+
+class TestBaseChartTest(unittest.TestCase):
+ def test_basic_deployments(self):
+ k8s_objects = render_chart("TEST-BASIC", {"chart": {'metadata': 'AA'}})
+ list_of_kind_names_tuples = [
+ (k8s_object['kind'], k8s_object['metadata']['name']) for k8s_object in k8s_objects
+ ]
+ self.assertEqual(
+ list_of_kind_names_tuples,
+ [
+ ('ServiceAccount', 'TEST-BASIC-scheduler'),
+ ('ServiceAccount', 'TEST-BASIC-webserver'),
+ ('ServiceAccount', 'TEST-BASIC-worker'),
+ ('Secret', 'TEST-BASIC-postgresql'),
+ ('Secret', 'TEST-BASIC-airflow-metadata'),
+ ('Secret', 'TEST-BASIC-airflow-result-backend'),
+ ('ConfigMap', 'TEST-BASIC-airflow-config'),
+ ('ClusterRole', 'TEST-BASIC-pod-launcher-role'),
+ ('ClusterRoleBinding', 'TEST-BASIC-pod-launcher-rolebinding'),
+ ('Service', 'TEST-BASIC-postgresql-headless'),
+ ('Service', 'TEST-BASIC-postgresql'),
+ ('Service', 'TEST-BASIC-statsd'),
+ ('Service', 'TEST-BASIC-webserver'),
+ ('Deployment', 'TEST-BASIC-scheduler'),
+ ('Deployment', 'TEST-BASIC-statsd'),
+ ('Deployment', 'TEST-BASIC-webserver'),
+ ('StatefulSet', 'TEST-BASIC-postgresql'),
+ ('Secret', 'TEST-BASIC-fernet-key'),
+ ('Secret', 'TEST-BASIC-redis-password'),
+ ('Secret', 'TEST-BASIC-broker-url'),
+ ('Job', 'TEST-BASIC-create-user'),
+ ('Job', 'TEST-BASIC-run-airflow-migrations'),
+ ],
+ )
+ self.assertEqual(OBJECT_COUNT_IN_BASIC_DEPLOYMENT, len(k8s_objects))
+
+ def test_basic_deployment_without_default_users(self):
+ k8s_objects = render_chart("TEST-BASIC", {"webserver": {'defaultUser': {'enabled': False}}})
+ list_of_kind_names_tuples = [
+ (k8s_object['kind'], k8s_object['metadata']['name']) for k8s_object in k8s_objects
+ ]
+ self.assertNotIn(('Job', 'TEST-BASIC-create-user'), list_of_kind_names_tuples)
+ self.assertEqual(OBJECT_COUNT_IN_BASIC_DEPLOYMENT - 1, len(k8s_objects))
diff --git a/chart/tests/test_git_sync_scheduler.py b/chart/tests/test_git_sync_scheduler.py
new file mode 100644
index 0000000..dd5fb0e
--- /dev/null
+++ b/chart/tests/test_git_sync_scheduler.py
@@ -0,0 +1,226 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import unittest
+
+import yaml
+from kubernetes.client import models as k8s
+
+from .helm_template_generator import render_chart, render_k8s_object
+
+OBJECT_COUNT_IN_BASIC_DEPLOYMENT = 22
+
+git_sync_basic = """
+dags:
+ gitSync:
+ enabled: true
+"""
+
+git_sync_existing_claim = """
+dags:
+ persistence:
+ enabled: true
+ existingClaim: test-claim
+"""
+
+git_sync_ssh_params = """
+dags:
+ gitSync:
+ enabled: true
+ containerName: git-sync-test
+ sshKeySecret: ssh-secret
+ knownHosts: ~
+ branch: test-branch
+"""
+
+git_sync_username = """
+dags:
+ gitSync:
+ enabled: true
+ credentialsSecret: user-pass-secret
+ sshKeySecret: ~
+"""
+
+git_sync_container_spec = """
+images:
+ gitSync:
+ repository: test-registry/test-repo
+ tag: test-tag
+dags:
+ gitSync:
+ enabled: true
+ containerName: git-sync-test
+ wait: 66
+ maxFailures: 70
+ subPath: "path1/path2"
+ dest: "test-dest"
+ root: "/git-root"
+ rev: HEAD
+ depth: 1
+ repo: https://github.com/apache/airflow.git
+ branch: test-branch
+ sshKeySecret: ~
+ credentialsSecret: ~
+ knownHosts: ~
+ persistence:
+ enabled: true
+"""
+
+
+class TestGitSyncScheduler(unittest.TestCase):
+ def test_basic(self):
+ helm_settings = yaml.safe_load(git_sync_basic)
+ res = render_chart(
+ 'GIT-SYNC', helm_settings, show_only=["templates/scheduler/scheduler-deployment.yaml"]
+ )
+ dep: k8s.V1Deployment = render_k8s_object(res[0], k8s.V1Deployment)
+ self.assertEqual("dags", dep.spec.template.spec.volumes[1].name)
+
+ def test_git_container_spec(self):
+ helm_settings = yaml.safe_load(git_sync_container_spec)
+ res = render_chart(
+ 'GIT-SYNC', helm_settings, show_only=["templates/scheduler/scheduler-deployment.yaml"]
+ )
+ dep: k8s.V1Deployment = render_k8s_object(res[0], k8s.V1Deployment)
+ git_sync_container = dep.spec.template.spec.containers[1]
+ self.assertEqual(git_sync_container.image, "test-registry/test-repo:test-tag")
+ self.assertEqual(git_sync_container.name, "git-sync-test")
+ self.assertEqual(git_sync_container.security_context.run_as_user, 65533)
+ env_dict = [e.to_dict() for e in git_sync_container.env]
+ self.assertEqual(
+ env_dict,
+ [
+ {'name': 'GIT_SYNC_REV', 'value': 'HEAD', 'value_from': None},
+ {'name': 'GIT_SYNC_BRANCH', 'value': 'test-branch', 'value_from': None},
+ {
+ 'name': 'GIT_SYNC_REPO',
+ 'value': 'https://github.com/apache/airflow.git',
+ 'value_from': None,
+ },
+ {'name': 'GIT_SYNC_DEPTH', 'value': '1', 'value_from': None},
+ {'name': 'GIT_SYNC_ROOT', 'value': '/git-root', 'value_from': None},
+ {'name': 'GIT_SYNC_DEST', 'value': 'test-dest', 'value_from': None},
+ {'name': 'GIT_SYNC_ADD_USER', 'value': 'true', 'value_from': None},
+ {'name': 'GIT_SYNC_WAIT', 'value': '66', 'value_from': None},
+ {'name': 'GIT_SYNC_MAX_SYNC_FAILURES', 'value': '70', 'value_from': None},
+ ],
+ )
+
+ self.assertEqual(
+ git_sync_container.volume_mounts, [k8s.V1VolumeMount(name="dags", mount_path="/git-root")]
+ )
+
+ def test_ssh_params_added(self):
+ helm_settings = yaml.safe_load(git_sync_ssh_params)
+ res = render_chart(
+ 'GIT-SYNC', helm_settings, show_only=["templates/scheduler/scheduler-deployment.yaml"]
+ )
+ dep: k8s.V1Deployment = render_k8s_object(res[0], k8s.V1Deployment)
+ git_sync_container = dep.spec.template.spec.containers[1]
+ env_dict = [e.to_dict() for e in git_sync_container.env]
+ self.assertEqual(
+ env_dict,
+ [
+ {'name': 'GIT_SSH_KEY_FILE', 'value': '/etc/git-secret/ssh', 'value_from': None},
+ {'name': 'GIT_SYNC_SSH', 'value': 'true', 'value_from': None},
+ {'name': 'GIT_KNOWN_HOSTS', 'value': 'false', 'value_from': None},
+ {'name': 'GIT_SYNC_REV', 'value': 'HEAD', 'value_from': None},
+ {'name': 'GIT_SYNC_BRANCH', 'value': 'test-branch', 'value_from': None},
+ {
+ 'name': 'GIT_SYNC_REPO',
+ 'value': 'https://github.com/apache/airflow.git',
+ 'value_from': None,
+ },
+ {'name': 'GIT_SYNC_DEPTH', 'value': '1', 'value_from': None},
+ {'name': 'GIT_SYNC_ROOT', 'value': '/git', 'value_from': None},
+ {'name': 'GIT_SYNC_DEST', 'value': 'repo', 'value_from': None},
+ {'name': 'GIT_SYNC_ADD_USER', 'value': 'true', 'value_from': None},
+ {'name': 'GIT_SYNC_WAIT', 'value': '60', 'value_from': None},
+ {'name': 'GIT_SYNC_MAX_SYNC_FAILURES', 'value': '0', 'value_from': None},
+ ],
+ )
+
+ def test_adds_git_username(self):
+ helm_settings = yaml.safe_load(git_sync_username)
+ res = render_chart(
+ 'GIT-SYNC', helm_settings, show_only=["templates/scheduler/scheduler-deployment.yaml"]
+ )
+ dep: k8s.V1Deployment = render_k8s_object(res[0], k8s.V1Deployment)
+ git_sync_container = dep.spec.template.spec.containers[1]
+ env_dict = [e.to_dict() for e in git_sync_container.env]
+ self.assertEqual(
+ env_dict,
+ [
+ {
+ 'name': 'GIT_SYNC_USERNAME',
+ 'value': None,
+ 'value_from': {
+ 'config_map_key_ref': None,
+ 'field_ref': None,
+ 'resource_field_ref': None,
+ 'secret_key_ref': {
+ 'key': 'GIT_SYNC_USERNAME',
+ 'name': 'user-pass-secret',
+ 'optional': None,
+ },
+ },
+ },
+ {
+ 'name': 'GIT_SYNC_PASSWORD',
+ 'value': None,
+ 'value_from': {
+ 'config_map_key_ref': None,
+ 'field_ref': None,
+ 'resource_field_ref': None,
+ 'secret_key_ref': {
+ 'key': 'GIT_SYNC_PASSWORD',
+ 'name': 'user-pass-secret',
+ 'optional': None,
+ },
+ },
+ },
+ {'name': 'GIT_SYNC_REV', 'value': 'HEAD', 'value_from': None},
+ {'name': 'GIT_SYNC_BRANCH', 'value': 'v1-10-stable', 'value_from': None},
+ {
+ 'name': 'GIT_SYNC_REPO',
+ 'value': 'https://github.com/apache/airflow.git',
+ 'value_from': None,
+ },
+ {'name': 'GIT_SYNC_DEPTH', 'value': '1', 'value_from': None},
+ {'name': 'GIT_SYNC_ROOT', 'value': '/git', 'value_from': None},
+ {'name': 'GIT_SYNC_DEST', 'value': 'repo', 'value_from': None},
+ {'name': 'GIT_SYNC_ADD_USER', 'value': 'true', 'value_from': None},
+ {'name': 'GIT_SYNC_WAIT', 'value': '60', 'value_from': None},
+ {'name': 'GIT_SYNC_MAX_SYNC_FAILURES', 'value': '0', 'value_from': None},
+ ],
+ )
+
+ def test_set_volume_claim_to_existing_claim(self):
+ helm_settings = yaml.safe_load(git_sync_existing_claim)
+ res = render_chart(
+ 'GIT-SYNC', helm_settings, show_only=["templates/scheduler/scheduler-deployment.yaml"]
+ )
+ dep: k8s.V1Deployment = render_k8s_object(res[0], k8s.V1Deployment)
+ volume_map = {vol.name: vol for vol in dep.spec.template.spec.volumes}
+ dag_volume = volume_map['dags']
+ self.assertEqual(
+ dag_volume,
+ k8s.V1Volume(
+ name="dags",
+ persistent_volume_claim=k8s.V1PersistentVolumeClaimVolumeSource(claim_name='test-claim'),
+ ),
+ )
diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml
index 37884f4..03822bd 100644
--- a/scripts/ci/docker-compose/local.yml
+++ b/scripts/ci/docker-compose/local.yml
@@ -55,6 +55,7 @@ services:
- ../../../setup.py:/opt/airflow/setup.py:cached
- ../../../tests:/opt/airflow/tests:cached
- ../../../kubernetes_tests:/opt/airflow/kubernetes_tests:cached
+ - ../../../chart:/opt/airflow/chart:cached
- ../../../tmp:/tmp:cached
- ../../../metastore_browser:/opt/airflow/metastore_browser:cached
# END automatically generated volumes from LOCAL_MOUNTS in _local_mounts.sh
diff --git a/scripts/ci/libraries/_local_mounts.sh b/scripts/ci/libraries/_local_mounts.sh
index 431c7c9..93676b6 100644
--- a/scripts/ci/libraries/_local_mounts.sh
+++ b/scripts/ci/libraries/_local_mounts.sh
@@ -51,6 +51,7 @@ function local_mounts::generate_local_mounts_list {
"$prefix"setup.py:/opt/airflow/setup.py:cached
"$prefix"tests:/opt/airflow/tests:cached
"$prefix"kubernetes_tests:/opt/airflow/kubernetes_tests:cached
+ "$prefix"chart:/opt/airflow/chart:cached
"$prefix"tmp:/tmp:cached
"$prefix"metastore_browser:/opt/airflow/metastore_browser:cached
)
diff --git a/scripts/in_container/entrypoint_ci.sh b/scripts/in_container/entrypoint_ci.sh
index ea133f3..ccfb2d1 100755
--- a/scripts/in_container/entrypoint_ci.sh
+++ b/scripts/in_container/entrypoint_ci.sh
@@ -185,7 +185,6 @@ if [[ "${GITHUB_ACTIONS}" == "true" ]]; then
# timeouts in seconds for individual tests
"--setup-timeout=20"
"--execution-timeout=60"
- "--with-db-init"
"--teardown-timeout=20"
# Only display summary for non-expected case
# f - failed
@@ -197,7 +196,12 @@ if [[ "${GITHUB_ACTIONS}" == "true" ]]; then
# p - passed
# P - passed with output
"-rfEX"
+ )
+ if [[ "${TEST_TYPE}" != "Helm" ]]; then
+ EXTRA_PYTEST_ARGS+=(
+ "--with-db-init"
)
+ fi
else
EXTRA_PYTEST_ARGS=(
"-rfEX"
@@ -242,6 +246,7 @@ else
"tests/utils"
)
WWW_TESTS=("tests/www")
+ HELM_CHART_TESTS=("chart/tests")
ALL_TESTS=("tests")
ALL_PRESELECTED_TESTS=(
"${CLI_TESTS[@]}"
@@ -261,6 +266,8 @@ else
SELECTED_TESTS=("${CORE_TESTS[@]}")
elif [[ ${TEST_TYPE:=""} == "WWW" ]]; then
SELECTED_TESTS=("${WWW_TESTS[@]}")
+ elif [[ ${TEST_TYPE:=""} == "Helm" ]]; then
+ SELECTED_TESTS=("${HELM_CHART_TESTS[@]}")
elif [[ ${TEST_TYPE:=""} == "Other" ]]; then
find_all_other_tests
SELECTED_TESTS=("${ALL_OTHER_TESTS[@]}")
diff --git a/setup.cfg b/setup.cfg
index 0b9620e..9cb0fa9 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -77,4 +77,4 @@ multi_line_output=5
skip=build,.tox,venv
# ToDo: Enable the below before Airflow 2.0
# profile = "black"
-skip_glob=*/api_connexion/**/*.py,*/providers/**/*.py,provider_packages/**
+skip_glob=*/api_connexion/**/*.py,*/providers/**/*.py,provider_packages/**,chart/tests/*.py
diff --git a/tests/airflow_pylint/disable_checks_for_tests.py b/tests/airflow_pylint/disable_checks_for_tests.py
index ddaae0b..e7b23b2 100644
--- a/tests/airflow_pylint/disable_checks_for_tests.py
+++ b/tests/airflow_pylint/disable_checks_for_tests.py
@@ -44,7 +44,8 @@ def transform(mod):
"""
if mod.name.startswith("test_") or \
mod.name.startswith("tests.") or \
- mod.name.startswith("kubernetes_tests."):
+ mod.name.startswith("kubernetes_tests.") or \
+ mod.name.startswith("chart."):
decoded_lines = mod.stream().read().decode("utf-8").split("\n")
if decoded_lines[0].startswith("# pylint: disable="):
decoded_lines[0] = decoded_lines[0] + " " + DISABLED_CHECKS_FOR_TESTS