You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2021/05/17 21:50:44 UTC
[airflow] branch master updated: Use different executors for Helm
Chart tests in CI (#15791)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 2a7298d Use different executors for Helm Chart tests in CI (#15791)
2a7298d is described below
commit 2a7298d6fc8a4f587ce51d9a2f75aa23231343c3
Author: Ephraim Anierobi <sp...@gmail.com>
AuthorDate: Mon May 17 22:50:23 2021 +0100
Use different executors for Helm Chart tests in CI (#15791)
closes https://github.com/apache/airflow/issues/14301
---
.github/workflows/ci.yml | 6 +++++-
kubernetes_tests/test_kubernetes_pod_operator.py | 2 ++
kubernetes_tests/test_kubernetes_pod_operator_backcompat.py | 5 ++++-
.../kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh | 1 +
.../ci_setup_cluster_and_run_kubernetes_tests_single_job.sh | 1 +
.../ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh | 1 -
scripts/ci/libraries/_initialization.sh | 4 ++--
scripts/ci/libraries/_kind.sh | 4 ----
8 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9ba110a..67526f5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -970,15 +970,19 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-kubernetes:
timeout-minutes: 50
- name: K8s tests
+ name: Helm Chart; ${{matrix.executor}}
runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
needs: [build-info, prod-images]
+ strategy:
+ matrix:
+ executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor]
env:
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
BACKEND: postgres
RUN_TESTS: "true"
RUNTIME: "kubernetes"
KUBERNETES_MODE: "image"
+ EXECUTOR: ${{matrix.executor}}
KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }}
diff --git a/kubernetes_tests/test_kubernetes_pod_operator.py b/kubernetes_tests/test_kubernetes_pod_operator.py
index 48e77aa..fbed49f 100644
--- a/kubernetes_tests/test_kubernetes_pod_operator.py
+++ b/kubernetes_tests/test_kubernetes_pod_operator.py
@@ -41,6 +41,7 @@ from airflow.providers.cncf.kubernetes.utils.pod_launcher import PodLauncher
from airflow.providers.cncf.kubernetes.utils.xcom_sidecar import PodDefaults
from airflow.utils import timezone
from airflow.version import version as airflow_version
+from kubernetes_tests.test_base import EXECUTOR
def create_context(task):
@@ -63,6 +64,7 @@ def get_kubeconfig_path():
return kubeconfig_path if kubeconfig_path else os.path.expanduser('~/.kube/config')
+@pytest.mark.skipif(EXECUTOR != 'KubernetesExecutor', reason="Only runs on KubernetesExecutor")
class TestKubernetesPodOperatorSystem(unittest.TestCase):
def get_current_task_name(self):
# reverse test name to make pod name unique (it has limited length)
diff --git a/kubernetes_tests/test_kubernetes_pod_operator_backcompat.py b/kubernetes_tests/test_kubernetes_pod_operator_backcompat.py
index 5facd47..07cd167 100644
--- a/kubernetes_tests/test_kubernetes_pod_operator_backcompat.py
+++ b/kubernetes_tests/test_kubernetes_pod_operator_backcompat.py
@@ -42,9 +42,11 @@ from airflow.providers.cncf.kubernetes.utils.xcom_sidecar import PodDefaults
from airflow.utils import timezone
from airflow.utils.state import State
from airflow.version import version as airflow_version
-
+from kubernetes_tests.test_base import EXECUTOR
# noinspection DuplicatedCode
+
+
def create_context(task):
dag = DAG(dag_id="dag")
tzinfo = pendulum.timezone("Europe/Amsterdam")
@@ -61,6 +63,7 @@ def create_context(task):
# noinspection DuplicatedCode,PyUnusedLocal
+@pytest.mark.skipif(EXECUTOR != 'KubernetesExecutor', reason="Only runs on KubernetesExecutor")
class TestKubernetesPodOperatorSystem(unittest.TestCase):
def get_current_task_name(self):
# reverse test name to make pod name unique (it has limited length)
diff --git a/scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh b/scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh
index 1e0fa36..fce0a4c 100755
--- a/scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh
+++ b/scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh
@@ -16,6 +16,7 @@
# specific language governing permissions and limitations
# under the License.
+
export SKIP_BUILDING_PROD_IMAGE="true"
# shellcheck source=scripts/ci/libraries/_script_init.sh
diff --git a/scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh b/scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh
index 9b0d86f..914e426 100755
--- a/scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh
+++ b/scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh
@@ -30,6 +30,7 @@ fi
export PYTHON_MAJOR_MINOR_VERSION=$1
shift
+
# Requires PARALLEL_JOB_STATUS
if [[ -z "${PARALLEL_JOB_STATUS=}" ]]; then
diff --git a/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh b/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh
index 88aa2cd..5790b94 100755
--- a/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh
+++ b/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh
@@ -101,6 +101,5 @@ parallel::make_sure_kubernetes_versions_are_specified
get_maximum_parallel_k8s_jobs
run_k8s_tests_in_parallel "${@}"
-
# this will exit with error code in case some of the tests failed
parallel::print_job_summary_and_return_status_code
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index 3c5a4d3..72a8a43 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -25,6 +25,7 @@ CURRENT_POSTGRES_VERSIONS=()
CURRENT_MYSQL_VERSIONS=()
CURRENT_KIND_VERSIONS=()
CURRENT_HELM_VERSIONS=()
+CURRENT_EXECUTOR=()
ALL_PYTHON_MAJOR_MINOR_VERSIONS=()
INSTALLED_PROVIDERS=()
@@ -793,9 +794,8 @@ function initialization::make_constants_read_only() {
readonly KUBERNETES_VERSION
readonly KIND_VERSION
readonly HELM_VERSION
- readonly EXECUTOR
readonly KUBECTL_VERSION
-
+ readonly EXECUTOR
readonly POSTGRES_VERSION
readonly MYSQL_VERSION
diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh
index cc42bf7..b38ad17 100644
--- a/scripts/ci/libraries/_kind.sh
+++ b/scripts/ci/libraries/_kind.sh
@@ -131,10 +131,6 @@ function kind::perform_kind_cluster_operation() {
echo "Kubernetes mode: ${KUBERNETES_MODE}"
echo
- echo
- echo "Executor: ${EXECUTOR}"
- echo
-
if [[ ${OPERATION} == "status" ]]; then
if [[ ${ALL_CLUSTERS} == *"${KIND_CLUSTER_NAME}"* ]]; then
echo