You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2023/03/07 20:05:21 UTC
[airflow] 02/02: Make skipping provider tests part of selective-checks (#29216)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-5-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 8ab8ac84b5065ff921e6ff9f7c73abc3236ee376
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Jan 29 14:45:25 2023 +0100
Make skipping provider tests part of selective-checks (#29216)
The "SKIP_PROVIDER_TESTS" was added as quick-fix when releasing
2.5.0 release, now we can make it a bit more consistent with
all other selective checks. It has now unit tests covering it
and it is produced as selective-checks output so that we do not
have to encode the logic in conditions in the ci.yml.
(cherry picked from commit e5d51fc50f6aad0f35f775367bc700708fe6d62b)
---
.github/workflows/ci.yml | 5 +++--
CI.rst | 2 ++
dev/breeze/src/airflow_breeze/utils/docker_command_utils.py | 2 ++
dev/breeze/src/airflow_breeze/utils/selective_checks.py | 4 ++++
dev/breeze/tests/test_selective_checks.py | 7 +++++++
scripts/ci/docker-compose/devcontainer.env | 1 +
6 files changed, 19 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f6b1e82442..63138de93f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -149,6 +149,7 @@ jobs:
mssql-exclude: ${{ steps.selective-checks.outputs.mssql-exclude }}
sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
providers-package-format-exclude: ${{ steps.selective-checks.outputs.providers-package-format-exclude }}
+ skip-provider-tests: ${{ steps.source-run-info.outputs.skip-provider-tests }}
run-tests: ${{ steps.selective-checks.outputs.run-tests }}
run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
@@ -1008,7 +1009,7 @@ jobs:
BACKEND_VERSION: "${{needs.build-info.outputs.default-python-version}}"
JOB_ID: "integration"
COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
- SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.default-branch != 'main'}}"
+ SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}"
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
@@ -1068,7 +1069,7 @@ jobs:
BACKEND_VERSION: "${{needs.build-info.outputs.default-python-version}}"
JOB_ID: "integration"
COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
- SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.default-branch != 'main'}}"
+ SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}"
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
diff --git a/CI.rst b/CI.rst
index 5c937e5794..178e536376 100644
--- a/CI.rst
+++ b/CI.rst
@@ -535,6 +535,8 @@ those via command line flags passed to ``breeze`` command.
| | | | | |
| | | | | \* set to true in pre-commits |
+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
+| ``SKIP_PROVIDER_TESTS`` | false\* | false\* | false\* | Skip running provider integration tests |
++-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
| ``SKIP_SSH_SETUP`` | false\* | false\* | false\* | Skip setting up SSH server for tests. |
| | | | | |
| | | | | \* set to true in GitHub CodeSpaces |
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index 7670b5bbdd..b990e44169 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -614,6 +614,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
set_value_to_default_if_not_set(env, "RUN_SYSTEM_TESTS", "false")
set_value_to_default_if_not_set(env, "RUN_TESTS", "false")
set_value_to_default_if_not_set(env, "SKIP_ENVIRONMENT_INITIALIZATION", "false")
+ set_value_to_default_if_not_set(env, "SKIP_PROVIDER_TESTS", "false")
set_value_to_default_if_not_set(env, "SKIP_SSH_SETUP", "false")
set_value_to_default_if_not_set(env, "TEST_TYPE", "")
set_value_to_default_if_not_set(env, "TEST_TIMEOUT", "60")
@@ -656,6 +657,7 @@ DERIVE_ENV_VARIABLES_FROM_ATTRIBUTES = {
"PYTHON_MAJOR_MINOR_VERSION": "python",
"SKIP_CONSTRAINTS": "skip_constraints",
"SKIP_ENVIRONMENT_INITIALIZATION": "skip_environment_initialization",
+ "SKIP_PROVIDER_TESS": "skip_provider_tests",
"SQLITE_URL": "sqlite_url",
"START_AIRFLOW": "start_airflow",
"USE_AIRFLOW_VERSION": "use_airflow_version",
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index eeaf65150e..2aee00491b 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -590,6 +590,10 @@ class SelectiveChecks:
def skip_pre_commits(self) -> str:
return "identity" if self._default_branch == "main" else "identity,check-airflow-2-2-compatibility"
+ @cached_property
+ def skip_provider_tests(self) -> bool:
+ return self._default_branch != "main"
+
@cached_property
def cache_directive(self) -> str:
return "disabled" if self._github_event == GithubEvents.SCHEDULE else "registry"
diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py
index 27333dd5c0..5bf2ac41c3 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -398,6 +398,7 @@ def test_expected_output_full_tests_needed(
"full-tests-needed": "false",
"providers-package-format-exclude": "[{'package-format': 'sdist'}]",
"upgrade-to-newer-dependencies": "false",
+ "skip-provider-tests": "true",
"test-types": "",
},
id="Nothing should run if only non-important files changed",
@@ -418,6 +419,7 @@ def test_expected_output_full_tests_needed(
"providers-package-format-exclude": "[{'package-format': 'sdist'}]",
"run-kubernetes-tests": "true",
"upgrade-to-newer-dependencies": "false",
+ "skip-provider-tests": "true",
"test-types": "Always",
},
id="No Helm tests, No providers should run if only chart/providers changed in non-main",
@@ -439,6 +441,7 @@ def test_expected_output_full_tests_needed(
"providers-package-format-exclude": "[{'package-format': 'sdist'}]",
"run-kubernetes-tests": "true",
"upgrade-to-newer-dependencies": "false",
+ "skip-provider-tests": "true",
"test-types": "Always CLI",
},
id="Only CLI tests and Kubernetes tests should run if cli/chart files changed in non-main branch",
@@ -459,6 +462,7 @@ def test_expected_output_full_tests_needed(
"providers-package-format-exclude": "[{'package-format': 'sdist'}]",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
+ "skip-provider-tests": "true",
"test-types": "API Always CLI Core Other WWW",
},
id="All tests except Providers and should run if core file changed in non-main branch",
@@ -492,6 +496,7 @@ def test_expected_output_pull_request_v2_3(
"run-tests": "false",
"docs-build": "false",
"upgrade-to-newer-dependencies": "false",
+ "skip-provider-tests": "false",
"test-types": "",
},
id="Nothing should run if only non-important files changed",
@@ -511,6 +516,7 @@ def test_expected_output_pull_request_v2_3(
"docs-build": "true",
"run-kubernetes-tests": "true",
"upgrade-to-newer-dependencies": "false",
+ "skip-provider-tests": "false",
"test-types": "Always CLI",
},
id="CLI tests and Kubernetes tests should run if cli/chart files changed",
@@ -529,6 +535,7 @@ def test_expected_output_pull_request_v2_3(
"docs-build": "true",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
+ "skip-provider-tests": "false",
"test-types": "API Always CLI Core Other Providers WWW",
},
id="All tests except should run if core file changed",
diff --git a/scripts/ci/docker-compose/devcontainer.env b/scripts/ci/docker-compose/devcontainer.env
index e0f326612b..c3ca6e8420 100644
--- a/scripts/ci/docker-compose/devcontainer.env
+++ b/scripts/ci/docker-compose/devcontainer.env
@@ -59,6 +59,7 @@ START_AIRFLOW="false"
SKIP_CONSTRAINTS="false"
SKIP_SSH_SETUP="true"
SKIP_ENVIRONMENT_INITIALIZATION="false"
+SKIP_PROVIDER_TESTS="false"
TEST_TYPE=
UPGRADE_TO_NEWER_DEPENDENCIES="false"
VERBOSE="false"