You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2024/07/14 06:22:55 UTC
(airflow) branch main updated: Do not trigger provider tests on core changes (#40776)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ee97d7b6f5 Do not trigger provider tests on core changes (#40776)
ee97d7b6f5 is described below
commit ee97d7b6f55f37c3dde46cd86cbd69152c78d6b6
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Jul 14 08:22:46 2024 +0200
Do not trigger provider tests on core changes (#40776)
Provider tests have been triggered when core airflow changed. This
has been done in order to avoid some core changes that would break
providers, however running provider tests - especially including
compatibility tests takes a long time now and especially after we
switched the committer PRs to run using public runners - adding
compatibility tests to be run whenever provider tests ar run also
does not help with speed.
Instead we should defer to the "canary" build to catch such changes
and rather than actively check all providers in PRs, the "canary"
builds should be enough to catch any of such changes.
---
dev/breeze/src/airflow_breeze/global_constants.py | 5 +++
.../src/airflow_breeze/utils/selective_checks.py | 12 ++++++--
dev/breeze/tests/test_selective_checks.py | 36 +++++++++++-----------
3 files changed, 32 insertions(+), 21 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py
index b2df6219db..bcf3b9b159 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -161,6 +161,11 @@ def all_selective_test_types() -> tuple[str, ...]:
return tuple(sorted(e.value for e in SelectiveUnitTestTypes))
+@lru_cache(maxsize=None)
+def all_selective_test_types_except_providers() -> tuple[str, ...]:
+ return tuple(sorted(e.value for e in SelectiveUnitTestTypes if e != SelectiveUnitTestTypes.PROVIDERS))
+
+
class SelectiveUnitTestTypes(Enum):
ALWAYS = "Always"
API = "API"
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 7e04449c1b..d31c6005d7 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -50,6 +50,7 @@ from airflow_breeze.global_constants import (
SelectiveUnitTestTypes,
all_helm_test_packages,
all_selective_test_types,
+ all_selective_test_types_except_providers,
)
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.exclude_from_matrix import excluded_combos
@@ -85,6 +86,10 @@ ALL_CI_SELECTIVE_TEST_TYPES = (
"PythonVenv Serialization WWW"
)
+ALL_CI_SELECTIVE_TEST_TYPES_WITHOUT_PROVIDERS = (
+ "API Always BranchExternalPython BranchPythonVenv CLI Core "
+ "ExternalPython Operators Other PlainAsserts PythonVenv Serialization WWW"
+)
ALL_PROVIDERS_SELECTIVE_TEST_TYPES = "Providers[-amazon,google] Providers[amazon] Providers[google]"
@@ -744,11 +749,12 @@ class SelectiveChecks:
break
if count_remaining_files > 0:
get_console().print(
- f"[warning]We should run all tests. There are {count_remaining_files} changed "
- "files that seems to fall into Core/Other category[/]"
+ f"[warning]We should run all core tests except providers."
+ f"There are {count_remaining_files} changed files that seems to fall "
+ f"into Core/Other category[/]"
)
get_console().print(remaining_files)
- candidate_test_types.update(all_selective_test_types())
+ candidate_test_types.update(all_selective_test_types_except_providers())
else:
if "Providers" in candidate_test_types or "API" in candidate_test_types:
affected_providers = self._find_all_providers_affected(
diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py
index e8031df61d..513edeff5f 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -33,6 +33,7 @@ from airflow_breeze.global_constants import (
from airflow_breeze.utils.packages import get_available_packages
from airflow_breeze.utils.selective_checks import (
ALL_CI_SELECTIVE_TEST_TYPES,
+ ALL_CI_SELECTIVE_TEST_TYPES_WITHOUT_PROVIDERS,
ALL_PROVIDERS_SELECTIVE_TEST_TYPES,
SelectiveChecks,
)
@@ -1279,7 +1280,6 @@ def test_expected_output_pull_request_v2_7(
pytest.param(
("airflow/models/test.py",),
{
- "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED,
"all-python-versions": "['3.8']",
"all-python-versions-list-as-string": "3.8",
"ci-image-build": "true",
@@ -1287,21 +1287,20 @@ def test_expected_output_pull_request_v2_7(
"needs-helm-tests": "false",
"run-tests": "true",
"docs-build": "true",
- "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD,
+ "docs-list-as-string": "apache-airflow",
"skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
- "skip-provider-tests": "false",
- "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES,
+ "skip-provider-tests": "true",
+ "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES_WITHOUT_PROVIDERS,
"needs-mypy": "true",
- "mypy-folders": "['airflow', 'providers']",
+ "mypy-folders": "['airflow']",
},
- id="Tests for all providers should run if model file changed",
+ id="Tests for all airflow core types except providers should run if model file changed",
),
pytest.param(
("airflow/file.py",),
{
- "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED,
"all-python-versions": "['3.8']",
"all-python-versions-list-as-string": "3.8",
"ci-image-build": "true",
@@ -1309,16 +1308,17 @@ def test_expected_output_pull_request_v2_7(
"needs-helm-tests": "false",
"run-tests": "true",
"docs-build": "true",
- "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD,
+ "docs-list-as-string": "apache-airflow",
"skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
- "skip-provider-tests": "false",
- "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES,
+ "skip-provider-tests": "true",
+ "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES_WITHOUT_PROVIDERS,
"needs-mypy": "true",
- "mypy-folders": "['airflow', 'providers']",
+ "mypy-folders": "['airflow']",
},
- id="Tests for all providers should run if any other than API/WWW/CLI/Operators file changed.",
+ id="Tests for all airflow core types except providers should run if "
+ "any other than API/WWW/CLI/Operators file changed.",
),
],
)
@@ -1601,9 +1601,9 @@ def test_upgrade_to_newer_dependencies(
pytest.param(
("airflow/test.py",),
{
- "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD,
+ "docs-list-as-string": "apache-airflow",
},
- id="Core files changed. All provider docs should also be built",
+ id="Core files changed. Apache-Airflow docs should also be built",
),
pytest.param(
("docs/docker-stack/test.rst",),
@@ -1613,9 +1613,9 @@ def test_upgrade_to_newer_dependencies(
pytest.param(
("airflow/test.py", "chart/airflow/values.yaml"),
{
- "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD,
+ "docs-list-as-string": "apache-airflow helm-chart",
},
- id="Core files and helm chart files changed. All provider docs should be built",
+ id="Core files and helm chart files changed. Apache Airflow and helm chart docs to build",
),
pytest.param(
("chart/airflow/values.yaml",),
@@ -2033,11 +2033,11 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs
("airflow/models/file.py",),
{
"needs-mypy": "true",
- "mypy-folders": "['airflow', 'providers']",
+ "mypy-folders": "['airflow']",
},
"main",
(),
- id="Airflow mypy checks on airflow files that can trigger provider tests",
+ id="Airflow mypy checks on airflow files with model changes.",
),
pytest.param(
("airflow/providers/a_file.py",),