You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/07/01 10:46:26 UTC
[airflow] branch main updated: Fix selective checks to work for non-main-branch (#24769)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new c8d3850fce Fix selective checks to work for non-main-branch (#24769)
c8d3850fce is described below
commit c8d3850fce4be19b998db35371657227f55ba6a9
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Fri Jul 1 12:46:08 2022 +0200
Fix selective checks to work for non-main-branch (#24769)
The recent changes in selective checks introduced a few problems
in non-main branches:
* constraints branch was not used by CI build in non-main branch
this was not a proble, (until it started to fail) because
other branches always work in "upgradeDepencies" case
* default branch calculation did not account for the selective-check
comment that was run before default branch was retrieved. Therefore
we did not run build as from 2.3 branch
* Some precomits should be skiped in non-main branch
* Integration was also excluded from Provider Check
* Two more commit outputs added:
* docs-filter - depends whether you are with more packages
* skip-pre-commit - allows to dynamically choose which pre-commits
should be skipped
---
.github/workflows/ci.yml | 87 ++++++++--------------
.../src/airflow_breeze/params/doc_build_params.py | 4 +
.../src/airflow_breeze/utils/selective_checks.py | 18 +++++
dev/breeze/tests/test_selective_checks.py | 14 ++--
docs/build_docs.py | 9 ++-
docs/conf.py | 2 +
docs/exts/docs_build/lint_checks.py | 8 +-
7 files changed, 76 insertions(+), 66 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5e9534defb..30a44fa000 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -126,6 +126,8 @@ jobs:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
defaultBranch: ${{ steps.selective-checks.outputs.default-branch }}
+ defaultConstraintsBranch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
+ debianVersion: ${{ steps.selective-checks.outputs.debian-version }}
cacheDirective: ${{ steps.dynamic-outputs.outputs.cacheDirective }}
waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }}
allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }}
@@ -164,6 +166,8 @@ jobs:
needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }}
default-branch: ${{ steps.selective-checks.outputs.default-branch }}
+ docs-filter: ${{ steps.selective-checks.outputs.docs-filter }}
+ skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }}
sourceHeadRepo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }}
pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }}
@@ -191,7 +195,6 @@ jobs:
ref: ${{ github.sha }}
fetch-depth: 2
persist-credentials: false
- if: github.event_name == 'pull_request'
- name: "Setup python"
uses: actions/setup-python@v4
with:
@@ -200,6 +203,22 @@ jobs:
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
+ - name: "Retrieve DEFAULTS from the _initialization.sh"
+ # We cannot "source" the script here because that would be a security problem (we cannot run
+ # any code that comes from the sources coming from the PR. Therefore, we extract the
+ # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
+ id: defaults
+ run: |
+ DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
+ awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+ echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
+ DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
+ scripts/ci/libraries/_initialization.sh | \
+ awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+ echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
+ DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
+ awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+ echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
- name: Selective checks
id: selective-checks
env:
@@ -294,6 +313,9 @@ jobs:
runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
needs: [build-info]
env:
+ DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+ DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
steps:
- name: Cleanup repo
@@ -310,23 +332,6 @@ jobs:
with:
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore, we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
- id: defaults
- run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
- DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
- run: ./scripts/ci/install_breeze.sh
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- name: "Free space"
@@ -361,6 +366,9 @@ jobs:
runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
needs: [build-info, build-ci-images]
env:
+ DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+ DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
BACKEND: sqlite
DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
@@ -380,23 +388,6 @@ jobs:
with:
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
- id: defaults
- run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
- DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
- run: ./scripts/ci/install_breeze.sh
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- name: "Free space"
@@ -644,7 +635,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: breeze static-checks --all-files --show-diff-on-failure --color always
env:
VERBOSE: "false"
- SKIP: "identity"
+ SKIP: ${{ needs.build-info.outputs.skip-pre-commits }}
COLUMNS: "250"
- name: "Fix ownership"
run: breeze fix-ownership
@@ -701,7 +692,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
env:
VERBOSE: "false"
SKIP_IMAGE_PRE_COMMITS: "true"
- SKIP: "identity"
+ SKIP: ${{ needs.build-info.outputs.skip-pre-commits }}
COLUMNS: "250"
- name: "Fix ownership"
run: breeze fix-ownership
@@ -745,7 +736,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
docs-inventory-${{ hashFiles('setup.py','setup.cfg','pyproject.toml;') }}
docs-inventory-
- name: "Build docs"
- run: breeze build-docs
+ run: breeze build-docs ${{ needs.build-info.outputs.docs-filter }}
- name: Configure AWS credentials
uses: ./.github/actions/configure-aws-credentials
if: >
@@ -1766,6 +1757,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- tests-mssql
- tests-postgres
env:
+ DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+ DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
if: needs.build-info.outputs.upgradeToNewerDependencies != 'false'
steps:
@@ -1783,23 +1777,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore, we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
- id: defaults
- run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
- DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
- run: ./scripts/ci/install_breeze.sh
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- name: "Free space"
diff --git a/dev/breeze/src/airflow_breeze/params/doc_build_params.py b/dev/breeze/src/airflow_breeze/params/doc_build_params.py
index 25350db93b..4ac4c3ebca 100644
--- a/dev/breeze/src/airflow_breeze/params/doc_build_params.py
+++ b/dev/breeze/src/airflow_breeze/params/doc_build_params.py
@@ -19,6 +19,8 @@ import os
from dataclasses import dataclass
from typing import List, Tuple
+from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
+
@dataclass
class DocBuildParams:
@@ -38,6 +40,8 @@ class DocBuildParams:
doc_args.append("--spellcheck-only")
if self.for_production:
doc_args.append("--for-production")
+ if AIRFLOW_BRANCH != "main":
+ doc_args.append("--disable-provider-checks")
if self.package_filter and len(self.package_filter) > 0:
for single_filter in self.package_filter:
doc_args.extend(["--package-filter", single_filter])
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index b2d018ddd3..8c2302292e 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -469,6 +469,12 @@ class SelectiveChecks:
f"is {self._default_branch} and not main[/]"
)
current_test_types.remove("Providers")
+ if "Integration" in current_test_types:
+ get_console().print(
+ "[warning]Removing 'Integration' because the target branch "
+ f"is {self._default_branch} and not main[/]"
+ )
+ current_test_types.remove("Integration")
return " ".join(sorted(current_test_types))
@cached_property
@@ -480,3 +486,15 @@ class SelectiveChecks:
return len(
self._matching_files(FileGroupForCi.SETUP_FILES, CI_FILE_GROUP_MATCHES)
) > 0 or self._github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE]
+
+ @cached_property
+ def docs_filter(self) -> str:
+ return (
+ ""
+ if self._default_branch == 'main'
+ else "--package-filter apache-airflow --package-filter docker-stack"
+ )
+
+ @cached_property
+ def skip_pre_commits(self) -> str:
+ return "identity" if self._default_branch == "main" else "identity,check-airflow-2-2-compatibility"
diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py
index 8b4a8e8d4d..e5c8321fe5 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -275,9 +275,10 @@ def test_expected_output_pull_request_main(
"run-tests": "true",
"docs-build": "true",
"upgrade-to-newer-dependencies": "false",
- "test-types": "API Always CLI Core Integration Other WWW",
+ "test-types": "API Always CLI Core Other WWW",
},
- id="Everything should run except Providers when full tests are needed for non-main branch",
+ id="Everything should run except Providers and Integration "
+ "when full tests are needed for non-main branch",
)
),
],
@@ -367,9 +368,10 @@ def test_expected_output_full_tests_needed(
"docs-build": "true",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
- "test-types": "API Always CLI Core Integration Other WWW",
+ "test-types": "API Always CLI Core Other WWW",
},
- id="All tests except providers should run if core file changed in non-main branch",
+ id="All tests except providers and Integration should "
+ "run if core file changed in non-main branch",
),
],
)
@@ -488,9 +490,9 @@ def test_expected_output_pull_request_target(
"run-tests": "true",
"docs-build": "true",
"upgrade-to-newer-dependencies": "true",
- "test-types": "API Always CLI Core Integration Other WWW",
+ "test-types": "API Always CLI Core Other WWW",
},
- id="All tests except Providers and Helm run on push"
+ id="All tests except Providers Integration and Helm run on push"
" even if unimportant file changed in non-main branch",
),
pytest.param(
diff --git a/docs/build_docs.py b/docs/build_docs.py
index 9f4217bd10..5d8812ea3a 100755
--- a/docs/build_docs.py
+++ b/docs/build_docs.py
@@ -107,6 +107,12 @@ def _get_parser():
parser.add_argument(
'--disable-checks', dest='disable_checks', action='store_true', help='Disables extra checks'
)
+ parser.add_argument(
+ '--disable-provider-checks',
+ dest='disable_provider_checks',
+ action='store_true',
+ help='Disables extra checks for providers',
+ )
parser.add_argument(
'--one-pass-only',
dest='one_pass_only',
@@ -441,6 +447,7 @@ def main():
available_packages = get_available_packages()
docs_only = args.docs_only
spellcheck_only = args.spellcheck_only
+ disable_provider_checks = args.disable_provider_checks
disable_checks = args.disable_checks
package_filters = args.package_filter
for_production = args.for_production
@@ -531,7 +538,7 @@ def main():
)
if not disable_checks:
- general_errors = lint_checks.run_all_check()
+ general_errors = lint_checks.run_all_check(disable_provider_checks=disable_provider_checks)
if general_errors:
all_build_errors[None] = general_errors
diff --git a/docs/conf.py b/docs/conf.py
index 72080b43cb..b75377ddc7 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -694,6 +694,8 @@ autoapi_ignore = [
]
if PACKAGE_NAME == 'apache-airflow':
autoapi_ignore.append('*/airflow/providers/*')
+elif PACKAGE_NAME == 'docker-stack':
+ autoapi_ignore.append('*/airflow/providers/*')
else:
autoapi_ignore.append('*/airflow/providers/cncf/kubernetes/backcompat/*')
# Keep the AutoAPI generated files on the filesystem after the run.
diff --git a/docs/exts/docs_build/lint_checks.py b/docs/exts/docs_build/lint_checks.py
index b29f091130..d8e213babf 100644
--- a/docs/exts/docs_build/lint_checks.py
+++ b/docs/exts/docs_build/lint_checks.py
@@ -323,13 +323,13 @@ def check_pypi_repository_in_provider_tocs() -> List[DocBuildError]:
return build_errors
-def run_all_check() -> List[DocBuildError]:
+def run_all_check(disable_provider_checks: bool = False) -> List[DocBuildError]:
"""Run all checks from this module"""
general_errors = []
general_errors.extend(check_guide_links_in_operator_descriptions())
general_errors.extend(check_enforce_code_block())
general_errors.extend(check_exampleinclude_for_example_dags())
- general_errors.extend(check_example_dags_in_provider_tocs())
- general_errors.extend(check_pypi_repository_in_provider_tocs())
-
+ if not disable_provider_checks:
+ general_errors.extend(check_pypi_repository_in_provider_tocs())
+ general_errors.extend(check_example_dags_in_provider_tocs())
return general_errors