You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2022/06/29 15:19:46 UTC
[airflow] 01/45: Fix selective checks to work for non-main-branch
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 6ceb7e245edd24ef4ffd70c233e15c2384a0e9d6
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Wed Jun 29 12:39:47 2022 +0200
Fix selective checks to work for non-main-branch
---
.github/workflows/build-images.yml | 88 +++++--------
.github/workflows/ci.yml | 82 ++++--------
.../src/airflow_breeze/commands/ci_commands.py | 13 ++
.../src/airflow_breeze/utils/selective_checks.py | 25 +++-
dev/breeze/tests/test_selective_checks.py | 23 +++-
images/breeze/output-commands-hash.txt | 2 +-
images/breeze/output-selective-check.svg | 116 ++++++++--------
tests/always/test_project_structure.py | 138 -------------------
.../google}/test_project_structure.py | 146 +--------------------
9 files changed, 190 insertions(+), 443 deletions(-)
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index e6bb57c40d..2d895e3c09 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -52,6 +52,9 @@ jobs:
runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' || 'ubuntu-20.04' }}
env:
targetBranch: ${{ github.event.pull_request.base.ref }}
+ DEFAULT_BRANCH: ${{ steps.selective-checks.outputs.default-constraints-branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ steps.selective-checks.outputs.default-constraints-branch }}
+ DEBIAN_VERSION: ${{ steps.selective-checks.outputs.debian-version }}
outputs:
runsOn: ${{ github.repository == 'apache/airflow' && '["self-hosted"]' || '["ubuntu-20.04"]' }}
pythonVersions: "${{ steps.selective-checks.python-versions }}"
@@ -65,6 +68,8 @@ jobs:
cacheDirective: ${{ steps.dynamic-outputs.outputs.cacheDirective }}
targetBranch: ${{ steps.dynamic-outputs.outputs.targetBranch }}
defaultBranch: ${{ steps.selective-checks.outputs.default-branch }}
+ defaultConstraintsBranch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
+ debianVersion: ${{ steps.selective-checks.outputs.debian-version }}
targetCommitSha: "${{steps.discover-pr-merge-commit.outputs.targetCommitSha ||
github.event.pull_request.head.sha ||
github.sha
@@ -104,9 +109,27 @@ jobs:
ref: ${{ env.TARGET_COMMIT_SHA }}
persist-credentials: false
fetch-depth: 2
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ - name: "Retrieve DEFAULTS from the _initialization.sh"
+ # We cannot "source" the script here because that would be a security problem (we cannot run
+ # any code that comes from the sources coming from the PR. Therefore we extract the
+ # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
+ id: defaults
+ run: |
+ DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
+ awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+ echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
+ DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
+ scripts/ci/libraries/_initialization.sh | \
+ awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+ echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
+ DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
+ cut -d "=" -f 3 | sed s'/["}]//g')
+ echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
+ - name: Checkout main branch to 'main-airflow' folder to use breeze from there.
uses: actions/checkout@v2
with:
+ path: "main-airflow"
+ ref: "main"
persist-credentials: false
submodules: recursive
- name: "Setup python"
@@ -165,6 +188,9 @@ jobs:
needs.build-info.outputs.image-build == 'true' &&
github.event.pull_request.head.repo.full_name != 'apache/airflow'
env:
+ DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+ DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
BACKEND: sqlite
outputs: ${{toJSON(needs.build-info.outputs) }}
@@ -176,29 +202,11 @@ jobs:
ref: ${{ needs.build-info.outputs.targetCommitSha }}
persist-credentials: false
submodules: recursive
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore, we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
- id: defaults
- run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
- DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
- - name: >
- Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder
- to use ci/scripts from there.
+ - name: Checkout main branch to 'main-airflow' folder to use ci/scripts from there.
uses: actions/checkout@v2
with:
path: "main-airflow"
- ref: "${{ needs.build-info.outputs.targetBranch }}"
+ ref: "main"
persist-credentials: false
submodules: recursive
- name: "Setup python"
@@ -258,6 +266,9 @@ jobs:
needs.build-info.outputs.image-build == 'true' &&
github.event.pull_request.head.repo.full_name != 'apache/airflow'
env:
+ DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+ DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
BACKEND: sqlite
steps:
@@ -268,22 +279,6 @@ jobs:
ref: ${{ needs.build-info.outputs.targetCommitSha }}
persist-credentials: false
submodules: recursive
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
- id: defaults
- run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
- DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
- cut -d "=" -f 3 | sed s'/["}]//g')
- echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
- name: >
Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder
to use ci/scripts from there.
@@ -372,6 +367,9 @@ jobs:
needs.build-info.outputs.upgradeToNewerDependencies != 'false' &&
github.event.pull_request.head.repo.full_name != 'apache/airflow'
env:
+ DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+ DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
BACKEND: sqlite
outputs: ${{toJSON(needs.build-info.outputs) }}
@@ -383,22 +381,6 @@ jobs:
ref: ${{ needs.build-info.outputs.targetCommitSha }}
persist-credentials: false
submodules: recursive
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore, we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
- id: defaults
- run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
- DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
- name: >
Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder
to use ci/scripts from there.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 383b311283..6fe6966c45 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -126,6 +126,8 @@ jobs:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
defaultBranch: ${{ steps.selective-checks.outputs.default-branch }}
+ defaultConstraintsBranch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
+ debianVersion: ${{ steps.selective-checks.outputs.debian-version }}
cacheDirective: ${{ steps.dynamic-outputs.outputs.cacheDirective }}
waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }}
allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }}
@@ -164,6 +166,7 @@ jobs:
needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }}
default-branch: ${{ steps.selective-checks.outputs.default-branch }}
+ docs-filter: ${{ steps.selective-checks.outputs.docs-filter }}
sourceHeadRepo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }}
pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }}
@@ -191,7 +194,6 @@ jobs:
ref: ${{ github.sha }}
fetch-depth: 2
persist-credentials: false
- if: github.event_name == 'pull_request'
- name: "Setup python"
uses: actions/setup-python@v2
with:
@@ -200,6 +202,22 @@ jobs:
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
+ - name: "Retrieve DEFAULTS from the _initialization.sh"
+ # We cannot "source" the script here because that would be a security problem (we cannot run
+ # any code that comes from the sources coming from the PR. Therefore, we extract the
+ # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
+ id: defaults
+ run: |
+ DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
+ awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+ echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
+ DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
+ scripts/ci/libraries/_initialization.sh | \
+ awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+ echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
+ DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
+ awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
+ echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
- name: Selective checks
id: selective-checks
env:
@@ -294,6 +312,9 @@ jobs:
runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
needs: [build-info]
env:
+ DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+ DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
steps:
- name: Cleanup repo
@@ -310,23 +331,6 @@ jobs:
with:
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore, we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
- id: defaults
- run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
- DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
- run: ./scripts/ci/install_breeze.sh
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- name: "Free space"
@@ -361,6 +365,9 @@ jobs:
runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
needs: [build-info, build-ci-images]
env:
+ DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+ DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
BACKEND: sqlite
DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
@@ -380,23 +387,6 @@ jobs:
with:
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
- id: defaults
- run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
- DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
- run: ./scripts/ci/install_breeze.sh
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- name: "Free space"
@@ -745,7 +735,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
docs-inventory-${{ hashFiles('setup.py','setup.cfg','pyproject.toml;') }}
docs-inventory-
- name: "Build docs"
- run: breeze build-docs
+ run: breeze build-docs ${{ needs.build-info.outputs.docs-filter }}
- name: Configure AWS credentials
uses: ./.github/actions/configure-aws-credentials
if: >
@@ -1766,6 +1756,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- tests-mssql
- tests-postgres
env:
+ DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+ DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
if: needs.build-info.outputs.upgradeToNewerDependencies != 'false'
steps:
@@ -1783,23 +1776,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore, we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
- id: defaults
- run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
- DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
- run: ./scripts/ci/install_breeze.sh
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- name: "Free space"
diff --git a/dev/breeze/src/airflow_breeze/commands/ci_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_commands.py
index c65753e1a6..826a77bf1b 100644
--- a/dev/breeze/src/airflow_breeze/commands/ci_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/ci_commands.py
@@ -30,6 +30,7 @@ from airflow_breeze.params.shell_params import ShellParams
from airflow_breeze.utils.common_options import (
option_airflow_constraints_reference,
option_answer,
+ option_debian_version,
option_dry_run,
option_github_repository,
option_max_age,
@@ -184,6 +185,13 @@ def get_changed_files(commit_ref: Optional[str], dry_run: bool, verbose: bool) -
envvar="DEFAULT_BRANCH",
show_default=True,
)
+@click.option(
+ '--default-constraints-branch',
+ help="Branch against which the constraints should be downloaded from",
+ default="constraints-main",
+ envvar="DEFAULT_CONSTRAINTS_BRANCH",
+ show_default=True,
+)
@click.option(
'--github-event-name',
type=BetterChoice(github_events()),
@@ -192,12 +200,15 @@ def get_changed_files(commit_ref: Optional[str], dry_run: bool, verbose: bool) -
envvar="GITHUB_EVENT_NAME",
show_default=True,
)
+@option_debian_version
@option_verbose
@option_dry_run
def selective_check(
commit_ref: Optional[str],
pr_labels: str,
default_branch: str,
+ default_constraints_branch: str,
+ debian_version: str,
github_event_name: str,
verbose: bool,
dry_run: bool,
@@ -213,6 +224,8 @@ def selective_check(
commit_ref=commit_ref,
files=changed_files,
default_branch=default_branch,
+ default_constraints_branch=default_constraints_branch,
+ debian_version=debian_version,
pr_labels=tuple(" ".split(pr_labels)) if pr_labels else (),
github_event=github_event,
)
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 22e74f4553..2a57cac5e0 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -189,12 +189,16 @@ class SelectiveChecks:
self,
files: tuple[str, ...] = (),
default_branch="main",
+ default_constraints_branch="constraints-main",
+ debian_version="bullseye",
commit_ref: str | None = None,
pr_labels: tuple[str, ...] = (),
github_event: GithubEvents = GithubEvents.PULL_REQUEST,
):
self._files = files
self._default_branch = default_branch
+ self._default_constraints_branch = default_constraints_branch
+ self._debian_version = debian_version
self._commit_ref = commit_ref
self._pr_labels = pr_labels
self._github_event = github_event
@@ -230,6 +234,14 @@ class SelectiveChecks:
def default_branch(self) -> str:
return self._default_branch
+ @cached_property
+ def default_constraints_branch(self) -> str:
+ return self._default_constraints_branch
+
+ @cached_property
+ def debian_version(self) -> str:
+ return self._debian_version
+
@cached_property
def _full_tests_needed(self) -> bool:
if self._github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE]:
@@ -395,7 +407,10 @@ class SelectiveChecks:
@cached_property
def needs_helm_tests(self) -> bool:
- return self._should_be_run(FileGroupForCi.HELM_FILES) and self._default_branch == "main"
+ if self._default_branch != 'main':
+ get_console().print(f"[warning]Not running helm tests in {self._default_branch} branch")
+ return False
+ return self._should_be_run(FileGroupForCi.HELM_FILES)
@cached_property
def run_tests(self) -> bool:
@@ -478,3 +493,11 @@ class SelectiveChecks:
return len(
self._matching_files(FileGroupForCi.SETUP_FILES, CI_FILE_GROUP_MATCHES)
) > 0 or self._github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE]
+
+ @cached_property
+ def docs_filter(self) -> str:
+ return (
+ ""
+ if self._default_branch == 'main'
+ else "--package-filter apache-airflow --package-filter docker-stack"
+ )
diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py
index 2d7e8fe83d..196aacf566 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -191,16 +191,21 @@ def test_expected_output_pull_request_main(
@pytest.mark.parametrize(
- "files, pr_labels, default_branch, expected_outputs,",
+ "files, pr_labels, default_branch, default_constraints_branch, debian_version, expected_outputs,",
[
(
pytest.param(
("INTHEWILD.md",),
("full tests needed",),
"main",
+ "constraints-main",
+ "bullseye",
{
"all-python-versions": "['3.7', '3.8', '3.9', '3.10']",
"all-python-versions-list-as-string": "3.7 3.8 3.9 3.10",
+ "default-branch": "main",
+ "default-constraints-branch": "constraints-main",
+ "debian-version": "bullseye",
"image-build": "true",
"run-tests": "true",
"docs-build": "true",
@@ -218,6 +223,8 @@ def test_expected_output_pull_request_main(
"full tests needed",
),
"main",
+ "constraints-main",
+ "bullseye",
{
"all-python-versions": "['3.7', '3.8', '3.9', '3.10']",
"all-python-versions-list-as-string": "3.7 3.8 3.9 3.10",
@@ -235,9 +242,14 @@ def test_expected_output_pull_request_main(
(),
("full tests needed",),
"main",
+ "constraints-main",
+ "bullseye",
{
"all-python-versions": "['3.7', '3.8', '3.9', '3.10']",
"all-python-versions-list-as-string": "3.7 3.8 3.9 3.10",
+ "default-branch": "main",
+ "default-constraints-branch": "constraints-main",
+ "debian-version": "bullseye",
"image-build": "true",
"run-tests": "true",
"docs-build": "true",
@@ -252,9 +264,14 @@ def test_expected_output_pull_request_main(
("INTHEWILD.md",),
("full tests needed",),
"v2-3-stable",
+ "constraints-2-3",
+ "bullseye",
{
"all-python-versions": "['3.7', '3.8', '3.9', '3.10']",
"all-python-versions-list-as-string": "3.7 3.8 3.9 3.10",
+ "default-branch": "v2-3-stable",
+ "default-constraints-branch": "constraints-2-3",
+ "debian-version": "bullseye",
"image-build": "true",
"run-tests": "true",
"docs-build": "true",
@@ -270,6 +287,8 @@ def test_expected_output_full_tests_needed(
files: Tuple[str, ...],
pr_labels: Tuple[str, ...],
default_branch: str,
+ default_constraints_branch: str,
+ debian_version: str,
expected_outputs: Dict[str, str],
):
sc = SelectiveChecks(
@@ -278,6 +297,8 @@ def test_expected_output_full_tests_needed(
github_event=GithubEvents.PULL_REQUEST,
pr_labels=pr_labels,
default_branch=default_branch,
+ default_constraints_branch=default_constraints_branch,
+ debian_version=debian_version,
)
output = str(sc)
assert_outputs_are_printed(expected_outputs, output)
diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt
index 15b755a6e5..217a9debb9 100644
--- a/images/breeze/output-commands-hash.txt
+++ b/images/breeze/output-commands-hash.txt
@@ -23,7 +23,7 @@ pull-prod-image:6e8467a2b8c833a392c8bdd65189363e
regenerate-command-images:4fd2e7ecbfd6eebb18b854f3eb0f29c8
release-prod-images:8858fe5a13989c7c65a79dc97a880928
resource-check:0fb929ac3496dbbe97acfe99e35accd7
-selective-check:eb1cf022ae43fa9c737b1647142e5a96
+selective-check:9b1cc2827b36be29141083dc9e9f6290
self-upgrade:b5437c0a1a91533a11ee9d0a9692369c
setup-autocomplete:355b72dee171c2fcba46fc90ac7c97b0
shell:ab7955da71048b3a695485c152d06786
diff --git a/images/breeze/output-selective-check.svg b/images/breeze/output-selective-check.svg
index 45a7740819..02ab2644d7 100644
--- a/images/breeze/output-selective-check.svg
+++ b/images/breeze/output-selective-check.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 513.5999999999999" xmlns="http://www.w3.org/2000/svg">
+<svg class="rich-terminal" viewBox="0 0 1482 586.8" xmlns="http://www.w3.org/2000/svg">
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -19,117 +19,129 @@
font-weight: 700;
}
- .terminal-2108240815-matrix {
+ .terminal-3947339877-matrix {
font-family: Fira Code, monospace;
font-size: 20px;
line-height: 24.4px;
font-variant-east-asian: full-width;
}
- .terminal-2108240815-title {
+ .terminal-3947339877-title {
font-size: 18px;
font-weight: bold;
font-family: arial;
}
- .terminal-2108240815-r1 { fill: #c5c8c6;font-weight: bold }
-.terminal-2108240815-r2 { fill: #c5c8c6 }
-.terminal-2108240815-r3 { fill: #d0b344;font-weight: bold }
-.terminal-2108240815-r4 { fill: #868887 }
-.terminal-2108240815-r5 { fill: #68a0b3;font-weight: bold }
-.terminal-2108240815-r6 { fill: #8d7b39 }
-.terminal-2108240815-r7 { fill: #98a84b;font-weight: bold }
+ .terminal-3947339877-r1 { fill: #c5c8c6;font-weight: bold }
+.terminal-3947339877-r2 { fill: #c5c8c6 }
+.terminal-3947339877-r3 { fill: #d0b344;font-weight: bold }
+.terminal-3947339877-r4 { fill: #868887 }
+.terminal-3947339877-r5 { fill: #68a0b3;font-weight: bold }
+.terminal-3947339877-r6 { fill: #8d7b39 }
+.terminal-3947339877-r7 { fill: #98a84b;font-weight: bold }
</style>
<defs>
- <clipPath id="terminal-2108240815-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="462.59999999999997" />
+ <clipPath id="terminal-3947339877-clip-terminal">
+ <rect x="0" y="0" width="1463.0" height="535.8" />
</clipPath>
- <clipPath id="terminal-2108240815-line-0">
+ <clipPath id="terminal-3947339877-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2108240815-line-1">
+<clipPath id="terminal-3947339877-line-1">
<rect x="0" y="25.9" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2108240815-line-2">
+<clipPath id="terminal-3947339877-line-2">
<rect x="0" y="50.3" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2108240815-line-3">
+<clipPath id="terminal-3947339877-line-3">
<rect x="0" y="74.7" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2108240815-line-4">
+<clipPath id="terminal-3947339877-line-4">
<rect x="0" y="99.1" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2108240815-line-5">
+<clipPath id="terminal-3947339877-line-5">
<rect x="0" y="123.5" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2108240815-line-6">
+<clipPath id="terminal-3947339877-line-6">
<rect x="0" y="147.9" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2108240815-line-7">
+<clipPath id="terminal-3947339877-line-7">
<rect x="0" y="172.3" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2108240815-line-8">
+<clipPath id="terminal-3947339877-line-8">
<rect x="0" y="196.7" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2108240815-line-9">
+<clipPath id="terminal-3947339877-line-9">
<rect x="0" y="221.1" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2108240815-line-10">
+<clipPath id="terminal-3947339877-line-10">
<rect x="0" y="245.5" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2108240815-line-11">
+<clipPath id="terminal-3947339877-line-11">
<rect x="0" y="269.9" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2108240815-line-12">
+<clipPath id="terminal-3947339877-line-12">
<rect x="0" y="294.3" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2108240815-line-13">
+<clipPath id="terminal-3947339877-line-13">
<rect x="0" y="318.7" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2108240815-line-14">
+<clipPath id="terminal-3947339877-line-14">
<rect x="0" y="343.1" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2108240815-line-15">
+<clipPath id="terminal-3947339877-line-15">
<rect x="0" y="367.5" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2108240815-line-16">
+<clipPath id="terminal-3947339877-line-16">
<rect x="0" y="391.9" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2108240815-line-17">
+<clipPath id="terminal-3947339877-line-17">
<rect x="0" y="416.3" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="terminal-3947339877-line-18">
+ <rect x="0" y="440.7" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="terminal-3947339877-line-19">
+ <rect x="0" y="465.1" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="terminal-3947339877-line-20">
+ <rect x="0" y="489.5" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="511.6" rx="8"/><text class="terminal-2108240815-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: selective-check</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="584.8" rx="8"/><text class="terminal-3947339877-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: selective-check</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
<circle cx="44" cy="0" r="7" fill="#28c840"/>
</g>
- <g transform="translate(9, 41)" clip-path="url(#terminal-2108240815-clip-terminal)">
+ <g transform="translate(9, 41)" clip-path="url(#terminal-3947339877-clip-terminal)">
- <g class="terminal-2108240815-matrix">
- <text class="terminal-2108240815-r2" x="1464" y="20" textLength="12.2" clip-path="url(#terminal-2108240815-line-0)">
-</text><text class="terminal-2108240815-r3" x="12.2" y="44.4" textLength="85.4" clip-path="url(#terminal-2108240815-line-1)">Usage: </text><text class="terminal-2108240815-r1" x="97.6" y="44.4" textLength="390.4" clip-path="url(#terminal-2108240815-line-1)">breeze selective-check [OPTIONS]</text><text class="terminal-2108240815-r2" x="1464" y="44.4" textLength="12.2" clip-path="url(#terminal-2108240815-line-1)">
-</text><text class="terminal-2108240815-r2" x="1464" y="68.8" textLength="12.2" clip-path="url(#terminal-2108240815-line-2)">
-</text><text class="terminal-2108240815-r2" x="12.2" y="93.2" textLength="768.6" clip-path="url(#terminal-2108240815-line-3)">Checks what kind of tests should be run for an incoming commit.</text><text class="terminal-2108240815-r2" x="1464" y="93.2" textLength="12.2" clip-path="url(#terminal-2108240815-line-3)">
-</text><text class="terminal-2108240815-r2" x="1464" y="117.6" textLength="12.2" clip-path="url(#terminal-2108240815-line-4)">
-</text><text class="terminal-2108240815-r4" x="0" y="142" textLength="24.4" clip-path="url(#terminal-2108240815-line-5)">╭─</text><text class="terminal-2108240815-r4" x="24.4" y="142" textLength="1415.2" clip-path="url(#terminal-2108240815-line-5)"> Selective check flags ─────────────────────────────────────────────────────────────────────────────────────────────</text><text class="terminal-2108240815-r4" x="1439.6" y="142" textLength="24.4" clip-path="url(#terminal-2 [...]
-</text><text class="terminal-2108240815-r4" x="0" y="166.4" textLength="12.2" clip-path="url(#terminal-2108240815-line-6)">│</text><text class="terminal-2108240815-r5" x="24.4" y="166.4" textLength="12.2" clip-path="url(#terminal-2108240815-line-6)">-</text><text class="terminal-2108240815-r5" x="36.6" y="166.4" textLength="85.4" clip-path="url(#terminal-2108240815-line-6)">-commit</text><text class="terminal-2108240815-r5" x="122" y="166.4" textLength="48.8" clip-path="url(#terminal-210 [...]
-</text><text class="terminal-2108240815-r4" x="0" y="190.8" textLength="12.2" clip-path="url(#terminal-2108240815-line-7)">│</text><text class="terminal-2108240815-r5" x="24.4" y="190.8" textLength="12.2" clip-path="url(#terminal-2108240815-line-7)">-</text><text class="terminal-2108240815-r5" x="36.6" y="190.8" textLength="36.6" clip-path="url(#terminal-2108240815-line-7)">-pr</text><text class="terminal-2108240815-r5" x="73.2" y="190.8" textLength="85.4" clip-path="url(#terminal-210824 [...]
-</text><text class="terminal-2108240815-r4" x="0" y="215.2" textLength="12.2" clip-path="url(#terminal-2108240815-line-8)">│</text><text class="terminal-2108240815-r5" x="24.4" y="215.2" textLength="12.2" clip-path="url(#terminal-2108240815-line-8)">-</text><text class="terminal-2108240815-r5" x="36.6" y="215.2" textLength="97.6" clip-path="url(#terminal-2108240815-line-8)">-default</text><text class="terminal-2108240815-r5" x="134.2" y="215.2" textLength="85.4" clip-path="url(#terminal- [...]
-</text><text class="terminal-2108240815-r4" x="0" y="239.6" textLength="12.2" clip-path="url(#terminal-2108240815-line-9)">│</text><text class="terminal-2108240815-r5" x="24.4" y="239.6" textLength="12.2" clip-path="url(#terminal-2108240815-line-9)">-</text><text class="terminal-2108240815-r5" x="36.6" y="239.6" textLength="85.4" clip-path="url(#terminal-2108240815-line-9)">-github</text><text class="terminal-2108240815-r5" x="122" y="239.6" textLength="134.2" clip-path="url(#terminal-21 [...]
-</text><text class="terminal-2108240815-r4" x="0" y="264" textLength="12.2" clip-path="url(#terminal-2108240815-line-10)">│</text><text class="terminal-2108240815-r6" x="305" y="264" textLength="1134.6" clip-path="url(#terminal-2108240815-line-10)">(pull_request | pull_request_review | pull_request_target | pull_request_workflow | push |   </text><text class="terminal-2108240815-r4" x="1451.8" y="264" textLength="12.2" clip-path [...]
-</text><text class="terminal-2108240815-r4" x="0" y="288.4" textLength="12.2" clip-path="url(#terminal-2108240815-line-11)">│</text><text class="terminal-2108240815-r6" x="305" y="288.4" textLength="1134.6" clip-path="url(#terminal-2108240815-line-11)">schedule | workflow_run)                                  & [...]
-</text><text class="terminal-2108240815-r4" x="0" y="312.8" textLength="12.2" clip-path="url(#terminal-2108240815-line-12)">│</text><text class="terminal-2108240815-r4" x="305" y="312.8" textLength="1134.6" clip-path="url(#terminal-2108240815-line-12)">[default: pull_request]                                   & [...]
-</text><text class="terminal-2108240815-r4" x="0" y="337.2" textLength="1464" clip-path="url(#terminal-2108240815-line-13)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="terminal-2108240815-r2" x="1464" y="337.2" textLength="12.2" clip-path="url(#terminal-2108240815-line-13)">
-</text><text class="terminal-2108240815-r4" x="0" y="361.6" textLength="24.4" clip-path="url(#terminal-2108240815-line-14)">╭─</text><text class="terminal-2108240815-r4" x="24.4" y="361.6" textLength="1415.2" clip-path="url(#terminal-2108240815-line-14)"> Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────</text><text class="terminal-2108240815-r4" x="1439.6" y="361.6" textLength="24.4" clip-path="url(#terminal-210 [...]
-</text><text class="terminal-2108240815-r4" x="0" y="386" textLength="12.2" clip-path="url(#terminal-2108240815-line-15)">│</text><text class="terminal-2108240815-r5" x="24.4" y="386" textLength="12.2" clip-path="url(#terminal-2108240815-line-15)">-</text><text class="terminal-2108240815-r5" x="36.6" y="386" textLength="97.6" clip-path="url(#terminal-2108240815-line-15)">-verbose</text><text class="terminal-2108240815-r7" x="158.6" y="386" textLength="24.4" clip-path="url(#terminal-21082 [...]
-</text><text class="terminal-2108240815-r4" x="0" y="410.4" textLength="12.2" clip-path="url(#terminal-2108240815-line-16)">│</text><text class="terminal-2108240815-r5" x="24.4" y="410.4" textLength="12.2" clip-path="url(#terminal-2108240815-line-16)">-</text><text class="terminal-2108240815-r5" x="36.6" y="410.4" textLength="48.8" clip-path="url(#terminal-2108240815-line-16)">-dry</text><text class="terminal-2108240815-r5" x="85.4" y="410.4" textLength="48.8" clip-path="url(#terminal-21 [...]
-</text><text class="terminal-2108240815-r4" x="0" y="434.8" textLength="12.2" clip-path="url(#terminal-2108240815-line-17)">│</text><text class="terminal-2108240815-r5" x="24.4" y="434.8" textLength="12.2" clip-path="url(#terminal-2108240815-line-17)">-</text><text class="terminal-2108240815-r5" x="36.6" y="434.8" textLength="61" clip-path="url(#terminal-2108240815-line-17)">-help</text><text class="terminal-2108240815-r7" x="158.6" y="434.8" textLength="24.4" clip-path="url(#terminal-21 [...]
-</text><text class="terminal-2108240815-r4" x="0" y="459.2" textLength="1464" clip-path="url(#terminal-2108240815-line-18)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="terminal-2108240815-r2" x="1464" y="459.2" textLength="12.2" clip-path="url(#terminal-2108240815-line-18)">
+ <g class="terminal-3947339877-matrix">
+ <text class="terminal-3947339877-r2" x="1464" y="20" textLength="12.2" clip-path="url(#terminal-3947339877-line-0)">
+</text><text class="terminal-3947339877-r3" x="12.2" y="44.4" textLength="85.4" clip-path="url(#terminal-3947339877-line-1)">Usage: </text><text class="terminal-3947339877-r1" x="97.6" y="44.4" textLength="390.4" clip-path="url(#terminal-3947339877-line-1)">breeze selective-check [OPTIONS]</text><text class="terminal-3947339877-r2" x="1464" y="44.4" textLength="12.2" clip-path="url(#terminal-3947339877-line-1)">
+</text><text class="terminal-3947339877-r2" x="1464" y="68.8" textLength="12.2" clip-path="url(#terminal-3947339877-line-2)">
+</text><text class="terminal-3947339877-r2" x="12.2" y="93.2" textLength="768.6" clip-path="url(#terminal-3947339877-line-3)">Checks what kind of tests should be run for an incoming commit.</text><text class="terminal-3947339877-r2" x="1464" y="93.2" textLength="12.2" clip-path="url(#terminal-3947339877-line-3)">
+</text><text class="terminal-3947339877-r2" x="1464" y="117.6" textLength="12.2" clip-path="url(#terminal-3947339877-line-4)">
+</text><text class="terminal-3947339877-r4" x="0" y="142" textLength="24.4" clip-path="url(#terminal-3947339877-line-5)">╭─</text><text class="terminal-3947339877-r4" x="24.4" y="142" textLength="1415.2" clip-path="url(#terminal-3947339877-line-5)"> Selective check flags ─────────────────────────────────────────────────────────────────────────────────────────────</text><text class="terminal-3947339877-r4" x="1439.6" y="142" textLength="24.4" clip-path="url(#terminal-3 [...]
+</text><text class="terminal-3947339877-r4" x="0" y="166.4" textLength="12.2" clip-path="url(#terminal-3947339877-line-6)">│</text><text class="terminal-3947339877-r5" x="24.4" y="166.4" textLength="12.2" clip-path="url(#terminal-3947339877-line-6)">-</text><text class="terminal-3947339877-r5" x="36.6" y="166.4" textLength="85.4" clip-path="url(#terminal-3947339877-line-6)">-commit</text><text class="terminal-3947339877-r5" x="122" y="166.4" textLength="48.8" clip-path="url(#terminal-394 [...]
+</text><text class="terminal-3947339877-r4" x="0" y="190.8" textLength="12.2" clip-path="url(#terminal-3947339877-line-7)">│</text><text class="terminal-3947339877-r5" x="24.4" y="190.8" textLength="12.2" clip-path="url(#terminal-3947339877-line-7)">-</text><text class="terminal-3947339877-r5" x="36.6" y="190.8" textLength="36.6" clip-path="url(#terminal-3947339877-line-7)">-pr</text><text class="terminal-3947339877-r5" x="73.2" y="190.8" textLength="85.4" clip-path="url(#terminal-394733 [...]
+</text><text class="terminal-3947339877-r4" x="0" y="215.2" textLength="12.2" clip-path="url(#terminal-3947339877-line-8)">│</text><text class="terminal-3947339877-r5" x="24.4" y="215.2" textLength="12.2" clip-path="url(#terminal-3947339877-line-8)">-</text><text class="terminal-3947339877-r5" x="36.6" y="215.2" textLength="97.6" clip-path="url(#terminal-3947339877-line-8)">-default</text><text class="terminal-3947339877-r5" x="134.2" y="215.2" textLength="85.4" clip-path="url(#terminal- [...]
+</text><text class="terminal-3947339877-r4" x="0" y="239.6" textLength="12.2" clip-path="url(#terminal-3947339877-line-9)">│</text><text class="terminal-3947339877-r5" x="24.4" y="239.6" textLength="12.2" clip-path="url(#terminal-3947339877-line-9)">-</text><text class="terminal-3947339877-r5" x="36.6" y="239.6" textLength="85.4" clip-path="url(#terminal-3947339877-line-9)">-github</text><text class="terminal-3947339877-r5" x="122" y="239.6" textLength="134.2" clip-path="url(#terminal-39 [...]
+</text><text class="terminal-3947339877-r4" x="0" y="264" textLength="12.2" clip-path="url(#terminal-3947339877-line-10)">│</text><text class="terminal-3947339877-r6" x="305" y="264" textLength="1134.6" clip-path="url(#terminal-3947339877-line-10)">(pull_request | pull_request_review | pull_request_target | pull_request_workflow | push |   </text><text class="terminal-3947339877-r4" x="1451.8" y="264" textLength="12.2" clip-path [...]
+</text><text class="terminal-3947339877-r4" x="0" y="288.4" textLength="12.2" clip-path="url(#terminal-3947339877-line-11)">│</text><text class="terminal-3947339877-r6" x="305" y="288.4" textLength="1134.6" clip-path="url(#terminal-3947339877-line-11)">schedule | workflow_run)                                  & [...]
+</text><text class="terminal-3947339877-r4" x="0" y="312.8" textLength="12.2" clip-path="url(#terminal-3947339877-line-12)">│</text><text class="terminal-3947339877-r4" x="305" y="312.8" textLength="1134.6" clip-path="url(#terminal-3947339877-line-12)">[default: pull_request]                                   & [...]
+</text><text class="terminal-3947339877-r4" x="0" y="337.2" textLength="1464" clip-path="url(#terminal-3947339877-line-13)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="terminal-3947339877-r2" x="1464" y="337.2" textLength="12.2" clip-path="url(#terminal-3947339877-line-13)">
+</text><text class="terminal-3947339877-r4" x="0" y="361.6" textLength="24.4" clip-path="url(#terminal-3947339877-line-14)">╭─</text><text class="terminal-3947339877-r4" x="24.4" y="361.6" textLength="1415.2" clip-path="url(#terminal-3947339877-line-14)"> Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────</text><text class="terminal-3947339877-r4" x="1439.6" y="361.6" textLength="24.4" clip-path="url(#terminal-394 [...]
+</text><text class="terminal-3947339877-r4" x="0" y="386" textLength="12.2" clip-path="url(#terminal-3947339877-line-15)">│</text><text class="terminal-3947339877-r5" x="24.4" y="386" textLength="12.2" clip-path="url(#terminal-3947339877-line-15)">-</text><text class="terminal-3947339877-r5" x="36.6" y="386" textLength="97.6" clip-path="url(#terminal-3947339877-line-15)">-default</text><text class="terminal-3947339877-r5" x="134.2" y="386" textLength="231.8" clip-path="url(#terminal-3947 [...]
+</text><text class="terminal-3947339877-r4" x="0" y="410.4" textLength="12.2" clip-path="url(#terminal-3947339877-line-16)">│</text><text class="terminal-3947339877-r4" x="439.2" y="410.4" textLength="756.4" clip-path="url(#terminal-3947339877-line-16)">[default: constraints-main]                                  &# [...]
+</text><text class="terminal-3947339877-r4" x="0" y="434.8" textLength="12.2" clip-path="url(#terminal-3947339877-line-17)">│</text><text class="terminal-3947339877-r5" x="24.4" y="434.8" textLength="12.2" clip-path="url(#terminal-3947339877-line-17)">-</text><text class="terminal-3947339877-r5" x="36.6" y="434.8" textLength="85.4" clip-path="url(#terminal-3947339877-line-17)">-debian</text><text class="terminal-3947339877-r5" x="122" y="434.8" textLength="97.6" clip-path="url(#terminal- [...]
+</text><text class="terminal-3947339877-r4" x="0" y="459.2" textLength="12.2" clip-path="url(#terminal-3947339877-line-18)">│</text><text class="terminal-3947339877-r5" x="24.4" y="459.2" textLength="12.2" clip-path="url(#terminal-3947339877-line-18)">-</text><text class="terminal-3947339877-r5" x="36.6" y="459.2" textLength="97.6" clip-path="url(#terminal-3947339877-line-18)">-verbose</text><text class="terminal-3947339877-r7" x="390.4" y="459.2" textLength="24.4" clip-path="url(#termin [...]
+</text><text class="terminal-3947339877-r4" x="0" y="483.6" textLength="12.2" clip-path="url(#terminal-3947339877-line-19)">│</text><text class="terminal-3947339877-r5" x="24.4" y="483.6" textLength="12.2" clip-path="url(#terminal-3947339877-line-19)">-</text><text class="terminal-3947339877-r5" x="36.6" y="483.6" textLength="48.8" clip-path="url(#terminal-3947339877-line-19)">-dry</text><text class="terminal-3947339877-r5" x="85.4" y="483.6" textLength="48.8" clip-path="url(#terminal-39 [...]
+</text><text class="terminal-3947339877-r4" x="0" y="508" textLength="12.2" clip-path="url(#terminal-3947339877-line-20)">│</text><text class="terminal-3947339877-r5" x="24.4" y="508" textLength="12.2" clip-path="url(#terminal-3947339877-line-20)">-</text><text class="terminal-3947339877-r5" x="36.6" y="508" textLength="61" clip-path="url(#terminal-3947339877-line-20)">-help</text><text class="terminal-3947339877-r7" x="390.4" y="508" textLength="24.4" clip-path="url(#terminal-3947339877 [...]
+</text><text class="terminal-3947339877-r4" x="0" y="532.4" textLength="1464" clip-path="url(#terminal-3947339877-line-21)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="terminal-3947339877-r2" x="1464" y="532.4" textLength="12.2" clip-path="url(#terminal-3947339877-line-21)">
</text>
</g>
</g>
diff --git a/tests/always/test_project_structure.py b/tests/always/test_project_structure.py
index bd26944233..8db759fac7 100644
--- a/tests/always/test_project_structure.py
+++ b/tests/always/test_project_structure.py
@@ -22,8 +22,6 @@ import os
import unittest
from typing import List
-from parameterized import parameterized
-
ROOT_FOLDER = os.path.realpath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir)
)
@@ -151,142 +149,6 @@ def get_classes_from_file(filepath: str):
return results
-class TestGoogleProviderProjectStructure(unittest.TestCase):
- MISSING_EXAMPLE_DAGS = {
- 'adls_to_gcs',
- 'sql_to_gcs',
- 'bigquery_to_mysql',
- 'cassandra_to_gcs',
- 'drive',
- 'ads_to_gcs',
- }
-
- # Those operators are deprecated and we do not need examples for them
- DEPRECATED_OPERATORS = {
- 'airflow.providers.google.cloud.operators.cloud_storage_transfer_service'
- '.CloudDataTransferServiceS3ToGCSOperator',
- 'airflow.providers.google.cloud.operators.cloud_storage_transfer_service'
- '.CloudDataTransferServiceGCSToGCSOperator',
- 'airflow.providers.google.cloud.operators.dataproc.DataprocSubmitHadoopJobOperator',
- 'airflow.providers.google.cloud.operators.dataproc.DataprocScaleClusterOperator',
- 'airflow.providers.google.cloud.operators.dataproc.DataprocSubmitSparkJobOperator',
- 'airflow.providers.google.cloud.operators.dataproc.DataprocSubmitSparkSqlJobOperator',
- 'airflow.providers.google.cloud.operators.dataproc.DataprocSubmitHiveJobOperator',
- 'airflow.providers.google.cloud.operators.dataproc.DataprocSubmitPigJobOperator',
- 'airflow.providers.google.cloud.operators.dataproc.DataprocSubmitPySparkJobOperator',
- 'airflow.providers.google.cloud.operators.mlengine.MLEngineManageModelOperator',
- 'airflow.providers.google.cloud.operators.mlengine.MLEngineManageVersionOperator',
- 'airflow.providers.google.cloud.operators.dataflow.DataflowCreateJavaJobOperator',
- 'airflow.providers.google.cloud.operators.bigquery.BigQueryPatchDatasetOperator',
- 'airflow.providers.google.cloud.operators.dataflow.DataflowCreatePythonJobOperator',
- 'airflow.providers.google.cloud.operators.bigquery.BigQueryExecuteQueryOperator',
- }
-
- # Those operators should not have examples as they are never used standalone (they are abstract)
- BASE_OPERATORS = {
- 'airflow.providers.google.cloud.operators.compute.ComputeEngineBaseOperator',
- 'airflow.providers.google.cloud.operators.cloud_sql.CloudSQLBaseOperator',
- 'airflow.providers.google.cloud.operators.dataproc.DataprocJobBaseOperator',
- }
-
- # Please at the examples to those operators at the earliest convenience :)
- MISSING_EXAMPLES_FOR_OPERATORS = {
- 'airflow.providers.google.cloud.operators.dataproc.DataprocInstantiateInlineWorkflowTemplateOperator',
- 'airflow.providers.google.cloud.operators.mlengine.MLEngineTrainingCancelJobOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPGetStoredInfoTypeOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPReidentifyContentOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPCreateDeidentifyTemplateOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPCreateDLPJobOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPUpdateDeidentifyTemplateOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPGetDLPJobTriggerOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPListDeidentifyTemplatesOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPGetDeidentifyTemplateOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPListInspectTemplatesOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPListStoredInfoTypesOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPUpdateInspectTemplateOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPDeleteDLPJobOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPListJobTriggersOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPCancelDLPJobOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPGetDLPJobOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPGetInspectTemplateOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPListInfoTypesOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPDeleteDeidentifyTemplateOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPListDLPJobsOperator',
- 'airflow.providers.google.cloud.operators.dlp.CloudDLPRedactImageOperator',
- 'airflow.providers.google.cloud.sensors.gcs.GCSObjectUpdateSensor',
- 'airflow.providers.google.cloud.sensors.gcs.GCSUploadSessionCompleteSensor',
- }
-
- def test_missing_example_for_operator(self):
- """
- Assert that all operators defined under operators, sensors and transfers directories
- are used in any of the example dags
- """
- all_operators = set()
- services = set()
- for resource_type in ["operators", "sensors", "transfers"]:
- operator_files = set(
- self.find_resource_files(top_level_directory="airflow", resource_type=resource_type)
- )
- for filepath in operator_files:
- service_name = os.path.basename(filepath)[: -(len(".py"))]
- if service_name in self.MISSING_EXAMPLE_DAGS:
- continue
- services.add(service_name)
- operators_paths = set(get_classes_from_file(f"{ROOT_FOLDER}/{filepath}"))
- all_operators.update(operators_paths)
-
- for service in services:
- example_dags = self.examples_for_service(service)
- example_paths = {
- path for example_dag in example_dags for path in get_imports_from_file(example_dag)
- }
- all_operators -= example_paths
-
- all_operators -= self.MISSING_EXAMPLES_FOR_OPERATORS
- all_operators -= self.DEPRECATED_OPERATORS
- all_operators -= self.BASE_OPERATORS
- assert set() == all_operators
-
- @parameterized.expand(
- itertools.product(["_system.py", "_system_helper.py"], ["operators", "sensors", "transfers"])
- )
- def test_detect_invalid_system_tests(self, resource_type, filename_suffix):
- operators_tests = self.find_resource_files(top_level_directory="tests", resource_type=resource_type)
- operators_files = self.find_resource_files(top_level_directory="airflow", resource_type=resource_type)
-
- files = {f for f in operators_tests if f.endswith(filename_suffix)}
-
- expected_files = (f"tests/{f[8:]}" for f in operators_files)
- expected_files = (f.replace(".py", filename_suffix).replace("/test_", "/") for f in expected_files)
- expected_files = {f'{f.rpartition("/")[0]}/test_{f.rpartition("/")[2]}' for f in expected_files}
-
- assert set() == files - expected_files
-
- @staticmethod
- def find_resource_files(
- top_level_directory: str = "airflow",
- department: str = "*",
- resource_type: str = "*",
- service: str = "*",
- ):
- python_files = glob.glob(
- f"{ROOT_FOLDER}/{top_level_directory}/providers/google/{department}/{resource_type}/{service}.py"
- )
- # Make path relative
- resource_files = (os.path.relpath(f, ROOT_FOLDER) for f in python_files)
- # Exclude __init__.py and pycache
- resource_files = (f for f in resource_files if not f.endswith("__init__.py"))
- return resource_files
-
- @staticmethod
- def examples_for_service(service_name):
- yield from glob.glob(
- f"{ROOT_FOLDER}/airflow/providers/google/*/example_dags/example_{service_name}*.py"
- )
- yield from glob.glob(f"{ROOT_FOLDER}/tests/system/providers/google/{service_name}/example_*.py")
-
-
class TestOperatorsHooks(unittest.TestCase):
def test_no_illegal_suffixes(self):
illegal_suffixes = ["_operator.py", "_hook.py", "_sensor.py"]
diff --git a/tests/always/test_project_structure.py b/tests/providers/google/test_project_structure.py
similarity index 57%
copy from tests/always/test_project_structure.py
copy to tests/providers/google/test_project_structure.py
index bd26944233..bd2fecade3 100644
--- a/tests/always/test_project_structure.py
+++ b/tests/providers/google/test_project_structure.py
@@ -14,141 +14,15 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-import ast
+
import glob
import itertools
-import mmap
import os
import unittest
-from typing import List
from parameterized import parameterized
-ROOT_FOLDER = os.path.realpath(
- os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir)
-)
-
-
-class TestProjectStructure(unittest.TestCase):
- def test_reference_to_providers_from_core(self):
- for filename in glob.glob(f"{ROOT_FOLDER}/example_dags/**/*.py", recursive=True):
- self.assert_file_not_contains(filename, "providers")
-
- def test_deprecated_packages(self):
- path_pattern = f"{ROOT_FOLDER}/airflow/contrib/**/*.py"
-
- for filename in glob.glob(path_pattern, recursive=True):
- if filename.endswith("/__init__.py"):
- self.assert_file_contains(filename, "This package is deprecated.")
- else:
- self.assert_file_contains(filename, "This module is deprecated.")
-
- def assert_file_not_contains(self, filename: str, pattern: str):
- with open(filename, 'rb', 0) as file, mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) as content:
- if content.find(bytes(pattern, 'utf-8')) != -1:
- self.fail(f"File {filename} not contains pattern - {pattern}")
-
- def assert_file_contains(self, filename: str, pattern: str):
- with open(filename, 'rb', 0) as file, mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) as content:
- if content.find(bytes(pattern, 'utf-8')) == -1:
- self.fail(f"File {filename} contains illegal pattern - {pattern}")
-
- def test_providers_modules_should_have_tests(self):
- """
- Assert every module in /airflow/providers has a corresponding test_ file in tests/airflow/providers.
- """
- # Deprecated modules that don't have corresponded test
- expected_missing_providers_modules = {
- (
- 'airflow/providers/amazon/aws/hooks/aws_dynamodb.py',
- 'tests/providers/amazon/aws/hooks/test_aws_dynamodb.py',
- )
- }
-
- # TODO: Should we extend this test to cover other directories?
- modules_files = glob.glob(f"{ROOT_FOLDER}/airflow/providers/**/*.py", recursive=True)
-
- # Make path relative
- modules_files = (os.path.relpath(f, ROOT_FOLDER) for f in modules_files)
- # Exclude example_dags
- modules_files = (f for f in modules_files if "/example_dags/" not in f)
- # Exclude __init__.py
- modules_files = (f for f in modules_files if not f.endswith("__init__.py"))
- # Change airflow/ to tests/
- expected_test_files = (
- f'tests/{f.partition("/")[2]}' for f in modules_files if not f.endswith("__init__.py")
- )
- # Add test_ prefix to filename
- expected_test_files = (
- f'{f.rpartition("/")[0]}/test_{f.rpartition("/")[2]}'
- for f in expected_test_files
- if not f.endswith("__init__.py")
- )
-
- current_test_files = glob.glob(f"{ROOT_FOLDER}/tests/providers/**/*.py", recursive=True)
- # Make path relative
- current_test_files = (os.path.relpath(f, ROOT_FOLDER) for f in current_test_files)
- # Exclude __init__.py
- current_test_files = (f for f in current_test_files if not f.endswith("__init__.py"))
-
- modules_files = set(modules_files)
- expected_test_files = set(expected_test_files)
- current_test_files = set(current_test_files)
-
- missing_tests_files = expected_test_files - expected_test_files.intersection(current_test_files)
-
- with self.subTest("Detect missing tests in providers module"):
- expected_missing_test_modules = {pair[1] for pair in expected_missing_providers_modules}
- missing_tests_files = missing_tests_files - set(expected_missing_test_modules)
- assert set() == missing_tests_files
-
- with self.subTest("Verify removed deprecated module also removed from deprecated list"):
- expected_missing_modules = {pair[0] for pair in expected_missing_providers_modules}
- removed_deprecated_module = expected_missing_modules - modules_files
- if removed_deprecated_module:
- self.fail(
- "You've removed a deprecated module:\n"
- f"{removed_deprecated_module}"
- "\n"
- "Thank you very much.\n"
- "Can you remove it from the list of expected missing modules tests, please?"
- )
-
-
-def get_imports_from_file(filepath: str):
- with open(filepath) as py_file:
- content = py_file.read()
- doc_node = ast.parse(content, filepath)
- import_names: List[str] = []
- for current_node in ast.walk(doc_node):
- if not isinstance(current_node, (ast.Import, ast.ImportFrom)):
- continue
- for alias in current_node.names:
- name = alias.name
- fullname = f'{current_node.module}.{name}' if isinstance(current_node, ast.ImportFrom) else name
- import_names.append(fullname)
- return import_names
-
-
-def filepath_to_module(filepath: str):
- filepath = os.path.relpath(os.path.abspath(filepath), ROOT_FOLDER)
- return filepath.replace("/", ".")[: -(len('.py'))]
-
-
-def get_classes_from_file(filepath: str):
- with open(filepath) as py_file:
- content = py_file.read()
- doc_node = ast.parse(content, filepath)
- module = filepath_to_module(filepath)
- results: List[str] = []
- for current_node in ast.walk(doc_node):
- if not isinstance(current_node, ast.ClassDef):
- continue
- name = current_node.name
- if not name.endswith("Operator") and not name.endswith("Sensor") and not name.endswith("Operator"):
- continue
- results.append(f"{module}.{name}")
- return results
+from tests.always.test_project_structure import ROOT_FOLDER, get_classes_from_file, get_imports_from_file
class TestGoogleProviderProjectStructure(unittest.TestCase):
@@ -285,19 +159,3 @@ class TestGoogleProviderProjectStructure(unittest.TestCase):
f"{ROOT_FOLDER}/airflow/providers/google/*/example_dags/example_{service_name}*.py"
)
yield from glob.glob(f"{ROOT_FOLDER}/tests/system/providers/google/{service_name}/example_*.py")
-
-
-class TestOperatorsHooks(unittest.TestCase):
- def test_no_illegal_suffixes(self):
- illegal_suffixes = ["_operator.py", "_hook.py", "_sensor.py"]
- files = itertools.chain(
- *(
- glob.glob(f"{ROOT_FOLDER}/{part}/providers/**/{resource_type}/*.py", recursive=True)
- for resource_type in ["operators", "hooks", "sensors", "example_dags"]
- for part in ["airflow", "tests"]
- )
- )
-
- invalid_files = [f for f in files if any(f.endswith(suffix) for suffix in illegal_suffixes)]
-
- assert [] == invalid_files