You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/04/24 15:34:02 UTC
[airflow] branch main updated: Unify all "breeze" tools under breeze sub-commands (#23193)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 87e3733bbc Unify all "breeze" tools under breeze sub-commands (#23193)
87e3733bbc is described below
commit 87e3733bbcb041416316df29b0ddf4edd65367cd
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Sun Apr 24 17:33:57 2022 +0200
Unify all "breeze" tools under breeze sub-commands (#23193)
Experimentally we've separated some CI-only tools like freespace
and find-newer-dependencies as separate scripts (similarly to
initialize-virtualenv) but contrary to the initialize, our tools
can and should be run in Breeze's virtualenv.
However we are just in the middle of using breeze commands in
the CI context, the commands are not "alien" any more and
they seem to fit the same pattern as other commands.
Since we now have the possibility of grouping sub-commmands in separate
logical groups, it makes perfect sense to bring the tools to the
breeze's sub-commands - mostly because of the auto-complete
support and familiarity of using breeze commands in CI.
This PR brings both freespace and find-newer-dependencies tools
already used in CI to become breeze sub-commands. It also
adds "fix-ownership" and "resource-check" commands, so that we can run
owership fixing on Linux machines (and adds fixing ownership as
mandatory stop in all breeze-run jobs in CI and that users
can run resource-check locally.
This is because thgere were some cases after introducing Breeze that
some root-owned left-overs prevented to checkout code by the
runner on self-hosted runners.
This was likely due to resource-check that was run without
PYTHONDONTWRITEBYTECODE=1 variable - this variable have been
added now to execution of the python script.
---
.github/workflows/build-images.yml | 12 +-
.github/workflows/ci.yml | 105 ++++++++++++----
BREEZE.rst | 128 ++++++++++++++-----
dev/TRACKING_BACKTRACKING_ISSUES.md | 27 ++--
dev/breeze/README.md | 2 +-
dev/breeze/setup.cfg | 2 -
dev/breeze/src/airflow_breeze/breeze.py | 128 ++++++++++++++++++-
dev/breeze/src/airflow_breeze/shell/enter_shell.py | 3 +-
dev/breeze/src/airflow_breeze/utils/confirm.py | 1 +
.../airflow_breeze/utils/docker_command_utils.py | 6 +-
.../utils}/find_newer_dependencies.py | 90 ++++---------
dev/breeze/src/airflow_ci/__init__.py | 16 ---
dev/breeze/src/airflow_ci/freespace.py | 59 ---------
docs/conf.py | 3 +-
docs/exts/docs_build/fetch_inventories.py | 4 +-
images/breeze/output-commands.svg | 6 +-
images/breeze/output-find-newer-dependencies.svg | 140 +++++++++++++++++++++
images/breeze/output-fix-ownership.svg | 133 ++++++++++++++++++++
images/breeze/output-free-space.svg | 135 ++++++++++++++++++++
images/breeze/output-resource-check.svg | 133 ++++++++++++++++++++
kubernetes_tests/test_other_executors.py | 1 +
scripts/ci/tools/fix_ownership.sh | 44 -------
scripts/in_container/_in_container_utils.sh | 10 ++
23 files changed, 918 insertions(+), 270 deletions(-)
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index 67fa372712..3d933dc2f2 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -223,7 +223,7 @@ jobs:
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: Build & Push CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze build-image --push-image
env:
@@ -237,7 +237,10 @@ jobs:
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Candidates for pip resolver backtrack triggers: ${{ matrix.python-version }}"
if: failure() || cancelled()
- run: airflow-find-newer-dependencies --max-age 1 --python "${{ matrix.python-version }}"
+ run: breeze find-newer-dependencies --max-age 1 --python "${{ matrix.python-version }}"
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
build-prod-images:
permissions:
@@ -310,7 +313,7 @@ jobs:
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: >
Pull CI image for PROD build:
${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
@@ -347,3 +350,6 @@ jobs:
run: breeze build-prod-image --cleanup-docker-context-files --push-image --empty-image
env:
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c8a249c404..4e9f133860 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -322,7 +322,7 @@ jobs:
echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- name: Build & Push CI image ${{ matrix.python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze build-image --push-image
@@ -334,7 +334,10 @@ jobs:
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- name: "Candidates for pip resolver backtrack triggers: ${{ matrix.python-version }}"
if: failure() || cancelled()
- run: airflow-find-newer-dependencies --max-age 1 --python "${{ matrix.python-version }}"
+ run: breeze find-newer-dependencies --max-age 1 --python "${{ matrix.python-version }}"
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always() && needs.build-info.outputs.inWorkflowBuild == 'true'
build-prod-images:
permissions:
@@ -385,7 +388,7 @@ jobs:
- run: ./scripts/ci/install_breeze.sh
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- name: >
Pull CI image for PROD build:
@@ -423,6 +426,9 @@ jobs:
DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always() && needs.build-info.outputs.inWorkflowBuild == 'true'
run-new-breeze-tests:
timeout-minutes: 10
@@ -548,13 +554,16 @@ jobs:
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
id: wait-for-images
run: breeze pull-image --run-in-parallel --verify-image --wait-for-image
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
static-checks:
timeout-minutes: 30
@@ -578,7 +587,7 @@ jobs:
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: >
Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-image --tag-as-latest
@@ -614,6 +623,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
VERBOSE: false
SKIP: "identity"
COLUMNS: 250
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
# Those checks are run if no image needs to be built for checks. This is for simple changes that
# Do not touch any of the python code or any of the important files that might require building
@@ -671,6 +683,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
--commit-ref "${{ github.sha }}"
env:
VERBOSE: false
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
+
docs:
timeout-minutes: 45
name: "Build docs"
@@ -693,7 +709,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-image --tag-as-latest
env:
@@ -722,6 +738,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' &&
github.event_name == 'push'
run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
prepare-test-provider-packages-wheel:
timeout-minutes: 40
@@ -746,7 +765,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: >
Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-image --tag-as-latest
@@ -772,6 +791,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
env:
USE_AIRFLOW_VERSION: "2.1.0"
PACKAGE_FORMAT: "wheel"
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
prepare-test-provider-packages-sdist:
timeout-minutes: 40
@@ -796,7 +818,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: >
Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-image --tag-as-latest
@@ -820,6 +842,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
USE_AIRFLOW_VERSION: "sdist"
PACKAGE_FORMAT: "sdist"
AIRFLOW_EXTRAS: "all"
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
tests-helm:
timeout-minutes: 80
@@ -850,7 +875,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: >
Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-image --tag-as-latest
@@ -881,6 +906,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
name: coverage-helm
path: "./files/coverage*.xml"
retention-days: 7
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
tests-postgres:
timeout-minutes: 130
@@ -915,7 +943,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-image --tag-as-latest
env:
@@ -949,6 +977,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
name: coverage-postgres-${{matrix.python-version}}-${{matrix.postgres-version}}
path: "./files/coverage*.xml"
retention-days: 7
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
tests-mysql:
timeout-minutes: 130
@@ -982,7 +1013,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-image --tag-as-latest
env:
@@ -1016,6 +1047,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
name: coverage-mysql-${{matrix.python-version}}-${{matrix.mysql-version}}
path: "./files/coverage*.xml"
retention-days: 7
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
tests-mssql:
timeout-minutes: 130
@@ -1049,7 +1083,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-image --tag-as-latest
env:
@@ -1081,6 +1115,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
name: coverage-mssql-${{matrix.python-version}}-${{matrix.mssql-version}}
path: "./files/coverage*.xml"
retention-days: 7
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
tests-sqlite:
timeout-minutes: 130
@@ -1112,7 +1149,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-image --tag-as-latest
env:
@@ -1144,6 +1181,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
name: coverage-sqlite-${{matrix.python-version}}
path: ./files/coverage*.xml
retention-days: 7
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
tests-quarantined:
timeout-minutes: 60
@@ -1183,7 +1223,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
echo "ISSUE_ID=10128" >> $GITHUB_ENV
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-image --tag-as-latest
env:
@@ -1220,6 +1260,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
name: coverage-quarantined-${{ matrix.backend }}
path: "./files/coverage*.xml"
retention-days: 7
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
upload-coverage:
timeout-minutes: 15
@@ -1253,6 +1296,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
uses: ./.github/actions/codecov-action
with:
directory: "./coverage-files"
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
wait-for-prod-images:
timeout-minutes: 120
@@ -1277,7 +1323,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: "Cache virtualenv environment"
uses: actions/cache@v2
with:
@@ -1292,6 +1338,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
test-docker-compose-quick-start:
timeout-minutes: 60
@@ -1315,13 +1364,16 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: Pull PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-prod-image --tag-as-latest
env:
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Test docker-compose quick start"
run: breeze docker-compose-tests
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
tests-kubernetes:
timeout-minutes: 70
@@ -1362,7 +1414,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-prod-image --run-in-parallel --tag-as-latest
env:
@@ -1387,6 +1439,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
name: kind-logs-${{matrix.executor}}
path: /tmp/kind_logs_*
retention-days: 7
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
tests-helm-executor-upgrade:
timeout-minutes: 100
@@ -1422,7 +1477,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-prod-image --run-in-parallel --tag-as-latest
env:
@@ -1458,6 +1513,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
name: kind-logs-KubernetesExecutor
path: /tmp/kind_logs_*
retention-days: 7
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
constraints:
permissions:
@@ -1491,7 +1549,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-image --run-in-parallel --tag-as-latest
env:
@@ -1504,7 +1562,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
breeze generate-constraints --run-in-parallel --generate-constraints-mode no-providers
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }}
- ANSWER: "yes"
- name: "Set constraints branch name"
id: constraints-branch
run: ./scripts/ci/constraints/ci_branch_constraints.sh
@@ -1526,6 +1583,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: ${{ steps.constraints-branch.outputs.branch }}
directory: "repo"
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
# Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build
# is executed as result of direct push to "main" or one of the "vX-Y-test" branches
@@ -1561,7 +1621,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
- run: airflow-freespace
+ run: breeze free-space
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- name: "Build & Push CI image ${{ matrix.python-version }}:latest"
@@ -1611,3 +1671,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
if: always()
+ - name: "Fix ownership"
+ run: breeze fix-ownership
+ if: always()
diff --git a/BREEZE.rst b/BREEZE.rst
index 243b51360b..5f60b12135 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -193,6 +193,9 @@ periodically. For details see
On WSL2 you might want to increase your Virtual Hard Disk by following:
`Expanding the size of your WSL 2 Virtual Hard Disk <https://docs.microsoft.com/en-us/windows/wsl/compare-versions#expanding-the-size-of-your-wsl-2-virtual-hard-disk>`_
+There is a command ``breeze resource-check`` that you can run to check available resources. See below
+for details.
+
Cleaning the environment
------------------------
@@ -445,57 +448,71 @@ Airflow Breeze is a bash script serving as a "swiss-army-knife" of Airflow testi
hood it uses other scripts that you can also run manually if you have problem with running the Breeze
environment. Breeze script allows performing the following tasks:
-Airflow developers tasks
-------------------------
+Development tasks
+-----------------
-Regular development tasks:
+Those are commands mostly used by contributors:
-* Setup autocomplete for Breeze with ``breeze setup-autocomplete`` command
+* Execute arbitrary command in the test environment with ``breeze shell`` command
* Enter interactive shell in CI container when ``shell`` (or no command) is specified
* Start containerised, development-friendly airflow installation with ``breeze start-airflow`` command
* Build documentation with ``breeze build-docs`` command
* Initialize local virtualenv with ``./scripts/tools/initialize_virtualenv.py`` command
-* Cleanup breeze with ``breeze cleanup`` command
* Run static checks with autocomplete support ``breeze static-checks`` command
* Run test specified with ``./breeze-legacy tests`` command
-
-CI Image tasks:
-
-* Build CI docker image with ``breeze build-image`` command
-* Pull CI images in parallel ``breeze pull-image`` command
-* Verify CI image ``breeze verify-image`` command
-
-PROD Image tasks:
-* Build PROD image with ``breeze build-prod-image`` command
-* Pull PROD image in parallel ``breeze pull-prod-image`` command
-* Verify CI image ``breeze verify-prod-image`` command
-
-Additional management tasks:
-
* Join running interactive shell with ``./breeze-legacy exec`` command
* Stop running interactive environment with ``breeze stop`` command
-* Execute arbitrary command in the test environment with ``breeze shell`` command
* Execute arbitrary docker-compose command with ``./breeze-legacy docker-compose`` command
-Docker compose tests:
+Tests
+-----
* Run docker-compose tests with ``breeze docker-compose-tests`` command.
-Kubernetes tests related:
+Kubernetes tests
+----------------
* Manage KinD Kubernetes cluster and deploy Airflow to KinD cluster ``./breeze-legacy kind-cluster`` commands
* Run Kubernetes tests specified with ``./breeze-legacy kind-cluster tests`` command
* Enter the interactive kubernetes test environment with ``./breeze-legacy kind-cluster shell`` command
-Airflow can also be used for managing Production images - this is a development-only feature,
-regular users of Airflow should use ``docker build`` commands to manage the images as described
-in the user documentation about `building the image <https://airflow.apache.org/docs/docker-stack/build.html>`_
+CI Image tasks
+--------------
+
+The image building is usually run for users automatically when needed,
+but sometimes Breeze users might want to manually build, pull or verify the CI images.
+
+* Build CI docker image with ``breeze build-image`` command
+* Pull CI images in parallel ``breeze pull-image`` command
+* Verify CI image ``breeze verify-image`` command
-Maintainer tasks
+PROD Image tasks
----------------
+Users can also build Production images when they are developing them. However when you want to
+use the PROD image, the regular docker build commands are recommended. See
+`building the image <https://airflow.apache.org/docs/docker-stack/build.html>`_
+
+* Build PROD image with ``breeze build-prod-image`` command
+* Pull PROD image in parallel ``breeze pull-prod-image`` command
+* Verify CI image ``breeze verify-prod-image`` command
+
+Configuration and maintenance
+-----------------------------
+
+* Cleanup breeze with ``breeze cleanup`` command
+* Self-upgrade breeze with ``breeze self-upgrade`` command
+* Setup autocomplete for Breeze with ``breeze setup-autocomplete`` command
+* Checking available resources for docker with ``breeze resource-check`` command
+* Freeing space needed to run CI tests with ``breeze free-space`` command
+* Fixing ownership of files in your repository with ``breeze fix-ownership`` command
+* Print Breeze version with ``breeze fix-ownership`` command
+
+Release tasks
+-------------
+
Maintainers also can use Breeze for other purposes (those are commands that regular contributors likely
-do not need):
+do not need or have no access to run). Those are usually connected with releasing Airflow:
* Prepare cache for CI: ``breeze build-image --prepare-build-cache`` and
``breeze build-prod image --prepare-build-cache``(needs buildx plugin and write access to registry ghcr.io)
@@ -503,6 +520,9 @@ do not need):
* Prepare airflow packages: ``breeze prepare-airflow-package`` (when releasing Airflow)
* Prepare provider documentation ``breeze prepare-provider-documentation`` and prepare provider packages
``breeze prepare-provider-packages`` (when releasing provider packages)
+* Finding the updated dependencies since the last successful build when we have conflict with
+ ``breeze find-newer-dependencies`` command
+
Details of Breeze usage
=======================
@@ -1150,6 +1170,49 @@ command but it is very similar to current ``breeze`` command):
</a>
</div>
+Resource check
+==============
+
+Breeze requires certain resources to be available - disk, memory, CPU. When you enter Breeze's shell,
+the resources are checked and information if there is enough resources is displayed. However you can
+manually run resource check any time by ``breeze resource-check`` command.
+
+Those are all available flags of ``resource-check`` command:
+
+.. image:: ./images/breeze/output-resource-check.svg
+ :width: 100%
+ :alt: Breeze resource-check
+
+
+Freeing the space
+=================
+
+When our CI runs a job, it needs all memory and disk it can have. We have a Breeze command that frees
+the memory and disk space used. You can also use it clear space locally but it performs a few operations
+that might be a bit invasive - such are removing swap file and complete pruning of docker disk space used.
+
+Those are all available flags of ``free-space`` command:
+
+.. image:: ./images/breeze/output-free-space.svg
+ :width: 100%
+ :alt: Breeze free-space
+
+
+Tracking backtracking issues for CI builds
+==========================================
+
+When our CI runs a job, we automatically upgrade our dependencies in the ``main`` build. However, this might
+lead to conflicts and ``pip`` backtracking for a long time (possibly forever) for dependency resolution.
+Unfortunately those issues are difficult to diagnose so we had to invent our own tool to help us with
+diagnosing them. This tool is ``find-newer-dependencies`` and it works in the way that it helps to guess
+which new dependency might have caused the backtracking. The whole process is described in
+`tracking backtracking issues <dev/TRACKING_BACKTRACKING_ISSUES.md>`_.
+
+Those are all available flags of ``find-newer-dependencies`` command:
+
+.. image:: ./images/breeze/output-find-newer-dependencies.svg
+ :width: 100%
+ :alt: Breeze find-newer-dependencies
Internal details of Breeze
==========================
@@ -1240,11 +1303,18 @@ On Linux, there is a problem with propagating ownership of created files (a know
files and directories created in the container are not owned by the host user (but by the root user in our
case). This may prevent you from switching branches, for example, if files owned by the root user are
created within your sources. In case you are on a Linux host and have some files in your sources created
-by the root user, you can fix the ownership of those files by running this script:
+by the root user, you can fix the ownership of those files by running :
.. code-block::
- ./scripts/ci/tools/fix_ownership.sh
+ breeze fix-ownership
+
+Those are all available flags of ``fix-ownership`` command:
+
+.. image:: ./images/breeze/output-fix-ownership.svg
+ :width: 100%
+ :alt: Breeze fix-ownership
+
Mounting Local Sources to Breeze
--------------------------------
diff --git a/dev/TRACKING_BACKTRACKING_ISSUES.md b/dev/TRACKING_BACKTRACKING_ISSUES.md
index a4c6e14efc..212c7dd059 100644
--- a/dev/TRACKING_BACKTRACKING_ISSUES.md
+++ b/dev/TRACKING_BACKTRACKING_ISSUES.md
@@ -192,28 +192,15 @@ You need to install the breeze:
* `pipx install -e ./dev/breeze` if you use pipx install.
-Then you can run airflow-find-newer-dependencies with optional flags. For example if you know that the build
-was likely broken on a given date and time (in your timezone) and you want to check python 3.8
-(because this is the only failing build) you can run:
+Then you can run ``breeze find-newer-dependencies`` with optional flags.
+For example if you know that the build was likely broken on a given date and time
+(in your timezone) and you want to check python 3.8 (because this is the only
+failing build) you can run:
```
-airflow-find-newer-dependencies --updated-on-or-after '2022-02-22 10:30:00' --timezone 'CET' --python 3.8
+breeze find-newer-dependencies --updated-on-or-after '2022-02-22 10:30:00' --timezone 'CET' --python 3.8
```
-Full list of options:
+The full list of options for `find-newer-dependencies` can be seen here
-```
-Usage: airflow-find-newer-dependencies [OPTIONS]
-
-Options:
- --max-age INTEGER Max age of the last release (used if no updated-
- after if specified)
-
- --updated-on-or-after TEXT Date when the release was updated after
- --python [3.7|3.8|3.9,3.10] Python version used
- --constraints-branch TEXT Constraint branch to use to find newer
- dependencies
-
- --timezone TEXT Timezone to use during the check
- --help Show this message and exit.
-```
+
diff --git a/dev/breeze/README.md b/dev/breeze/README.md
index 604fdb1bf8..fbf2edcc1e 100644
--- a/dev/breeze/README.md
+++ b/dev/breeze/README.md
@@ -52,6 +52,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT.
---------------------------------------------------------------------------------------------------------
-Package config hash: ea8611f8d231310d9c8dad11e7f61b5d8dd9f995473451a0ae60856e51d2264a6c1dc0a58be8fdc665673945197018b41d6b4b5aad1fdffcd011aa10c25790bc
+Package config hash: a710b1c85a4ab4308cfab0ef5d2e118516f9574708d72648dc4063aae04e3cc1b44d35de446832017801b2b63aeec0a3d381d3050d517acb785e0236fd2b8588
---------------------------------------------------------------------------------------------------------
diff --git a/dev/breeze/setup.cfg b/dev/breeze/setup.cfg
index 79e4d9a3f4..5ef8675c98 100644
--- a/dev/breeze/setup.cfg
+++ b/dev/breeze/setup.cfg
@@ -72,8 +72,6 @@ where=src
[options.entry_points]
console_scripts=
breeze=airflow_breeze.breeze:main
- airflow-freespace=airflow_ci.freespace:main
- airflow-find-newer-dependencies=airflow_ci.find_newer_dependencies:main
[bdist_wheel]
python-tag=py3
diff --git a/dev/breeze/src/airflow_breeze/breeze.py b/dev/breeze/src/airflow_breeze/breeze.py
index 5e28860649..381c93560d 100755
--- a/dev/breeze/src/airflow_breeze/breeze.py
+++ b/dev/breeze/src/airflow_breeze/breeze.py
@@ -33,6 +33,7 @@ from airflow_breeze.shell.shell_params import ShellParams
from airflow_breeze.utils.ci_group import ci_group
from airflow_breeze.utils.confirm import Answer, set_forced_answer, user_confirm
from airflow_breeze.utils.constraints import run_generate_constraints, run_generate_constraints_in_parallel
+from airflow_breeze.utils.find_newer_dependencies import find_newer_dependencies
from airflow_breeze.utils.pulll_image import run_pull_image, run_pull_in_parallel
from airflow_breeze.utils.reinstall import ask_to_reinstall_breeze, reinstall_breeze, warn_non_editable
from airflow_breeze.utils.run_tests import run_docker_compose_tests, verify_an_image
@@ -383,6 +384,18 @@ try:
"breeze prepare-provider-documentation": [
{"name": "Provider documentation preparation flags", "options": ["--skip-package-verification"]}
],
+ "breeze find-newer-dependencies": [
+ {
+ "name": "Find newer dependencies flags",
+ "options": [
+ "--python",
+ "--timezone",
+ "--constraints-branch",
+ "--updated-on-or-after",
+ "--max-age",
+ ],
+ }
+ ],
"breeze generate-constraints": [
{
"name": "Generate constraints flags",
@@ -431,7 +444,16 @@ try:
},
{
"name": "Configuration & maintenance",
- "commands": ["cleanup", "self-upgrade", "setup-autocomplete", "config", "version"],
+ "commands": [
+ "cleanup",
+ "self-upgrade",
+ "setup-autocomplete",
+ "config",
+ "resource-check",
+ "free-space",
+ "fix-ownership",
+ "version",
+ ],
},
{
"name": "CI Image tools",
@@ -456,6 +478,7 @@ try:
"prepare-provider-packages",
"prepare-airflow-package",
"generate-constraints",
+ "find-newer-dependencies",
],
},
]
@@ -1753,7 +1776,6 @@ def build_docs(
"""Build documentation in the container."""
params = BuildCiParams(github_repository=github_repository, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION)
ci_image_name = params.airflow_image_name
- check_docker_resources(verbose, ci_image_name)
doc_builder = DocParams(
package_filter=package_filter,
docs_only=docs_only,
@@ -2142,7 +2164,7 @@ def generate_constraints(
synchronize_parameters_with_cache(shell_params, {"python": python})
console.print("\n[yellow]Use this command to build the image:[/]\n")
console.print(
- f" breeze build-image --python '{shell_params.python}' "
+ f" breeze build-image --python'{shell_params.python}' "
f"--upgrade-to-newer-dependencies true\n"
)
sys.exit(1)
@@ -2180,6 +2202,106 @@ def generate_constraints(
sys.exit(return_code)
+@main.command(name="free-space", help="Free space for jobs run in CI.")
+@option_verbose
+@option_dry_run
+@option_answer
+def free_space(verbose: bool, dry_run: bool, answer: str):
+ set_forced_answer(answer)
+ if user_confirm("Are you sure to run free-space and perform cleanup?", timeout=None) == Answer.YES:
+ run_command(["sudo", "swapoff", "-a"], verbose=verbose, dry_run=dry_run)
+ run_command(["sudo", "rm", "-f", "/swapfile"], verbose=verbose, dry_run=dry_run)
+ run_command(["sudo", "apt-get", "clean"], verbose=verbose, dry_run=dry_run, check=False)
+ run_command(
+ ["docker", "system", "prune", "--all", "--force", "--volumes"], verbose=verbose, dry_run=dry_run
+ )
+ run_command(["df", "-h"], verbose=verbose, dry_run=dry_run)
+ run_command(["docker", "logout", "ghcr.io"], verbose=verbose, dry_run=dry_run, check=False)
+
+
+@main.command(name="resource-check", help="Check if available docker resources are enough.")
+@option_verbose
+@option_dry_run
+def resource_check(verbose: bool, dry_run: bool):
+ shell_params = ShellParams(verbose=verbose, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION)
+ check_docker_resources(verbose, shell_params.airflow_image_name, dry_run=dry_run)
+
+
+option_timezone = click.option(
+ "--timezone",
+ default="UTC",
+ type=str,
+ help="Timezone to use during the check",
+)
+
+option_updated_on_or_after = click.option(
+ "--updated-on-or-after",
+ type=str,
+ help="Date when the release was updated after",
+)
+
+option_max_age = click.option(
+ "--max-age",
+ type=int,
+ default=3,
+ help="Max age of the last release (used if no updated-on-or-after if specified)",
+)
+
+option_branch = click.option(
+ "--constraints-branch",
+ default='constraints-main',
+ help="Constraint branch to use to find newer dependencies",
+)
+
+
+@main.command(name="find-newer-dependencies", help="Finds which dependencies are being upgraded.")
+@option_timezone
+@option_branch
+@option_python
+@option_updated_on_or_after
+@option_max_age
+def breeze_find_newer_dependencies(
+ constraints_branch: str, python: str, timezone: str, updated_on_or_after: str, max_age: int
+):
+ return find_newer_dependencies(
+ constraints_branch=constraints_branch,
+ python=python,
+ timezone=timezone,
+ updated_on_or_after=updated_on_or_after,
+ max_age=max_age,
+ )
+
+
+@main.command(name="fix-ownership", help="Fix ownership of source files to be same as host user.")
+@option_verbose
+@option_dry_run
+def fix_ownership(verbose: bool, dry_run: bool):
+ shell_params = ShellParams(
+ verbose=verbose,
+ mount_sources=MOUNT_ALL,
+ python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
+ )
+ extra_docker_flags = get_extra_docker_flags(MOUNT_ALL)
+ env = construct_env_variables_docker_compose_command(shell_params)
+ cmd = [
+ "docker",
+ "run",
+ "-t",
+ *extra_docker_flags,
+ "-e",
+ "GITHUB_ACTIONS=",
+ "-e",
+ "SKIP_ENVIRONMENT_INITIALIZATION=true",
+ "--pull",
+ "never",
+ shell_params.airflow_image_name_with_tag,
+ "/opt/airflow/scripts/in_container/run_fix_ownership.sh",
+ ]
+ run_command(cmd, verbose=verbose, dry_run=dry_run, text=True, env=env, check=False)
+ # Always succeed
+ sys.exit(0)
+
+
@main.command(
name="cleanup",
help="Cleans the cache of parameters, docker cache and optionally - currently downloaded images.",
diff --git a/dev/breeze/src/airflow_breeze/shell/enter_shell.py b/dev/breeze/src/airflow_breeze/shell/enter_shell.py
index 855a24f1d6..309cf316ea 100644
--- a/dev/breeze/src/airflow_breeze/shell/enter_shell.py
+++ b/dev/breeze/src/airflow_breeze/shell/enter_shell.py
@@ -115,11 +115,10 @@ def run_shell_with_build_image_checks(
:param dry_run: do not execute "write" commands - just print what would happen
:param shell_params: parameters of the execution
"""
- check_docker_resources(verbose, shell_params.airflow_image_name)
+ check_docker_resources(verbose, shell_params.airflow_image_name, dry_run=dry_run)
build_ci_image_check_cache = Path(
BUILD_CACHE_DIR, shell_params.airflow_branch, f".built_{shell_params.python}"
)
-
ci_image_params = get_ci_image_build_params(
{"python": shell_params.python, "upgrade_to_newer_dependencies": "false"}
)
diff --git a/dev/breeze/src/airflow_breeze/utils/confirm.py b/dev/breeze/src/airflow_breeze/utils/confirm.py
index 5fcfe217c1..45a9026edb 100644
--- a/dev/breeze/src/airflow_breeze/utils/confirm.py
+++ b/dev/breeze/src/airflow_breeze/utils/confirm.py
@@ -59,6 +59,7 @@ def user_confirm(
force = forced_answer or os.environ.get('ANSWER')
if force:
user_status = force
+ print(f"Forced answer for '{message}': {force}")
else:
user_status = inputimeout(
prompt=f'\n{message} \nPress {allowed_answers}'
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index f6e59d1ea0..e100986b9b 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -110,11 +110,12 @@ def get_extra_docker_flags(mount_sources: str) -> List[str]:
def check_docker_resources(
- verbose: bool, airflow_image_name: str
+ verbose: bool, airflow_image_name: str, dry_run: bool
) -> Union[subprocess.CompletedProcess, subprocess.CalledProcessError]:
"""
Check if we have enough resources to run docker. This is done via running script embedded in our image.
:param verbose: print commands when running
+ :param dry_run: whether to run it in dry run mode
:param airflow_image_name: name of the airflow image to use.
"""
return run_command(
@@ -124,11 +125,14 @@ def check_docker_resources(
"-t",
"--entrypoint",
"/bin/bash",
+ "-e",
+ "PYTHONDONTWRITEBYTECODE=true",
airflow_image_name,
"-c",
"python /opt/airflow/scripts/in_container/run_resource_check.py",
],
verbose=verbose,
+ dry_run=dry_run,
text=True,
)
diff --git a/dev/breeze/src/airflow_ci/find_newer_dependencies.py b/dev/breeze/src/airflow_breeze/utils/find_newer_dependencies.py
similarity index 82%
rename from dev/breeze/src/airflow_ci/find_newer_dependencies.py
rename to dev/breeze/src/airflow_breeze/utils/find_newer_dependencies.py
index 7742ff2910..17d577a090 100644
--- a/dev/breeze/src/airflow_ci/find_newer_dependencies.py
+++ b/dev/breeze/src/airflow_breeze/utils/find_newer_dependencies.py
@@ -32,75 +32,19 @@ import json
from datetime import timedelta
from typing import Any, Dict, List, Tuple
-import click
-import pendulum
-import requests
-from dateutil.parser import isoparse
-from packaging import version
from rich.console import Console
from rich.progress import Progress
console = Console(width=400, color_system="standard")
-option_branch = click.option(
- "--constraints-branch",
- default='constraints-main',
- help="Constraint branch to use to find newer dependencies",
-)
-option_python = click.option(
- "--python",
- type=click.Choice(["3.7", "3.8", "3.9", "3.10"]),
- default="3.7",
- help="Python version used",
-)
+def find_newer_dependencies(
+ constraints_branch: str, python: str, timezone: str, updated_on_or_after: str, max_age: int
+):
+ import pendulum
+ import requests
+ from packaging import version
-option_timezone = click.option(
- "--timezone",
- default="UTC",
- type=str,
- help="Timezone to use during the check",
-)
-
-option_updated_on_or_after = click.option(
- "--updated-on-or-after",
- type=str,
- help="Date when the release was updated after",
-)
-
-option_max_age = click.option(
- "--max-age",
- type=int,
- default=3,
- help="Max age of the last release (used if no updated-on-or-after if specified)",
-)
-
-
-def get_releases_and_upload_times(package, min_date, current_version, tz) -> List[Tuple[str, Any]]:
- package_info = json.loads(requests.get(f"https://pypi.python.org/pypi/{package}/json").text)
- releases: List[Tuple[Any, Any]] = []
- for release_version, release_info in package_info['releases'].items():
- if release_info and not release_info[0]['yanked']:
- parsed_version = version.parse(release_version)
- if (
- parsed_version.is_prerelease
- or parsed_version.is_devrelease
- or parsed_version == current_version
- ):
- continue
- upload_date = tz.convert(isoparse(release_info[0]['upload_time_iso_8601'])).replace(microsecond=0)
- if upload_date >= min_date:
- releases.append((parsed_version, upload_date))
- return releases
-
-
-@option_timezone
-@option_branch
-@option_python
-@option_updated_on_or_after
-@option_max_age
-@click.command()
-def main(constraints_branch: str, python: str, timezone: str, updated_on_or_after: str, max_age: int):
constraints = requests.get(
f"https://raw.githubusercontent.com/" f"apache/airflow/{constraints_branch}/constraints-{python}.txt"
).text
@@ -153,5 +97,23 @@ def main(constraints_branch: str, python: str, timezone: str, updated_on_or_afte
)
-if __name__ == '__main__':
- main() # type: ignore[misc]
+def get_releases_and_upload_times(package, min_date, current_version, tz) -> List[Tuple[str, Any]]:
+ import requests
+ from dateutil.parser import isoparse
+ from packaging import version
+
+ package_info = json.loads(requests.get(f"https://pypi.python.org/pypi/{package}/json").text)
+ releases: List[Tuple[Any, Any]] = []
+ for release_version, release_info in package_info['releases'].items():
+ if release_info and not release_info[0]['yanked']:
+ parsed_version = version.parse(release_version)
+ if (
+ parsed_version.is_prerelease
+ or parsed_version.is_devrelease
+ or parsed_version == current_version
+ ):
+ continue
+ upload_date = tz.convert(isoparse(release_info[0]['upload_time_iso_8601'])).replace(microsecond=0)
+ if upload_date >= min_date:
+ releases.append((parsed_version, upload_date))
+ return releases
diff --git a/dev/breeze/src/airflow_ci/__init__.py b/dev/breeze/src/airflow_ci/__init__.py
deleted file mode 100644
index 13a83393a9..0000000000
--- a/dev/breeze/src/airflow_ci/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/dev/breeze/src/airflow_ci/freespace.py b/dev/breeze/src/airflow_ci/freespace.py
deleted file mode 100755
index 5c235a35be..0000000000
--- a/dev/breeze/src/airflow_ci/freespace.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/env python3
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""Cleans up the environment before starting CI."""
-
-
-import rich_click as click
-from rich.console import Console
-
-from airflow_breeze.utils.run_utils import run_command
-
-console = Console(force_terminal=True, color_system="standard", width=180)
-
-option_verbose = click.option(
- "--verbose",
- envvar='VERBOSE',
- is_flag=True,
- help="Print verbose information about free space steps",
-)
-
-option_dry_run = click.option(
- "--dry-run",
- is_flag=True,
- help="Just prints commands without executing them",
-)
-
-
-@click.command()
-@option_verbose
-@option_dry_run
-def main(verbose, dry_run):
- run_command(["sudo", "swapoff", "-a"], verbose=verbose, dry_run=dry_run)
- run_command(["sudo", "rm", "-f", "/swapfile"], verbose=verbose, dry_run=dry_run)
- run_command(["sudo", "apt-get", "clean"], verbose=verbose, dry_run=dry_run, check=False)
- run_command(
- ["docker", "system", "prune", "--all", "--force", "--volumes"], verbose=verbose, dry_run=dry_run
- )
- run_command(["df", "-h"], verbose=verbose, dry_run=dry_run)
- run_command(["docker", "logout", "ghcr.io"], verbose=verbose, dry_run=dry_run, check=False)
-
-
-if __name__ == '__main__':
- main()
diff --git a/docs/conf.py b/docs/conf.py
index ef889f641a..b1742f0c00 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -44,10 +44,9 @@ try:
except ImportError:
from yaml import SafeLoader # type: ignore[misc]
-from docs.exts.docs_build.third_party_inventories import THIRD_PARTY_INDEXES
-
import airflow
from airflow.configuration import AirflowConfigParser, default_config_yaml
+from docs.exts.docs_build.third_party_inventories import THIRD_PARTY_INDEXES
sys.path.append(os.path.join(os.path.dirname(__file__), 'exts'))
diff --git a/docs/exts/docs_build/fetch_inventories.py b/docs/exts/docs_build/fetch_inventories.py
index 0833b6de46..d55e28ae14 100644
--- a/docs/exts/docs_build/fetch_inventories.py
+++ b/docs/exts/docs_build/fetch_inventories.py
@@ -27,11 +27,11 @@ from typing import Iterator, List, Tuple
import requests
import urllib3.exceptions
-from docs.exts.docs_build.docs_builder import get_available_providers_packages
-from docs.exts.docs_build.third_party_inventories import THIRD_PARTY_INDEXES
from requests.adapters import DEFAULT_POOLSIZE
from airflow.utils.helpers import partition
+from docs.exts.docs_build.docs_builder import get_available_providers_packages
+from docs.exts.docs_build.third_party_inventories import THIRD_PARTY_INDEXES
CURRENT_DIR = os.path.dirname(__file__)
ROOT_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir, os.pardir))
diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg
index 4ef31a5f6f..78dcf803f3 100644
--- a/images/breeze/output-commands.svg
+++ b/images/breeze/output-commands.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="1726" viewBox="0 0 1720.0 1726"
+<svg width="1720.0" height="1814" viewBox="0 0 1720.0 1814"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -158,6 +158,9 @@
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> self-upgrade </span><span class="r1"> </span><span class="r1">Self upgrade Breeze. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> setup-autocomplete </span><span class="r1"> </span><span class="r1">Enables autocompletion of breeze commands. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> config </span><span class="r1"> </span><span class="r1">Show/update configuration (Python, Backend, Cheatsheet, ASCIIART). </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> resource-check </span><span class="r1"> </span><span class="r1">Check if available docker resources are enough. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> free-space </span><span class="r1"> </span><span class="r1">Free space for jobs run in CI. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> fix-ownership </span><span class="r1"> </span><span class="r1">Fix ownership of source files to be same as host user. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> version </span><span class="r1"> </span><span class="r1">Print information about version of apache-airflow-breeze. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
@@ -176,6 +179,7 @@
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> prepare-provider-packages </span><span class="r1"> </span><span class="r1">Prepare sdist/whl packages of Airflow Providers. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> prepare-airflow-package </span><span class="r1"> </span><span class="r1">Prepare sdist/whl package of Airflow. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> generate-constraints </span><span class="r1"> </span><span class="r1">Generates pinned constraint files with all extras from setup.py in parallel.</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> find-newer-dependencies </span><span class="r1"> </span><span class="r1">Finds which dependencies are being upgraded. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r1"></span><span class="r1"> </span></div>
</div>
diff --git a/images/breeze/output-find-newer-dependencies.svg b/images/breeze/output-find-newer-dependencies.svg
new file mode 100644
index 0000000000..cab8045a1b
--- /dev/null
+++ b/images/breeze/output-find-newer-dependencies.svg
@@ -0,0 +1,140 @@
+<svg width="1720.0" height="714" viewBox="0 0 1720.0 714"
+ xmlns="http://www.w3.org/2000/svg">
+ <style>
+ @font-face {
+ font-family: "Fira Code";
+ src: local("FiraCode-Regular"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff2/FiraCode-Regular.woff2") format("woff2"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff/FiraCode-Regular.woff") format("woff");
+ font-style: normal;
+ font-weight: 400;
+ }
+ @font-face {
+ font-family: "Fira Code";
+ src: local("FiraCode-Bold"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff2/FiraCode-Bold.woff2") format("woff2"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff/FiraCode-Bold.woff") format("woff");
+ font-style: bold;
+ font-weight: 700;
+ }
+ span {
+ display: inline-block;
+ white-space: pre;
+ vertical-align: top;
+ font-size: 18px;
+ font-family:'Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
+ }
+ a {
+ text-decoration: none;
+ color: inherit;
+ }
+ .blink {
+ animation: blinker 1s infinite;
+ }
+ @keyframes blinker {
+ from { opacity: 1.0; }
+ 50% { opacity: 0.3; }
+ to { opacity: 1.0; }
+ }
+ #wrapper {
+ padding: 140px;
+ padding-top: 100px;
+ }
+ #terminal {
+ position: relative;
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ background-color: #0c0c0c;
+ border-radius: 14px;
+ outline: 1px solid #484848;
+ }
+ #terminal:after {
+ position: absolute;
+ width: 100%;
+ height: 100%;
+ content: '';
+ border-radius: 14px;
+ background: rgb(71,77,102);
+ background: linear-gradient(90deg, #804D69 0%, #4E4B89 100%);
+ transform: rotate(-4.5deg);
+ z-index: -1;
+ }
+ #terminal-header {
+ position: relative;
+ width: 100%;
+ background-color: #2e2e2e;
+ margin-bottom: 12px;
+ font-weight: bold;
+ border-radius: 14px 14px 0 0;
+ color: #f2f2f2;
+ font-size: 18px;
+ box-shadow: inset 0px -1px 0px 0px #4e4e4e,
+ inset 0px -4px 8px 0px #1a1a1a;
+ }
+ #terminal-title-tab {
+ display: inline-block;
+ margin-top: 14px;
+ margin-left: 124px;
+ font-family: sans-serif;
+ padding: 14px 28px;
+ border-radius: 6px 6px 0 0;
+ background-color: #0c0c0c;
+ box-shadow: inset 0px 1px 0px 0px #4e4e4e,
+ 0px -4px 4px 0px #1e1e1e,
+ inset 1px 0px 0px 0px #4e4e4e,
+ inset -1px 0px 0px 0px #4e4e4e;
+ }
+ #terminal-traffic-lights {
+ position: absolute;
+ top: 24px;
+ left: 20px;
+ }
+ #terminal-body {
+ line-height: 22px;
+ padding: 14px;
+ }
+ .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+.r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+ </style>
+ <foreignObject x="0" y="0" width="100%" height="100%">
+ <body xmlns="http://www.w3.org/1999/xhtml">
+ <div id="wrapper">
+ <div id="terminal">
+ <div id='terminal-header'>
+ <svg id="terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <circle cx="14" cy="8" r="8" fill="#ff6159"/>
+ <circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
+ <circle cx="62" cy="8" r="8" fill="#28c941"/>
+ </svg>
+ <div id="terminal-title-tab">Command: find-newer-dependencies</div>
+ </div>
+ <div id='terminal-body'>
+ <div><span class="r2"> </span></div>
+<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze find-newer-dependencies [OPTIONS] </span></div>
+<div><span class="r2"> </span></div>
+<div><span class="r1"> </span><span class="r1">Finds which dependencies are being upgraded. </span><span class="r1"> </span></div>
+<div><span class="r1"> </span></div>
+<div><span class="r4">╭─ Find newer dependencies flags ──────────────────────────────────────────────────────────────────────────────────────╮</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for PROD/CI images. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(3.7 | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--timezone</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Timezone to use during the check</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--constraints-branch</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Constraint branch to use to find newer dependencies</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--updated-on-or-after</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Date when the release was updated after</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--max-age</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Max age of the last release (used if no updated-on-or-after if specified)</span><span class="r1"> </span><span class="r7">(INTEGER)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
+<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r6">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
+<div><span class="r1"></span><span class="r1"> </span></div>
+ </div>
+ </div>
+ </div>
+ </body>
+ </foreignObject>
+</svg>
diff --git a/images/breeze/output-fix-ownership.svg b/images/breeze/output-fix-ownership.svg
new file mode 100644
index 0000000000..2b89bb86e5
--- /dev/null
+++ b/images/breeze/output-fix-ownership.svg
@@ -0,0 +1,133 @@
+<svg width="1720.0" height="582" viewBox="0 0 1720.0 582"
+ xmlns="http://www.w3.org/2000/svg">
+ <style>
+ @font-face {
+ font-family: "Fira Code";
+ src: local("FiraCode-Regular"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff2/FiraCode-Regular.woff2") format("woff2"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff/FiraCode-Regular.woff") format("woff");
+ font-style: normal;
+ font-weight: 400;
+ }
+ @font-face {
+ font-family: "Fira Code";
+ src: local("FiraCode-Bold"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff2/FiraCode-Bold.woff2") format("woff2"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff/FiraCode-Bold.woff") format("woff");
+ font-style: bold;
+ font-weight: 700;
+ }
+ span {
+ display: inline-block;
+ white-space: pre;
+ vertical-align: top;
+ font-size: 18px;
+ font-family:'Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
+ }
+ a {
+ text-decoration: none;
+ color: inherit;
+ }
+ .blink {
+ animation: blinker 1s infinite;
+ }
+ @keyframes blinker {
+ from { opacity: 1.0; }
+ 50% { opacity: 0.3; }
+ to { opacity: 1.0; }
+ }
+ #wrapper {
+ padding: 140px;
+ padding-top: 100px;
+ }
+ #terminal {
+ position: relative;
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ background-color: #0c0c0c;
+ border-radius: 14px;
+ outline: 1px solid #484848;
+ }
+ #terminal:after {
+ position: absolute;
+ width: 100%;
+ height: 100%;
+ content: '';
+ border-radius: 14px;
+ background: rgb(71,77,102);
+ background: linear-gradient(90deg, #804D69 0%, #4E4B89 100%);
+ transform: rotate(-4.5deg);
+ z-index: -1;
+ }
+ #terminal-header {
+ position: relative;
+ width: 100%;
+ background-color: #2e2e2e;
+ margin-bottom: 12px;
+ font-weight: bold;
+ border-radius: 14px 14px 0 0;
+ color: #f2f2f2;
+ font-size: 18px;
+ box-shadow: inset 0px -1px 0px 0px #4e4e4e,
+ inset 0px -4px 8px 0px #1a1a1a;
+ }
+ #terminal-title-tab {
+ display: inline-block;
+ margin-top: 14px;
+ margin-left: 124px;
+ font-family: sans-serif;
+ padding: 14px 28px;
+ border-radius: 6px 6px 0 0;
+ background-color: #0c0c0c;
+ box-shadow: inset 0px 1px 0px 0px #4e4e4e,
+ 0px -4px 4px 0px #1e1e1e,
+ inset 1px 0px 0px 0px #4e4e4e,
+ inset -1px 0px 0px 0px #4e4e4e;
+ }
+ #terminal-traffic-lights {
+ position: absolute;
+ top: 24px;
+ left: 20px;
+ }
+ #terminal-body {
+ line-height: 22px;
+ padding: 14px;
+ }
+ .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+ </style>
+ <foreignObject x="0" y="0" width="100%" height="100%">
+ <body xmlns="http://www.w3.org/1999/xhtml">
+ <div id="wrapper">
+ <div id="terminal">
+ <div id='terminal-header'>
+ <svg id="terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <circle cx="14" cy="8" r="8" fill="#ff6159"/>
+ <circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
+ <circle cx="62" cy="8" r="8" fill="#28c941"/>
+ </svg>
+ <div id="terminal-title-tab">Command: fix-ownership</div>
+ </div>
+ <div id='terminal-body'>
+ <div><span class="r2"> </span></div>
+<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze fix-ownership [OPTIONS] </span></div>
+<div><span class="r2"> </span></div>
+<div><span class="r1"> </span><span class="r1">Fix ownership of source files to be same as host user. </span><span class="r1"> </span></div>
+<div><span class="r1"> </span></div>
+<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verbose</span><span class="r1"> </span><span class="r6">-v</span><span class="r1"> </span><span class="r1">Print verbose information about performed steps. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--dry-run</span><span class="r1"> </span><span class="r6">-D</span><span class="r1"> </span><span class="r1">If dry-run is set, commands are only printed, not executed. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r6">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
+<div><span class="r1"></span><span class="r1"> </span></div>
+ </div>
+ </div>
+ </div>
+ </body>
+ </foreignObject>
+</svg>
diff --git a/images/breeze/output-free-space.svg b/images/breeze/output-free-space.svg
new file mode 100644
index 0000000000..9250d459c7
--- /dev/null
+++ b/images/breeze/output-free-space.svg
@@ -0,0 +1,135 @@
+<svg width="1720.0" height="604" viewBox="0 0 1720.0 604"
+ xmlns="http://www.w3.org/2000/svg">
+ <style>
+ @font-face {
+ font-family: "Fira Code";
+ src: local("FiraCode-Regular"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff2/FiraCode-Regular.woff2") format("woff2"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff/FiraCode-Regular.woff") format("woff");
+ font-style: normal;
+ font-weight: 400;
+ }
+ @font-face {
+ font-family: "Fira Code";
+ src: local("FiraCode-Bold"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff2/FiraCode-Bold.woff2") format("woff2"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff/FiraCode-Bold.woff") format("woff");
+ font-style: bold;
+ font-weight: 700;
+ }
+ span {
+ display: inline-block;
+ white-space: pre;
+ vertical-align: top;
+ font-size: 18px;
+ font-family:'Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
+ }
+ a {
+ text-decoration: none;
+ color: inherit;
+ }
+ .blink {
+ animation: blinker 1s infinite;
+ }
+ @keyframes blinker {
+ from { opacity: 1.0; }
+ 50% { opacity: 0.3; }
+ to { opacity: 1.0; }
+ }
+ #wrapper {
+ padding: 140px;
+ padding-top: 100px;
+ }
+ #terminal {
+ position: relative;
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ background-color: #0c0c0c;
+ border-radius: 14px;
+ outline: 1px solid #484848;
+ }
+ #terminal:after {
+ position: absolute;
+ width: 100%;
+ height: 100%;
+ content: '';
+ border-radius: 14px;
+ background: rgb(71,77,102);
+ background: linear-gradient(90deg, #804D69 0%, #4E4B89 100%);
+ transform: rotate(-4.5deg);
+ z-index: -1;
+ }
+ #terminal-header {
+ position: relative;
+ width: 100%;
+ background-color: #2e2e2e;
+ margin-bottom: 12px;
+ font-weight: bold;
+ border-radius: 14px 14px 0 0;
+ color: #f2f2f2;
+ font-size: 18px;
+ box-shadow: inset 0px -1px 0px 0px #4e4e4e,
+ inset 0px -4px 8px 0px #1a1a1a;
+ }
+ #terminal-title-tab {
+ display: inline-block;
+ margin-top: 14px;
+ margin-left: 124px;
+ font-family: sans-serif;
+ padding: 14px 28px;
+ border-radius: 6px 6px 0 0;
+ background-color: #0c0c0c;
+ box-shadow: inset 0px 1px 0px 0px #4e4e4e,
+ 0px -4px 4px 0px #1e1e1e,
+ inset 1px 0px 0px 0px #4e4e4e,
+ inset -1px 0px 0px 0px #4e4e4e;
+ }
+ #terminal-traffic-lights {
+ position: absolute;
+ top: 24px;
+ left: 20px;
+ }
+ #terminal-body {
+ line-height: 22px;
+ padding: 14px;
+ }
+ .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+.r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+ </style>
+ <foreignObject x="0" y="0" width="100%" height="100%">
+ <body xmlns="http://www.w3.org/1999/xhtml">
+ <div id="wrapper">
+ <div id="terminal">
+ <div id='terminal-header'>
+ <svg id="terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <circle cx="14" cy="8" r="8" fill="#ff6159"/>
+ <circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
+ <circle cx="62" cy="8" r="8" fill="#28c941"/>
+ </svg>
+ <div id="terminal-title-tab">Command: free-space</div>
+ </div>
+ <div id='terminal-body'>
+ <div><span class="r2"> </span></div>
+<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze free-space [OPTIONS] </span></div>
+<div><span class="r2"> </span></div>
+<div><span class="r1"> </span><span class="r1">Free space for jobs run in CI. </span><span class="r1"> </span></div>
+<div><span class="r1"> </span></div>
+<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verbose</span><span class="r1"> </span><span class="r6">-v</span><span class="r1"> </span><span class="r1">Print verbose information about performed steps. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--dry-run</span><span class="r1"> </span><span class="r6">-D</span><span class="r1"> </span><span class="r1">If dry-run is set, commands are only printed, not executed. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--answer</span><span class="r1"> </span><span class="r6">-a</span><span class="r1"> </span><span class="r1">Force answer to questions.</span><span class="r1"> </span><span class="r7">(y | n | q | yes | no | quit)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r6">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
+<div><span class="r1"></span><span class="r1"> </span></div>
+ </div>
+ </div>
+ </div>
+ </body>
+ </foreignObject>
+</svg>
diff --git a/images/breeze/output-resource-check.svg b/images/breeze/output-resource-check.svg
new file mode 100644
index 0000000000..c5a2bcb473
--- /dev/null
+++ b/images/breeze/output-resource-check.svg
@@ -0,0 +1,133 @@
+<svg width="1720.0" height="582" viewBox="0 0 1720.0 582"
+ xmlns="http://www.w3.org/2000/svg">
+ <style>
+ @font-face {
+ font-family: "Fira Code";
+ src: local("FiraCode-Regular"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff2/FiraCode-Regular.woff2") format("woff2"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff/FiraCode-Regular.woff") format("woff");
+ font-style: normal;
+ font-weight: 400;
+ }
+ @font-face {
+ font-family: "Fira Code";
+ src: local("FiraCode-Bold"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff2/FiraCode-Bold.woff2") format("woff2"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff/FiraCode-Bold.woff") format("woff");
+ font-style: bold;
+ font-weight: 700;
+ }
+ span {
+ display: inline-block;
+ white-space: pre;
+ vertical-align: top;
+ font-size: 18px;
+ font-family:'Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
+ }
+ a {
+ text-decoration: none;
+ color: inherit;
+ }
+ .blink {
+ animation: blinker 1s infinite;
+ }
+ @keyframes blinker {
+ from { opacity: 1.0; }
+ 50% { opacity: 0.3; }
+ to { opacity: 1.0; }
+ }
+ #wrapper {
+ padding: 140px;
+ padding-top: 100px;
+ }
+ #terminal {
+ position: relative;
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ background-color: #0c0c0c;
+ border-radius: 14px;
+ outline: 1px solid #484848;
+ }
+ #terminal:after {
+ position: absolute;
+ width: 100%;
+ height: 100%;
+ content: '';
+ border-radius: 14px;
+ background: rgb(71,77,102);
+ background: linear-gradient(90deg, #804D69 0%, #4E4B89 100%);
+ transform: rotate(-4.5deg);
+ z-index: -1;
+ }
+ #terminal-header {
+ position: relative;
+ width: 100%;
+ background-color: #2e2e2e;
+ margin-bottom: 12px;
+ font-weight: bold;
+ border-radius: 14px 14px 0 0;
+ color: #f2f2f2;
+ font-size: 18px;
+ box-shadow: inset 0px -1px 0px 0px #4e4e4e,
+ inset 0px -4px 8px 0px #1a1a1a;
+ }
+ #terminal-title-tab {
+ display: inline-block;
+ margin-top: 14px;
+ margin-left: 124px;
+ font-family: sans-serif;
+ padding: 14px 28px;
+ border-radius: 6px 6px 0 0;
+ background-color: #0c0c0c;
+ box-shadow: inset 0px 1px 0px 0px #4e4e4e,
+ 0px -4px 4px 0px #1e1e1e,
+ inset 1px 0px 0px 0px #4e4e4e,
+ inset -1px 0px 0px 0px #4e4e4e;
+ }
+ #terminal-traffic-lights {
+ position: absolute;
+ top: 24px;
+ left: 20px;
+ }
+ #terminal-body {
+ line-height: 22px;
+ padding: 14px;
+ }
+ .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+ </style>
+ <foreignObject x="0" y="0" width="100%" height="100%">
+ <body xmlns="http://www.w3.org/1999/xhtml">
+ <div id="wrapper">
+ <div id="terminal">
+ <div id='terminal-header'>
+ <svg id="terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <circle cx="14" cy="8" r="8" fill="#ff6159"/>
+ <circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
+ <circle cx="62" cy="8" r="8" fill="#28c941"/>
+ </svg>
+ <div id="terminal-title-tab">Command: resource-check</div>
+ </div>
+ <div id='terminal-body'>
+ <div><span class="r2"> </span></div>
+<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze resource-check [OPTIONS] </span></div>
+<div><span class="r2"> </span></div>
+<div><span class="r1"> </span><span class="r1">Check if available docker resources are enough. </span><span class="r1"> </span></div>
+<div><span class="r1"> </span></div>
+<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verbose</span><span class="r1"> </span><span class="r6">-v</span><span class="r1"> </span><span class="r1">Print verbose information about performed steps. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--dry-run</span><span class="r1"> </span><span class="r6">-D</span><span class="r1"> </span><span class="r1">If dry-run is set, commands are only printed, not executed. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r6">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
+<div><span class="r1"></span><span class="r1"> </span></div>
+ </div>
+ </div>
+ </div>
+ </body>
+ </foreignObject>
+</svg>
diff --git a/kubernetes_tests/test_other_executors.py b/kubernetes_tests/test_other_executors.py
index e2b7037779..9c92dbca4a 100644
--- a/kubernetes_tests/test_other_executors.py
+++ b/kubernetes_tests/test_other_executors.py
@@ -18,6 +18,7 @@
import time
import pytest
+
from kubernetes_tests.test_base import EXECUTOR, TestBase
diff --git a/scripts/ci/tools/fix_ownership.sh b/scripts/ci/tools/fix_ownership.sh
deleted file mode 100755
index 1dc7b92e4e..0000000000
--- a/scripts/ci/tools/fix_ownership.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#
-# Fixes ownership for files created inside container (files owned by root will be owned by host user)
-#
-# shellcheck source=scripts/ci/libraries/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-
-if [[ ${OSTYPE} == "darwin"* ]]; then
- # No need to fix ownership on MacOS - the filesystem there takes care about ownership mapping
- exit
-fi
-
-declare -a EXTRA_DOCKER_FLAGS
-
-sanity_checks::sanitize_mounted_files
-
-read -r -a EXTRA_DOCKER_FLAGS <<<"$(local_mounts::convert_local_mounts_to_docker_params)"
-
-if docker image inspect "${AIRFLOW_CI_IMAGE_WITH_TAG}" >/dev/null 2>&1; then
- docker_v run --entrypoint /bin/bash "${EXTRA_DOCKER_FLAGS[@]}" \
- --rm \
- --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \
- "${AIRFLOW_CI_IMAGE_WITH_TAG}" \
- -c /opt/airflow/scripts/in_container/run_fix_ownership.sh || true
-else
- echo "Skip fixing ownership as seems that you do not have the ${AIRFLOW_CI_IMAGE_WITH_TAG} image yet"
-fi
diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh
index bb81e7aa34..57f4e4d40d 100644
--- a/scripts/in_container/_in_container_utils.sh
+++ b/scripts/in_container/_in_container_utils.sh
@@ -123,8 +123,18 @@ function in_container_fix_ownership() {
"/opt/airflow/dags"
"${AIRFLOW_SOURCES}"
)
+ echo
+ echo "${COLOR_BLUE}Fixing ownership of generated files as Host OS is ${HOST_OS}${COLOR_RESET}"
+ echo "${COLOR_BLUE}Directories: ${DIRECTORIES_TO_FIX[*]}${COLOR_RESET}"
+ echo
find "${DIRECTORIES_TO_FIX[@]}" -print0 -user root 2>/dev/null |
xargs --null chown "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference || true >/dev/null 2>&1
+ echo "${COLOR_BLUE}Fixed ownership of generated files."
+ echo
+ else
+ echo
+ echo "${COLOR_YELLOW}Skip fixing ownership of generated files as Host OS is ${HOST_OS}${COLOR_RESET}"
+ echo
fi
}