You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2023/11/19 02:33:57 UTC
(airflow) branch switch-building-airflow-packages-to-generic-image updated (da277223dc -> 44e5e379a3)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch switch-building-airflow-packages-to-generic-image
in repository https://gitbox.apache.org/repos/asf/airflow.git
omit da277223dc Switch building airlfow packages to gneneric images instead of CI image
new 44e5e379a3 Switch building airlfow packages to gneneric images instead of CI image
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (da277223dc)
\
N -- N -- N refs/heads/switch-building-airflow-packages-to-generic-image (44e5e379a3)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
scripts/in_container/run_prepare_airflow_packages.py | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
(airflow) 01/01: Switch building airlfow packages to gneneric images instead of CI image
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch switch-building-airflow-packages-to-generic-image
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 44e5e379a39dfda62febf66ce1663e2ba9c17a9a
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Nov 19 02:49:26 2023 +0100
Switch building airlfow packages to gneneric images instead of CI image
Since we are using setuptools (for now) to build airflow packages,
we still want to run the build in docker image in order to not take
the risk that code from forks is executed as part of the job that can
have access to secrets and package writing. However we do not need
the whole CI image. In order to build airflow packages we need just
few dependencies and we can easily build image needed during the
build.
This PR switches the image used and uses Python script to do it
rather than bash script used previously.
Building PROD image is also faster now as it does not need to pull
CI image to build packages.
---
.github/workflows/build-images.yml | 7 --
.github/workflows/ci.yml | 15 ----
.../commands/release_management_commands.py | 65 ++++++++++-----
.../commands/release_management_commands_config.py | 2 -
.../airflow_breeze/utils/docker_command_utils.py | 1 -
..._release-management_prepare-airflow-package.svg | 30 +++----
..._release-management_prepare-airflow-package.txt | 2 +-
scripts/ci/docker-compose/_docker.env | 1 -
scripts/ci/docker-compose/base.yml | 1 -
scripts/ci/docker-compose/devcontainer.env | 1 -
scripts/in_container/_in_container_utils.sh | 62 --------------
.../in_container/run_prepare_airflow_packages.py | 96 ++++++++++++++++++++++
.../in_container/run_prepare_airflow_packages.sh | 94 ---------------------
13 files changed, 152 insertions(+), 225 deletions(-)
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index 5872978158..82498ebde0 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -287,13 +287,6 @@ jobs:
mv -v "target-airflow/.github/actions" ".github"
- name: "Install Breeze"
uses: ./.github/actions/breeze
- - name: >
- Pull CI image for PROD build:
- ${{ steps.selective-checks.outputs.default-python-version }}:${{ env.IMAGE_TAG }}"
- shell: bash
- run: breeze ci-image pull --tag-as-latest
- env:
- PYTHON_MAJOR_MINOR_VERSION: ${{ steps.selective-checks.outputs.default-python-version }}
- name: >
Build PROD Images
${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e4a50eb0fa..6c81eea14a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -418,14 +418,6 @@ jobs:
- name: "Install Breeze"
uses: ./.github/actions/breeze
if: needs.build-info.outputs.in-workflow-build == 'true'
- - name: >
- Pull CI image for PROD build:
- ${{ steps.selective-checks.outputs.default-python-version }}:${{ env.IMAGE_TAG }}"
- shell: bash
- run: breeze ci-image pull --tag-as-latest
- env:
- PYTHON_MAJOR_MINOR_VERSION: ${{ steps.selective-checks.outputs.default-python-version }}
- if: needs.build-info.outputs.in-workflow-build == 'true'
- name: >
Build PROD Images
${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
@@ -467,13 +459,6 @@ jobs:
submodules: recursive
- name: "Install Breeze"
uses: ./.github/actions/breeze
- - name: >
- Pull CI image for PROD build:
- ${{ steps.selective-checks.outputs.default-python-version }}:${{ env.IMAGE_TAG }}"
- shell: bash
- run: breeze ci-image pull --tag-as-latest
- env:
- PYTHON_MAJOR_MINOR_VERSION: ${{ steps.selective-checks.outputs.default-python-version }}
- name: >
Build Bullseye PROD Images
${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index fe3494541b..f13ac9462e 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -36,11 +36,11 @@ from rich.syntax import Syntax
from airflow_breeze.commands.ci_image_commands import rebuild_or_pull_ci_image_if_needed
from airflow_breeze.commands.release_management_group import release_management
from airflow_breeze.global_constants import (
+ ALLOWED_DEBIAN_VERSIONS,
ALLOWED_PLATFORMS,
APACHE_AIRFLOW_GITHUB_REPOSITORY,
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS,
DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
- MOUNT_ALL,
MOUNT_SELECTED,
MULTI_PLATFORM,
)
@@ -200,43 +200,66 @@ echo -e '\\e[34mRun this command to debug:
)
+AIRFLOW_PIP_VERSION = "23.3.1"
+WHEEL_VERSION = "0.36.2"
+GITPYTHON_VERSION = "3.1.40"
+RICH_VERSION = "13.7.0"
+
+
+AIRFLOW_BUILD_DOCKERFILE = f"""
+FROM python:{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}-slim-{ALLOWED_DEBIAN_VERSIONS[0]}
+RUN apt-get update && apt-get install -y --no-install-recommends git
+RUN pip install pip=={AIRFLOW_PIP_VERSION} \\
+ wheel=={WHEEL_VERSION} gitpython=={GITPYTHON_VERSION} rich=={RICH_VERSION}
+"""
+
+AIRFLOW_BUILD_IMAGE_TAG = "airflow-build-image"
+
+
@release_management.command(
name="prepare-airflow-package",
help="Prepare sdist/whl package of Airflow.",
)
@option_package_format
@option_version_suffix_for_pypi
-@option_debug_release_management
-@option_github_repository
@option_verbose
@option_dry_run
def prepare_airflow_packages(
package_format: str,
version_suffix_for_pypi: str,
- debug: bool,
- github_repository: str,
):
perform_environment_checks()
cleanup_python_generated_files()
assert_pre_commit_installed()
- run_compile_www_assets(dev=False, run_in_background=False)
- shell_params = ShellParams(
- github_repository=github_repository,
- python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
- package_format=package_format,
- version_suffix_for_pypi=version_suffix_for_pypi,
- skip_environment_initialization=True,
- install_providers_from_sources=False,
- mount_sources=MOUNT_ALL,
+ result = run_compile_www_assets(dev=False, run_in_background=False)
+ if result.returncode != 0:
+ get_console().print("[error]Failed to compile assets\n")
+ sys.exit(result.returncode)
+ get_console().print("[success]Compiled assets\n")
+ run_command(
+ ["docker", "build", "--tag", AIRFLOW_BUILD_IMAGE_TAG, "-"],
+ input=AIRFLOW_BUILD_DOCKERFILE,
+ text=True,
+ check=True,
+ env={"DOCKER_CLI_HINTS": "false"},
)
- rebuild_or_pull_ci_image_if_needed(command_params=shell_params)
- result_command = run_docker_command_with_debug(
- params=shell_params,
- command=["/opt/airflow/scripts/in_container/run_prepare_airflow_packages.sh"],
- debug=debug,
- output_outside_the_group=True,
+ run_command(
+ cmd=[
+ "docker",
+ "run",
+ "-t",
+ "-v",
+ f"{AIRFLOW_SOURCES_ROOT}:/opt/airflow:cached",
+ "-e",
+ f"VERSION_SUFFIX_FOR_PYPI={version_suffix_for_pypi}",
+ "-e",
+ f"PACKAGE_FORMAT={package_format}",
+ AIRFLOW_BUILD_IMAGE_TAG,
+ "python",
+ "/opt/airflow/scripts/in_container/run_prepare_airflow_packages.py",
+ ],
+ check=True,
)
- sys.exit(result_command.returncode)
def provider_action_summary(description: str, message_type: MessageType, packages: list[str]):
diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
index a5b84869b4..c1bf905f93 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
@@ -55,8 +55,6 @@ RELEASE_MANAGEMENT_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = {
"options": [
"--package-format",
"--version-suffix-for-pypi",
- "--debug",
- "--github-repository",
],
}
],
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index df6817b459..b92361d7d6 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -612,7 +612,6 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
set_value_to_default_if_not_set(env, "VERBOSE", "false")
set_value_to_default_if_not_set(env, "VERBOSE_COMMANDS", "false")
set_value_to_default_if_not_set(env, "VERSION_SUFFIX_FOR_PYPI", "")
- set_value_to_default_if_not_set(env, "WHEEL_VERSION", "0.36.2")
DERIVE_ENV_VARIABLES_FROM_ATTRIBUTES = {
diff --git a/images/breeze/output_release-management_prepare-airflow-package.svg b/images/breeze/output_release-management_prepare-airflow-package.svg
index 6d2d35508c..c8490f5700 100644
--- a/images/breeze/output_release-management_prepare-airflow-package.svg
+++ b/images/breeze/output_release-management_prepare-airflow-package.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 440.4" xmlns="http://www.w3.org/2000/svg">
+<svg class="rich-terminal" viewBox="0 0 1482 391.59999999999997" xmlns="http://www.w3.org/2000/svg">
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -43,7 +43,7 @@
<defs>
<clipPath id="breeze-release-management-prepare-airflow-package-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="389.4" />
+ <rect x="0" y="0" width="1463.0" height="340.59999999999997" />
</clipPath>
<clipPath id="breeze-release-management-prepare-airflow-package-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -84,15 +84,9 @@
<clipPath id="breeze-release-management-prepare-airflow-package-line-12">
<rect x="0" y="294.3" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="breeze-release-management-prepare-airflow-package-line-13">
- <rect x="0" y="318.7" width="1464" height="24.65"/>
- </clipPath>
-<clipPath id="breeze-release-management-prepare-airflow-package-line-14">
- <rect x="0" y="343.1" width="1464" height="24.65"/>
- </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="438.4" rx="8"/><text class="breeze-release-management-prepare-airflow-package-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management prepare-airflow-package</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="389.6" rx="8"/><text class="breeze-release-management-prepare-airflow-package-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management prepare-airflow-package</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -108,16 +102,14 @@
</text><text class="breeze-release-management-prepare-airflow-package-r1" x="12.2" y="93.2" textLength="451.4" clip-path="url(#breeze-release-management-prepare-airflow-package-line-3)">Prepare sdist/whl package of Airflow.</text><text class="breeze-release-management-prepare-airflow-package-r1" x="1464" y="93.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-3)">
</text><text class="breeze-release-management-prepare-airflow-package-r1" x="1464" y="117.6" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-4)">
</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="142" textLength="24.4" clip-path="url(#breeze-release-management-prepare-airflow-package-line-5)">╭─</text><text class="breeze-release-management-prepare-airflow-package-r5" x="24.4" y="142" textLength="183" clip-path="url(#breeze-release-management-prepare-airflow-package-line-5)"> Package flags </text><text class="breeze-release-management-prepare-airflow-package-r5" x="207.4" y="142" text [...]
-</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="166.4" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-6)">│</text><text class="breeze-release-management-prepare-airflow-package-r4" x="24.4" y="166.4" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-6)">-</text><text class="breeze-release-management-prepare-airflow-package-r4" x="36.6" y="166.4" textLength="97.6" clip-path= [...]
-</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="190.8" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-7)">│</text><text class="breeze-release-management-prepare-airflow-package-r4" x="24.4" y="190.8" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-7)">-</text><text class="breeze-release-management-prepare-airflow-package-r4" x="36.6" y="190.8" textLength="97.6" clip-path= [...]
-</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="215.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-8)">│</text><text class="breeze-release-management-prepare-airflow-package-r4" x="24.4" y="215.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-8)">-</text><text class="breeze-release-management-prepare-airflow-package-r4" x="36.6" y="215.2" textLength="73.2" clip-path= [...]
-</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-9)">│</text><text class="breeze-release-management-prepare-airflow-package-r4" x="24.4" y="239.6" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-9)">-</text><text class="breeze-release-management-prepare-airflow-package-r4" x="36.6" y="239.6" textLength="85.4" clip-path= [...]
-</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="264" textLength="1464" clip-path="url(#breeze-release-management-prepare-airflow-package-line-10)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-prepare-airflow-package-r1" x="1464" y="264" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-10)">
-</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="288.4" textLength="24.4" clip-path="url(#breeze-release-management-prepare-airflow-package-line-11)">╭─</text><text class="breeze-release-management-prepare-airflow-package-r5" x="24.4" y="288.4" textLength="195.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-11)"> Common options </text><text class="breeze-release-management-prepare-airflow-package-r5" x="219.6" y=" [...]
-</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-12)">│</text><text class="breeze-release-management-prepare-airflow-package-r4" x="24.4" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-12)">-</text><text class="breeze-release-management-prepare-airflow-package-r4" x="36.6" y="312.8" textLength="97.6" clip-pat [...]
-</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-13)">│</text><text class="breeze-release-management-prepare-airflow-package-r4" x="24.4" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-13)">-</text><text class="breeze-release-management-prepare-airflow-package-r4" x="36.6" y="337.2" textLength="48.8" clip-pat [...]
-</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-14)">│</text><text class="breeze-release-management-prepare-airflow-package-r4" x="24.4" y="361.6" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-14)">-</text><text class="breeze-release-management-prepare-airflow-package-r4" x="36.6" y="361.6" textLength="61" clip-path= [...]
-</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="386" textLength="1464" clip-path="url(#breeze-release-management-prepare-airflow-package-line-15)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-prepare-airflow-package-r1" x="1464" y="386" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-15)">
+</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="166.4" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-6)">│</text><text class="breeze-release-management-prepare-airflow-package-r4" x="24.4" y="166.4" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-6)">-</text><text class="breeze-release-management-prepare-airflow-package-r4" x="36.6" y="166.4" textLength="97.6" clip-path= [...]
+</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="190.8" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-7)">│</text><text class="breeze-release-management-prepare-airflow-package-r4" x="24.4" y="190.8" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-7)">-</text><text class="breeze-release-management-prepare-airflow-package-r4" x="36.6" y="190.8" textLength="97.6" clip-path= [...]
+</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="215.2" textLength="1464" clip-path="url(#breeze-release-management-prepare-airflow-package-line-8)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-prepare-airflow-package-r1" x="1464" y="215.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-8)">
+</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="239.6" textLength="24.4" clip-path="url(#breeze-release-management-prepare-airflow-package-line-9)">╭─</text><text class="breeze-release-management-prepare-airflow-package-r5" x="24.4" y="239.6" textLength="195.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-9)"> Common options </text><text class="breeze-release-management-prepare-airflow-package-r5" x="219.6" y="23 [...]
+</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="264" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-10)">│</text><text class="breeze-release-management-prepare-airflow-package-r4" x="24.4" y="264" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-10)">-</text><text class="breeze-release-management-prepare-airflow-package-r4" x="36.6" y="264" textLength="97.6" clip-path="url [...]
+</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-11)">│</text><text class="breeze-release-management-prepare-airflow-package-r4" x="24.4" y="288.4" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-11)">-</text><text class="breeze-release-management-prepare-airflow-package-r4" x="36.6" y="288.4" textLength="48.8" clip-pat [...]
+</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-12)">│</text><text class="breeze-release-management-prepare-airflow-package-r4" x="24.4" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-12)">-</text><text class="breeze-release-management-prepare-airflow-package-r4" x="36.6" y="312.8" textLength="61" clip-path= [...]
+</text><text class="breeze-release-management-prepare-airflow-package-r5" x="0" y="337.2" textLength="1464" clip-path="url(#breeze-release-management-prepare-airflow-package-line-13)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-prepare-airflow-package-r1" x="1464" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-airflow-package-line-13)">
</text>
</g>
</g>
diff --git a/images/breeze/output_release-management_prepare-airflow-package.txt b/images/breeze/output_release-management_prepare-airflow-package.txt
index d2e8102ace..f295947476 100644
--- a/images/breeze/output_release-management_prepare-airflow-package.txt
+++ b/images/breeze/output_release-management_prepare-airflow-package.txt
@@ -1 +1 @@
-85d01c57e5b5ee0fb9e5f9d9706ed3b5
+8f786828b9862f9ae63f1b691921979f
diff --git a/scripts/ci/docker-compose/_docker.env b/scripts/ci/docker-compose/_docker.env
index 4b7fc60a02..ae3c567bfd 100644
--- a/scripts/ci/docker-compose/_docker.env
+++ b/scripts/ci/docker-compose/_docker.env
@@ -83,4 +83,3 @@ DOWNGRADE_SQLALCHEMY
VERBOSE
VERBOSE_COMMANDS
VERSION_SUFFIX_FOR_PYPI
-WHEEL_VERSION
diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml
index 86c51a0b9a..76e41adfcf 100644
--- a/scripts/ci/docker-compose/base.yml
+++ b/scripts/ci/docker-compose/base.yml
@@ -93,7 +93,6 @@ services:
- VERBOSE=${VERBOSE}
- VERBOSE_COMMANDS=${VERBOSE_COMMANDS}
- VERSION_SUFFIX_FOR_PYPI=${VERSION_SUFFIX_FOR_PYPI}
- - WHEEL_VERSION=${WHEEL_VERSION}
volumes:
# Pass docker to inside of the container so that Kind and Moto tests can use it.
# NOTE! Even if we are using "desktop-linux" context where "/var/run/docker.sock" is not used,
diff --git a/scripts/ci/docker-compose/devcontainer.env b/scripts/ci/docker-compose/devcontainer.env
index dd7da1ac86..aa5798beb0 100644
--- a/scripts/ci/docker-compose/devcontainer.env
+++ b/scripts/ci/docker-compose/devcontainer.env
@@ -74,5 +74,4 @@ UPGRADE_TO_NEWER_DEPENDENCIES="false"
VERBOSE="false"
VERBOSE_COMMANDS="false"
VERSION_SUFFIX_FOR_PYPI=
-WHEEL_VERSION=0.36.2
AIRFLOW_SOURCES=/workspaces/airflow
diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh
index 6be73d2936..b3606973c4 100644
--- a/scripts/in_container/_in_container_utils.sh
+++ b/scripts/in_container/_in_container_utils.sh
@@ -105,29 +105,6 @@ function in_container_basic_sanity_check() {
in_container_go_to_airflow_sources
}
-export DISABLE_CHECKS_FOR_TESTS="missing-docstring,no-self-use,too-many-public-methods,protected-access,do-not-use-asserts"
-
-function start_output_heartbeat() {
- MESSAGE=${1:-"Still working!"}
- INTERVAL=${2:=10}
- echo
- echo "Starting output heartbeat"
- echo
-
- bash 2>/dev/null <<EOF &
-while true; do
- echo "\$(date): ${MESSAGE} "
- sleep ${INTERVAL}
-done
-EOF
- export HEARTBEAT_PID=$!
-}
-
-function stop_output_heartbeat() {
- kill "${HEARTBEAT_PID}" || true
- wait "${HEARTBEAT_PID}" || true 2>/dev/null
-}
-
function dump_airflow_logs() {
local dump_file
dump_file=/files/airflow_logs_$(date "+%Y-%m-%d")_${CI_BUILD_ID}_${CI_JOB_ID}.log.tar.gz
@@ -331,45 +308,6 @@ function install_all_providers_from_pypi_with_eager_upgrade() {
set +x
}
-function install_all_provider_packages_from_wheels() {
- echo
- echo "Installing all provider packages from wheels"
- echo
- uninstall_providers
- echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
- ls -w 1 /dist
- echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
- pip install /dist/apache_airflow_providers_*.whl
-}
-
-function install_all_provider_packages_from_sdist() {
- echo
- echo "Installing all provider packages from .tar.gz"
- echo
- uninstall_providers
- echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
- ls -w 1 /dist
- echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
- pip install /dist/apache-airflow-providers-*.tar.gz
-}
-
-function twine_check_provider_packages_from_wheels() {
- echo
- echo "Twine check of all provider packages from wheels"
- echo
- echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
- ls -w 1 /dist
- echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
- twine check /dist/apache_airflow_providers_*.whl
-}
-
-function twine_check_provider_packages_from_sdist() {
- echo
- echo "Twine check all provider packages from sdist"
- echo
- twine check /dist/apache-airflow-providers-*.tar.gz
-}
-
function install_supported_pip_version() {
if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
pip install --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}"
diff --git a/scripts/in_container/run_prepare_airflow_packages.py b/scripts/in_container/run_prepare_airflow_packages.py
new file mode 100755
index 0000000000..1e15598ff2
--- /dev/null
+++ b/scripts/in_container/run_prepare_airflow_packages.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+from shutil import rmtree
+
+import rich
+
+AIRFLOW_SOURCES_ROOT = Path(__file__).parents[2].resolve()
+
+rich.print("[green]Installed packages\n")
+
+rich.print("[bright_blue]Cleaning build directories\n")
+
+for egg_info_file in AIRFLOW_SOURCES_ROOT.glob("*egg-info*"):
+ rmtree(egg_info_file, ignore_errors=True)
+
+rmtree(AIRFLOW_SOURCES_ROOT / "build", ignore_errors=True)
+
+rich.print("[bright_blue]Cleaned build directories\n")
+
+version_suffix = os.environ.get("VERSION_SUFFIX_FOR_PYPI", "")
+package_format = os.environ.get("PACKAGE_FORMAT", "wheel")
+
+command = [sys.executable, "setup.py"]
+
+airflow_version = subprocess.check_output(
+ [sys.executable, "setup.py", "--version"], text=True, cwd=AIRFLOW_SOURCES_ROOT
+)
+
+subprocess.run(
+ ["git", "config", "--global", "--unset-all", "safe.directory"],
+ cwd=AIRFLOW_SOURCES_ROOT,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ check=False,
+)
+
+subprocess.run(
+ ["git", "config", "--global", "--add", "safe.directory", AIRFLOW_SOURCES_ROOT],
+ cwd=AIRFLOW_SOURCES_ROOT,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ check=True,
+)
+
+RELEASED_VERSION_MATCHER = re.compile(r"^\d+\.\d+\.\d+$")
+
+if version_suffix:
+ if RELEASED_VERSION_MATCHER.match(airflow_version):
+ rich.print(f"[bright_blue]Adding {version_suffix} suffix to the {airflow_version}")
+ command.extend(["egg_info", "--tag-build", version_suffix])
+ elif not airflow_version.endswith(version_suffix):
+ rich.print(f"[red]Version {airflow_version} does not end with {version_suffix}. Exiting!")
+ sys.exit(1)
+
+if package_format in ["both", "wheel"]:
+ command.append("bdist_wheel")
+if package_format in ["both", "sdist"]:
+ command.append("sdist")
+
+rich.print(f"[bright_blue]Building packages: {package_format}\n")
+
+process = subprocess.run(command, capture_output=True, text=True, cwd=AIRFLOW_SOURCES_ROOT)
+
+if process.returncode != 0:
+ rich.print("[red]Packages build failed")
+ print(process.stdout)
+ print(process.stderr)
+ sys.exit(process.returncode)
+else:
+ rich.print("[green]Packages built successfully:\n")
+ for file in (AIRFLOW_SOURCES_ROOT / "dist").glob("apache*"):
+ rich.print(file.name)
+ rich.print()
diff --git a/scripts/in_container/run_prepare_airflow_packages.sh b/scripts/in_container/run_prepare_airflow_packages.sh
deleted file mode 100755
index afcceedbf0..0000000000
--- a/scripts/in_container/run_prepare_airflow_packages.sh
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# shellcheck source=scripts/in_container/_in_container_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
-
-function prepare_airflow_packages() {
- echo "-----------------------------------------------------------------------------------"
- if [[ "${VERSION_SUFFIX_FOR_PYPI}" == '' ]]; then
- echo
- echo "Preparing official version of provider with no suffixes"
- echo
- else
- echo
- echo " Package Version of providers suffix set for PyPI version: ${VERSION_SUFFIX_FOR_PYPI}"
- echo
- fi
- echo "-----------------------------------------------------------------------------------"
-
- rm -rf -- *egg-info*
- rm -rf -- build
-
- install_supported_pip_version
- pip install "wheel==${WHEEL_VERSION}"
-
- local packages=()
-
- if [[ ${PACKAGE_FORMAT} == "wheel" || ${PACKAGE_FORMAT} == "both" ]] ; then
- packages+=("bdist_wheel")
- fi
- if [[ ${PACKAGE_FORMAT} == "sdist" || ${PACKAGE_FORMAT} == "both" ]] ; then
- packages+=("sdist")
- fi
- local tag_build=()
- if [[ -n ${VERSION_SUFFIX_FOR_PYPI} ]]; then
- AIRFLOW_VERSION=$(python setup.py --version)
- if [[ ${AIRFLOW_VERSION} =~ ^[0-9\.]+$ ]]; then
- echo
- echo "${COLOR_BLUE}Adding ${VERSION_SUFFIX_FOR_PYPI} suffix to ${AIRFLOW_VERSION}${COLOR_RESET}"
- echo
- tag_build=('egg_info' '--tag-build' "${VERSION_SUFFIX_FOR_PYPI}")
- else
- if [[ ${AIRFLOW_VERSION} != *${VERSION_SUFFIX_FOR_PYPI} ]]; then
- echo
- echo "${COLOR_YELLOW}The requested PyPI suffix ${VERSION_SUFFIX_FOR_PYPI} does not match the one in ${AIRFLOW_VERSION}. Overriding it with one from ${VERSION_SUFFIX_FOR_PYPI}.${COLOR_RESET}"
- echo
- fi
- fi
- fi
-
- # Prepare airflow's wheel
- PYTHONUNBUFFERED=1 python setup.py "${tag_build[@]}" "${packages[@]}"
-
- # clean-up
- rm -rf -- *egg-info*
- rm -rf -- build
-
- echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
- echo
- echo "${COLOR_GREEN}Airflow package prepared in format: ${PACKAGE_FORMAT}${COLOR_RESET}"
- echo
- echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
- ls -w 1 ./dist
- echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
-}
-
-function mark_directory_as_safe() {
- git config --global --unset-all safe.directory || true
- git config --global --add safe.directory /opt/airflow
-}
-
-install_supported_pip_version
-
-mark_directory_as_safe
-
-prepare_airflow_packages
-
-echo
-echo "${COLOR_GREEN}All good! Airflow packages are prepared in dist folder${COLOR_RESET}"
-echo