You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2023/12/01 00:09:55 UTC
(airflow) branch main updated: Add feature to build "chicken-egg" packages from sources (#35890)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new cf052dc64f Add feature to build "chicken-egg" packages from sources (#35890)
cf052dc64f is described below
commit cf052dc64f00e851427a41a34ffe576fd39be51b
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Fri Dec 1 01:09:48 2023 +0100
Add feature to build "chicken-egg" packages from sources (#35890)
When we build on ci a pre-release version of provider, and we want
to include packages that have >= CURRENTLY_RELEASED_VERSION
we have to make sure that those packages are built from sources
during building of PROD image. Otherwise they will not be installable
on CURRENT_VERSION.dev0, CURRENT_VERSION.rc* etc.
Until we "Actually" release a provider we should have a way to
build such provider from sources.
This is the CI version of it, once we have it working we can also
apply it to the workflow that releases images to dockerhub.
---
.github/actions/build-prod-images/action.yml | 12 +++++
.github/workflows/build-images.yml | 2 +
.github/workflows/ci.yml | 15 ++++++
airflow/providers/amazon/provider.yaml | 2 +-
airflow/providers/google/provider.yaml | 2 +-
airflow/providers/microsoft/azure/provider.yaml | 2 +-
.../commands/release_management_commands.py | 25 +++++++---
.../commands/release_management_commands_config.py | 1 +
.../src/airflow_breeze/params/shell_params.py | 2 +
dev/breeze/src/airflow_breeze/utils/packages.py | 4 --
.../src/airflow_breeze/utils/selective_checks.py | 5 ++
docs/apache-airflow-providers-google/index.rst | 2 +-
.../index.rst | 2 +-
generated/provider_dependencies.json | 4 +-
...put_release-management_generate-constraints.svg | 58 ++++++++++++++--------
...put_release-management_generate-constraints.txt | 2 +-
scripts/in_container/_in_container_utils.sh | 23 +++++++--
setup.cfg | 2 +-
setup.py | 2 +-
19 files changed, 120 insertions(+), 47 deletions(-)
diff --git a/.github/actions/build-prod-images/action.yml b/.github/actions/build-prod-images/action.yml
index 5fdbb795c4..f038234087 100644
--- a/.github/actions/build-prod-images/action.yml
+++ b/.github/actions/build-prod-images/action.yml
@@ -22,6 +22,9 @@ inputs:
build-provider-packages:
description: 'Whether to build provider packages from sources'
required: true
+ chicken-egg-providers:
+ description: 'List of chicken-egg provider packages to build from sources'
+ required: true
runs:
using: "composite"
steps:
@@ -41,6 +44,15 @@ runs:
--package-list-file ./airflow/providers/installed_providers.txt
--package-format wheel --version-suffix-for-pypi dev0
if: ${{ inputs.build-provider-packages == 'true' }}
+ - name: "Prepare chicken-eggs provider packages"
+ # In case of provider packages which use latest dev0 version of providers, we should prepare them
+ # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency
+ # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0
+ shell: bash
+ run: >
+ breeze release-management prepare-provider-packages
+ --package-format wheel --version-suffix-for-pypi dev0 ${{ inputs.chicken-egg-providers }}
+ if: ${{ inputs.build-provider-packages != 'true' && inputs.chicken-egg-providers != '' }}
- name: "Prepare airflow package"
shell: bash
run: >
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index b29d49de17..82cc6e5987 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -74,6 +74,7 @@ jobs:
is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
+ chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha ||
github.event.pull_request.head.sha ||
github.sha
@@ -293,6 +294,7 @@ jobs:
uses: ./.github/actions/build-prod-images
with:
build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
+ chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
env:
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 74093010c0..3d7e9146f5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -121,6 +121,7 @@ jobs:
is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
+ chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
has-migrations: ${{ steps.selective-checks.outputs.has-migrations }}
source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }}
pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }}
@@ -474,6 +475,7 @@ jobs:
RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+ VERSION_SUFFIX_FOR_PYPI: "dev0"
if: needs.build-info.outputs.ci-image-build == 'true'
steps:
- name: Cleanup repo
@@ -502,12 +504,23 @@ jobs:
run: >
breeze release-management generate-constraints --run-in-parallel
--airflow-constraints-mode constraints-no-providers
+ - name: "Prepare chicken-eggs provider packages"
+ # In case of provider packages which use latest dev0 version of providers, we should prepare them
+ # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency
+ # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0
+ shell: bash
+ run: >
+ breeze release-management prepare-provider-packages
+ --package-format wheel --version-suffix-for-pypi dev0
+ ${{ needs.build-info.outputs.chicken-egg-providers }}
+ if: needs.build-info.outputs.chicken-egg-providers != ''
- name: "PyPI constraints"
shell: bash
timeout-minutes: 25
run: >
breeze release-management generate-constraints --run-in-parallel
--airflow-constraints-mode constraints
+ --chicken-egg-providers "${{ needs.build-info.outputs.chicken-egg-providers }}"
- name: "Dependency upgrade summary"
shell: bash
run: |
@@ -1608,6 +1621,7 @@ jobs:
if: needs.build-info.outputs.in-workflow-build == 'true'
with:
build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
+ chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
env:
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
@@ -1647,6 +1661,7 @@ jobs:
uses: ./.github/actions/build-prod-images
with:
build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
+ chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
env:
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml
index d6869683d0..798043abe8 100644
--- a/airflow/providers/amazon/provider.yaml
+++ b/airflow/providers/amazon/provider.yaml
@@ -753,7 +753,7 @@ additional-extras:
- apache-airflow-providers-cncf-kubernetes>=7.2.0
- name: s3fs
dependencies:
- - s3fs>=2023.9.2
+ - s3fs>=2023.10.0
- name: python3-saml
dependencies:
- python3-saml>=1.16.0
diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml
index 86a03ff900..7e6f69349c 100644
--- a/airflow/providers/google/provider.yaml
+++ b/airflow/providers/google/provider.yaml
@@ -87,7 +87,7 @@ dependencies:
- gcloud-aio-auth>=4.0.0,<5.0.0
- gcloud-aio-bigquery>=6.1.2
- gcloud-aio-storage
- - gcsfs>=2023.9.2
+ - gcsfs>=2023.10.0
- google-ads>=22.1.0
- google-api-core>=2.11.0
- google-api-python-client>=1.6.0
diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml
index 4f2308803d..186719a9af 100644
--- a/airflow/providers/microsoft/azure/provider.yaml
+++ b/airflow/providers/microsoft/azure/provider.yaml
@@ -70,7 +70,7 @@ versions:
dependencies:
- apache-airflow>=2.5.0
- - adlfs>=2023.9.2
+ - adlfs>=2023.10.0
- azure-batch>=8.0.0
- azure-cosmos>=4.0.0
- azure-mgmt-cosmosdb
diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index bbcf525e77..5aa1f65bf1 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -684,21 +684,30 @@ def run_generate_constraints_in_parallel(
@option_image_tag_for_running
@option_debug_release_management
@option_airflow_constraints_mode_ci
+@click.option(
+ "--chicken-egg-providers",
+ default="",
+ help="List of chicken-egg provider packages - "
+ "those that have airflow_version >= current_version and should "
+ "be installed in CI from locally built packages with >= current_version.dev0 ",
+ envvar="CHICKEN_EGG_PROVIDERS",
+)
@option_github_repository
@option_verbose
@option_dry_run
@option_answer
def generate_constraints(
+ airflow_constraints_mode: str,
+ debug: bool,
+ debug_resources: bool,
+ github_repository: str,
+ image_tag: str | None,
+ parallelism: int,
python: str,
+ python_versions: str,
run_in_parallel: bool,
- parallelism: int,
skip_cleanup: bool,
- debug_resources: bool,
- python_versions: str,
- image_tag: str | None,
- debug: bool,
- airflow_constraints_mode: str,
- github_repository: str,
+ chicken_egg_providers: str,
):
perform_environment_checks()
check_remote_ghcr_io_commands()
@@ -742,6 +751,7 @@ def generate_constraints(
python=python,
github_repository=github_repository,
airflow_constraints_mode=airflow_constraints_mode,
+ chicken_egg_providers=chicken_egg_providers,
)
for python in python_version_list
]
@@ -762,6 +772,7 @@ def generate_constraints(
skip_image_upgrade_check=True,
quiet=True,
airflow_constraints_mode=airflow_constraints_mode,
+ chicken_egg_providers=chicken_egg_providers,
)
return_code, info = run_generate_constraints(
shell_params=shell_params,
diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
index e27ccc3dd6..ff0c9cdcd0 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
@@ -134,6 +134,7 @@ RELEASE_MANAGEMENT_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = {
"--image-tag",
"--python",
"--airflow-constraints-mode",
+ "--chicken-egg-providers",
"--debug",
"--github-repository",
],
diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py
index 32f6a111ce..abfe2c4ddd 100644
--- a/dev/breeze/src/airflow_breeze/params/shell_params.py
+++ b/dev/breeze/src/airflow_breeze/params/shell_params.py
@@ -118,6 +118,7 @@ class ShellParams:
builder: str = "autodetect"
celery_broker: str = DEFAULT_CELERY_BROKER
celery_flower: bool = False
+ chicken_egg_providers: str = ""
collect_only: bool = False
database_isolation: bool = False
db_reset: bool = False
@@ -435,6 +436,7 @@ class ShellParams:
_set_var(_env, "BREEZE", "true")
_set_var(_env, "BREEZE_INIT_COMMAND", None, "")
_set_var(_env, "CELERY_FLOWER", self.celery_flower)
+ _set_var(_env, "CHICKEN_EGG_PROVIDERS", self.chicken_egg_providers)
_set_var(_env, "CI", None, "false")
_set_var(_env, "CI_BUILD_ID", None, "0")
_set_var(_env, "CI_EVENT_TYPE", None, "pull_request")
diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py
index 3d59643dbc..5518fdebd7 100644
--- a/dev/breeze/src/airflow_breeze/utils/packages.py
+++ b/dev/breeze/src/airflow_breeze/utils/packages.py
@@ -56,10 +56,6 @@ LONG_PROVIDERS_PREFIX = "apache-airflow-providers-"
# TODO: use single source of truth for those
# for now we need to keep them in sync with the ones in setup.py
PREINSTALLED_PROVIDERS = [
- # Until we cut off the 2.8.0 branch and bump current airflow version to 2.9.0, we should
- # Keep common.io commented out in order ot be able to generate PyPI constraints because
- # The version from PyPI has requirement of apache-airflow>=2.8.0
- # "common.io",
"common.sql",
"ftp",
"http",
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 0a751565ef..eba3023a6c 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -1019,3 +1019,8 @@ class SelectiveChecks:
@cached_property
def has_migrations(self) -> bool:
return any([file.startswith("airflow/migrations/") for file in self._files])
+
+ @cached_property
+ def chicken_egg_providers(self) -> str:
+ """Space separated list of providers with chicken-egg problem and should be built from sources."""
+ return "common.io"
diff --git a/docs/apache-airflow-providers-google/index.rst b/docs/apache-airflow-providers-google/index.rst
index d642d19ee2..b9defcd150 100644
--- a/docs/apache-airflow-providers-google/index.rst
+++ b/docs/apache-airflow-providers-google/index.rst
@@ -116,7 +116,7 @@ PIP package Version required
``gcloud-aio-auth`` ``>=4.0.0,<5.0.0``
``gcloud-aio-bigquery`` ``>=6.1.2``
``gcloud-aio-storage``
-``gcsfs`` ``>=2023.9.2``
+``gcsfs`` ``>=2023.10.0``
``google-ads`` ``>=22.1.0``
``google-api-core`` ``>=2.11.0``
``google-api-python-client`` ``>=1.6.0``
diff --git a/docs/apache-airflow-providers-microsoft-azure/index.rst b/docs/apache-airflow-providers-microsoft-azure/index.rst
index 95a23f3957..cc1467e64f 100644
--- a/docs/apache-airflow-providers-microsoft-azure/index.rst
+++ b/docs/apache-airflow-providers-microsoft-azure/index.rst
@@ -106,7 +106,7 @@ The minimum Apache Airflow version supported by this provider package is ``2.5.0
PIP package Version required
================================ ==================
``apache-airflow`` ``>=2.5.0``
-``adlfs`` ``>=2023.9.2``
+``adlfs`` ``>=2023.10.0``
``azure-batch`` ``>=8.0.0``
``azure-cosmos`` ``>=4.0.0``
``azure-mgmt-cosmosdb``
diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json
index 3b9d543169..e6405cfe77 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -415,7 +415,7 @@
"gcloud-aio-auth>=4.0.0,<5.0.0",
"gcloud-aio-bigquery>=6.1.2",
"gcloud-aio-storage",
- "gcsfs>=2023.9.2",
+ "gcsfs>=2023.10.0",
"google-ads>=22.1.0",
"google-api-core>=2.11.0",
"google-api-python-client>=1.6.0",
@@ -557,7 +557,7 @@
"microsoft.azure": {
"deps": [
"adal>=1.2.7",
- "adlfs>=2023.9.2",
+ "adlfs>=2023.10.0",
"apache-airflow>=2.5.0",
"azure-batch>=8.0.0",
"azure-cosmos>=4.0.0",
diff --git a/images/breeze/output_release-management_generate-constraints.svg b/images/breeze/output_release-management_generate-constraints.svg
index 9801441959..0fd9c471c2 100644
--- a/images/breeze/output_release-management_generate-constraints.svg
+++ b/images/breeze/output_release-management_generate-constraints.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 855.1999999999999" xmlns="http://www.w3.org/2000/svg">
+<svg class="rich-terminal" viewBox="0 0 1482 952.8" xmlns="http://www.w3.org/2000/svg">
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -43,7 +43,7 @@
<defs>
<clipPath id="breeze-release-management-generate-constraints-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="804.1999999999999" />
+ <rect x="0" y="0" width="1463.0" height="901.8" />
</clipPath>
<clipPath id="breeze-release-management-generate-constraints-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -141,9 +141,21 @@
<clipPath id="breeze-release-management-generate-constraints-line-31">
<rect x="0" y="757.9" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-release-management-generate-constraints-line-32">
+ <rect x="0" y="782.3" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-release-management-generate-constraints-line-33">
+ <rect x="0" y="806.7" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-release-management-generate-constraints-line-34">
+ <rect x="0" y="831.1" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-release-management-generate-constraints-line-35">
+ <rect x="0" y="855.5" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="853.2" rx="8"/><text class="breeze-release-management-generate-constraints-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management generate-constraints</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="950.8" rx="8"/><text class="breeze-release-management-generate-constraints-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management generate-constraints</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -168,24 +180,28 @@
</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-12)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-12)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="312.8" textLength="97.6" clip-path="url(#breeze- [...]
</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-13)">│</text><text class="breeze-release-management-generate-constraints-r6" x="414.8" y="337.2" textLength="866.2" clip-path="url(#breeze-release-management-generate-constraints-line-13)">(constraints-source-providers | constraints | constraints-no-providers)</text><text class="breeze-release-mana [...]
</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-14)">│</text><text class="breeze-release-management-generate-constraints-r5" x="414.8" y="361.6" textLength="866.2" clip-path="url(#breeze-release-management-generate-constraints-line-14)">[default: constraints-source-providers]              &# [...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="386" textLength="73.2" clip-path="url(#breeze-releas [...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-16)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-16)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="410.4" textLength="85.4" clip-path="url(#breeze- [...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="434.8" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-17)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-17)">
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="459.2" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-18)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="459.2" textLength="219.6" clip-path="url(#breeze-release-management-generate-constraints-line-18)"> Parallel running </text><text class="breeze-release-management-generate-constraints-r5" x="244" y="459.2" textLeng [...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="483.6" textLength="48.8" clip-path="url(#breeze- [...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-20)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="508" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-20)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="508" textLength="146.4" clip-path="url(#breeze-relea [...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-21)">│</text><text class="breeze-release-management-generate-constraints-r5" x="280.6" y="532.4" textLength="915" clip-path="url(#breeze-release-management-generate-constraints-line-21)">[default: 4; 1<=x<=8]                 [...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="556.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-22)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="556.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-22)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="556.8" textLength="85.4" clip-path="url(#breeze- [...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-23)">│</text><text class="breeze-release-management-generate-constraints-r5" x="280.6" y="581.2" textLength="951.6" clip-path="url(#breeze-release-management-generate-constraints-line-23)">[default: 3.8 3.9 3.10 3.11]              [...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-24)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-24)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="605.6" textLength="61" clip-path="url(#breeze-re [...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="630" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-25)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="630" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-25)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="630" textLength="73.2" clip-path="url(#breeze-releas [...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="654.4" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-26)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-26)">
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="678.8" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-27)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="678.8" textLength="195.2" clip-path="url(#breeze-release-management-generate-constraints-line-27)"> Common options </text><text class="breeze-release-management-generate-constraints-r5" x="219.6" y="678.8" textLeng [...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-28)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-28)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="703.2" textLength="97.6" clip-path="url(#breeze- [...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-29)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-29)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="727.6" textLength="48.8" clip-path="url(#breeze- [...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="752" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-30)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="752" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-30)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="752" textLength="85.4" clip-path="url(#breeze-releas [...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="776.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-31)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="776.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-31)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="776.4" textLength="61" clip-path="url(#breeze-re [...]
-</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="800.8" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-32)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="800.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-32)">
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="386" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-15)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="386" textLength="97.6" clip-path="url(#breeze-releas [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-16)">│</text><text class="breeze-release-management-generate-constraints-r1" x="414.8" y="410.4" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-16)">current_version and should be installed in CI from locally built packages  [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-17)">│</text><text class="breeze-release-management-generate-constraints-r1" x="414.8" y="434.8" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-17)">current_version.dev0                  & [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-18)">│</text><text class="breeze-release-management-generate-constraints-r6" x="414.8" y="459.2" textLength="1024.8" clip-path="url(#breeze-release-management-generate-constraints-line-18)">(TEXT)                     [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-19)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="483.6" textLength="73.2" clip-path="url(#breeze- [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-20)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="508" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-20)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="508" textLength="85.4" clip-path="url(#breeze-releas [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="532.4" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-21)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="532.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-21)">
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="556.8" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-22)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="556.8" textLength="219.6" clip-path="url(#breeze-release-management-generate-constraints-line-22)"> Parallel running </text><text class="breeze-release-management-generate-constraints-r5" x="244" y="556.8" textLeng [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-23)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-23)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="581.2" textLength="48.8" clip-path="url(#breeze- [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-24)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-24)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="605.6" textLength="146.4" clip-path="url(#breeze [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="630" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-25)">│</text><text class="breeze-release-management-generate-constraints-r5" x="280.6" y="630" textLength="915" clip-path="url(#breeze-release-management-generate-constraints-line-25)">[default: 4; 1<=x<=8]                 [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-26)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-26)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="654.4" textLength="85.4" clip-path="url(#breeze- [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="678.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-27)">│</text><text class="breeze-release-management-generate-constraints-r5" x="280.6" y="678.8" textLength="951.6" clip-path="url(#breeze-release-management-generate-constraints-line-27)">[default: 3.8 3.9 3.10 3.11]              [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-28)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-28)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="703.2" textLength="61" clip-path="url(#breeze-re [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-29)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-29)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="727.6" textLength="73.2" clip-path="url(#breeze- [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="752" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-30)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="752" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-30)">
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="776.4" textLength="24.4" clip-path="url(#breeze-release-management-generate-constraints-line-31)">╭─</text><text class="breeze-release-management-generate-constraints-r5" x="24.4" y="776.4" textLength="195.2" clip-path="url(#breeze-release-management-generate-constraints-line-31)"> Common options </text><text class="breeze-release-management-generate-constraints-r5" x="219.6" y="776.4" textLeng [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="800.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-32)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="800.8" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-32)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="800.8" textLength="97.6" clip-path="url(#breeze- [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="825.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-33)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="825.2" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-33)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="825.2" textLength="48.8" clip-path="url(#breeze- [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="849.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-34)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="849.6" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-34)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="849.6" textLength="85.4" clip-path="url(#breeze- [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="874" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-35)">│</text><text class="breeze-release-management-generate-constraints-r4" x="24.4" y="874" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-35)">-</text><text class="breeze-release-management-generate-constraints-r4" x="36.6" y="874" textLength="61" clip-path="url(#breeze-release- [...]
+</text><text class="breeze-release-management-generate-constraints-r5" x="0" y="898.4" textLength="1464" clip-path="url(#breeze-release-management-generate-constraints-line-36)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-generate-constraints-r1" x="1464" y="898.4" textLength="12.2" clip-path="url(#breeze-release-management-generate-constraints-line-36)">
</text>
</g>
</g>
diff --git a/images/breeze/output_release-management_generate-constraints.txt b/images/breeze/output_release-management_generate-constraints.txt
index c714ab26a1..4ac25d8890 100644
--- a/images/breeze/output_release-management_generate-constraints.txt
+++ b/images/breeze/output_release-management_generate-constraints.txt
@@ -1 +1 @@
-01aef235b11e59ed7f10c970a5cdaba7
+a2ba8a1912a7e9eca347cc616e717dba
diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh
index a3e09e0f1e..a321469dc1 100644
--- a/scripts/in_container/_in_container_utils.sh
+++ b/scripts/in_container/_in_container_utils.sh
@@ -233,13 +233,26 @@ function install_all_providers_from_pypi_with_eager_upgrade() {
local packages_to_install=()
local provider_package
local res
+ local chicken_egg_prefixes
+ chicken_egg_prefixes=""
+ if [[ ${CHICKEN_EGG_PROVIDERS=} != "" ]]; then
+ echo "${COLOR_BLUE}Finding providers to install from dist: ${CHICKEN_EGG_PROVIDERS}${COLOR_RESET}"
+ for chicken_egg_provider in ${CHICKEN_EGG_PROVIDERS}
+ do
+ chicken_egg_prefixes="${chicken_egg_prefixes} apache-airflow-providers-${chicken_egg_provider//./-}"
+ done
+ echo "${COLOR_BLUE}Chicken egg prefixes: ${chicken_egg_prefixes}${COLOR_RESET}"
+ ls /dist/
+ fi
for provider_package in ${ALL_PROVIDERS_PACKAGES}
do
- # Remove common.io provider in main branch until we cut-off v2-8-test branch and change
- # version in main to 2.9.0 - otherwise we won't be able to generate PyPI constraints as
- # released common-io provider has apache-airflow>2.8.0 as dependency and we cannot install
- # the provider from PyPI
- if [[ ${provider_package} == "apache-airflow-providers-common-io" ]]; then
+ if [[ "${chicken_egg_prefixes}" == *"${provider_package}"* ]]; then
+ # add the provider prepared in dist folder where chicken - egg problem is mitigated
+ for file in /dist/"${provider_package//-/_}"*.whl
+ do
+ packages_to_install+=( "${file}" )
+ echo "Added ${file} from dist folder as this is a chicken-egg package ${COLOR_GREEN}OK${COLOR_RESET}"
+ done
continue
fi
echo -n "Checking if ${provider_package} is available in PyPI: "
diff --git a/setup.cfg b/setup.cfg
index 47bcddf04c..916031e305 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -106,7 +106,7 @@ install_requires =
flask-login>=0.6.2
flask-session>=0.4.0
flask-wtf>=0.15
- fsspec>=2023.9.2
+ fsspec>=2023.10.0
google-re2>=1.0
graphviz>=0.12
gunicorn>=20.1.0
diff --git a/setup.py b/setup.py
index 2a610b185f..3819185ec4 100644
--- a/setup.py
+++ b/setup.py
@@ -515,7 +515,7 @@ aiobotocore = [
s3fs = [
# This is required for support of S3 file system which uses aiobotocore
# which can have a conflict with boto3 as mentioned above
- "s3fs>=2023.9.2",
+ "s3fs>=2023.10.0",
]
saml = [