You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2023/10/05 10:46:24 UTC
[airflow] 37/44: Update min-sqlalchemy version to account for latest features used (#34293)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-7-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 6d62f3febeaf213ffc11b1a47dcbeb7292ed711e
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Tue Sep 12 11:49:56 2023 +0200
Update min-sqlalchemy version to account for latest features used (#34293)
Some of the recent sqlalchemy changes are not working with minimum
version of sqlalchemy of ours - for example `where` syntax does
not allow moe than one clause and we are already passing more
in _do_delete_old_records (added in #33527). This syntax however
was added in SQL Alchemy 1.4.28 and our minimum version was
1.4.27.
This change bumps the minimum SQLAlchemy version to 1.4.28 but it also
adds a special test job that only runs on Postgres that downgrades
the SQLAlchemy to the minimum supported version (retrieved from
setup.cfg). This way, we will be able to detect such incompatible
changes at the PR time. This is a new flag `--downgrade-sqlalchemy`
on test command that works similar to earlier `--upgrade-boto`.
We also enable the `--upgrade-boto` and `--downgrade-sqlalchemy` flags
to be used for `breeze shell` command - thanks to that we can
easily test both flags with `breeze shell` command.
(cherry picked from commit efbead9fe7462b3634b6d9c842bd9a7ac78a0207)
---
.github/workflows/ci.yml | 49 ++++++++++++++++++++++
Dockerfile.ci | 23 ++++++----
.../airflow_breeze/commands/developer_commands.py | 8 ++++
.../commands/developer_commands_config.py | 7 ++++
.../airflow_breeze/commands/testing_commands.py | 12 +++---
.../commands/testing_commands_config.py | 1 +
.../src/airflow_breeze/params/shell_params.py | 1 +
.../src/airflow_breeze/utils/common_options.py | 12 ++++++
.../airflow_breeze/utils/docker_command_utils.py | 2 +
images/breeze/output-commands-hash.txt | 6 +--
images/breeze/output_shell.svg | 34 +++++++++++----
images/breeze/output_testing_tests.svg | 26 +++++++-----
scripts/ci/docker-compose/_docker.env | 1 +
scripts/ci/docker-compose/base.yml | 1 +
scripts/ci/docker-compose/devcontainer.env | 1 +
scripts/docker/entrypoint_ci.sh | 23 ++++++----
setup.cfg | 2 +-
17 files changed, 165 insertions(+), 44 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6ca7543136..39ed3ac08d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1021,6 +1021,55 @@ jobs:
uses: ./.github/actions/post_tests_failure
if: failure()
+ tests-postgres-min-sqlalchemy:
+ timeout-minutes: 130
+ name: >
+ MinSQLAlchemy${{needs.build-info.outputs.default-postgres-version}},
+ Py${{needs.build-info.outputs.default-python-version}}:
+ ${{needs.build-info.outputs.parallel-test-types-list-as-string}}
+ runs-on: "${{needs.build-info.outputs.runs-on}}"
+ needs: [build-info, wait-for-ci-images]
+ env:
+ RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
+ PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
+ PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
+ FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
+ DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
+ BACKEND: "postgres"
+ PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
+ PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
+ POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
+ BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
+ DOWNGRADE_SQLALCHEMY: "true"
+ JOB_ID: >
+ postgres-min-sqlalchemy-${{needs.build-info.outputs.default-python-version}}-
+ ${{needs.build-info.outputs.default-postgres-version}}
+ COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
+ if: needs.build-info.outputs.run-tests == 'true'
+ steps:
+ - name: Cleanup repo
+ shell: bash
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v3
+ with:
+ persist-credentials: false
+ - name: >
+ Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
+ uses: ./.github/actions/prepare_breeze_and_image
+ - name: >
+ Tests: ${{needs.build-info.outputs.default-python-version}}:
+ ${{needs.build-info.outputs.parallel-test-types-list-as-string}}
+ run: breeze testing tests --run-in-parallel
+ - name: >
+ Post Tests success: ${{needs.build-info.outputs.default-python-version}}:Boto"
+ uses: ./.github/actions/post_tests_success
+ if: success()
+ - name: >
+ Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:Boto"
+ uses: ./.github/actions/post_tests_failure
+ if: failure()
+
tests-postgres-in-progress-features-disabled:
timeout-minutes: 130
name: >
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 9092465c82..a0f47d86d3 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -937,6 +937,22 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
fi
rm -f "${AIRFLOW_SOURCES}/pytest.ini"
+if [[ ${UPGRADE_BOTO=} == "true" ]]; then
+ echo
+ echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
+ echo
+ pip uninstall --root-user-action ignore aiobotocore -y || true
+ pip install --root-user-action ignore --upgrade boto3 botocore
+ pip check
+fi
+if [[ ${DOWNGRADE_SQLALCHEMY=} == "true" ]]; then
+ min_sqlalchemy_version=$(grep "sqlalchemy>=" setup.cfg | sed "s/.*>=\([0-9\.]*\).*/\1/")
+ echo
+ echo "${COLOR_BLUE}Downgrading sqlalchemy to minimum supported version: ${min_sqlalchemy_version}${COLOR_RESET}"
+ echo
+ pip install --root-user-action ignore "sqlalchemy==${min_sqlalchemy_version}"
+ pip check
+fi
set +u
if [[ "${RUN_TESTS}" != "true" ]]; then
@@ -1166,13 +1182,6 @@ else
exit 1
fi
fi
-if [[ ${UPGRADE_BOTO=} == "true" ]]; then
- echo
- echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
- echo
- pip uninstall aiobotocore -y || true
- pip install --upgrade boto3 botocore
-fi
readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \
ALL_TESTS ALL_PRESELECTED_TESTS
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index 92b3f7baa0..5c569ba811 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -51,6 +51,7 @@ from airflow_breeze.utils.common_options import (
option_celery_broker,
option_celery_flower,
option_db_reset,
+ option_downgrade_sqlalchemy,
option_dry_run,
option_executor,
option_force_build,
@@ -70,6 +71,7 @@ from airflow_breeze.utils.common_options import (
option_platform_single,
option_postgres_version,
option_python,
+ option_upgrade_boto,
option_use_airflow_version,
option_use_packages_from_dist,
option_verbose,
@@ -162,6 +164,8 @@ class TimerThread(threading.Thread):
@option_image_tag_for_running
@option_max_time
@option_include_mypy_volume
+@option_upgrade_boto
+@option_downgrade_sqlalchemy
@option_verbose
@option_dry_run
@option_github_repository
@@ -197,6 +201,8 @@ def shell(
celery_broker: str,
celery_flower: bool,
extra_args: tuple,
+ upgrade_boto: bool,
+ downgrade_sqlalchemy: bool,
):
"""Enter breeze environment. this is the default command use when no other is selected."""
if get_verbose() or get_dry_run():
@@ -234,6 +240,8 @@ def shell(
executor=executor,
celery_broker=celery_broker,
celery_flower=celery_flower,
+ upgrade_boto=upgrade_boto,
+ downgrade_sqlalchemy=downgrade_sqlalchemy,
)
sys.exit(result.returncode)
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
index 81c99e793f..9ef4f8ca96 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
@@ -99,6 +99,13 @@ DEVELOPER_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = {
"--package-format",
],
},
+ {
+ "name": "Upgrading/downgrading selected packages",
+ "options": [
+ "--upgrade-boto",
+ "--downgrade-sqlalchemy",
+ ],
+ },
],
"breeze compile-www-assets": [
{
diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
index d892727ee4..7acd6cffd6 100644
--- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
@@ -38,6 +38,7 @@ from airflow_breeze.utils.common_options import (
option_backend,
option_db_reset,
option_debug_resources,
+ option_downgrade_sqlalchemy,
option_dry_run,
option_github_repository,
option_image_name,
@@ -52,6 +53,7 @@ from airflow_breeze.utils.common_options import (
option_python,
option_run_in_parallel,
option_skip_cleanup,
+ option_upgrade_boto,
option_use_airflow_version,
option_verbose,
)
@@ -367,12 +369,8 @@ def run_tests_in_parallel(
show_default=True,
envvar="PARALLEL_TEST_TYPES",
)
-@click.option(
- "--upgrade-boto",
- help="Remove aiobotocore and upgrade botocore and boto to the latest version.",
- is_flag=True,
- envvar="UPGRADE_BOTO",
-)
+@option_upgrade_boto
+@option_downgrade_sqlalchemy
@click.option(
"--collect-only",
help="Collect tests only, do not run them.",
@@ -416,6 +414,7 @@ def command_for_tests(
mount_sources: str,
extra_pytest_args: tuple,
upgrade_boto: bool,
+ downgrade_sqlalchemy: bool,
collect_only: bool,
remove_arm_packages: bool,
github_repository: str,
@@ -436,6 +435,7 @@ def command_for_tests(
forward_ports=False,
test_type=test_type,
upgrade_boto=upgrade_boto,
+ downgrade_sqlalchemy=downgrade_sqlalchemy,
collect_only=collect_only,
remove_arm_packages=remove_arm_packages,
github_repository=github_repository,
diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py
index 908843c224..84d0d2b6ec 100644
--- a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py
+++ b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py
@@ -56,6 +56,7 @@ TESTING_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = {
"--use-airflow-version",
"--mount-sources",
"--upgrade-boto",
+ "--downgrade-sqlalchemy",
"--remove-arm-packages",
"--skip-docker-compose-down",
],
diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py
index 99bc8c4d3c..b2240a42c9 100644
--- a/dev/breeze/src/airflow_breeze/params/shell_params.py
+++ b/dev/breeze/src/airflow_breeze/params/shell_params.py
@@ -120,6 +120,7 @@ class ShellParams:
dry_run: bool = False
verbose: bool = False
upgrade_boto: bool = False
+ downgrade_sqlalchemy: bool = False
executor: str = START_AIRFLOW_DEFAULT_ALLOWED_EXECUTORS
celery_broker: str = DEFAULT_CELERY_BROKER
celery_flower: bool = False
diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py
index f9a82064b8..5e8e50da61 100644
--- a/dev/breeze/src/airflow_breeze/utils/common_options.py
+++ b/dev/breeze/src/airflow_breeze/utils/common_options.py
@@ -601,3 +601,15 @@ option_eager_upgrade_additional_requirements = click.option(
help="Optional additional requirements to upgrade eagerly to avoid backtracking "
"(see `breeze ci find-backtracking-candidates`).",
)
+option_upgrade_boto = click.option(
+ "--upgrade-boto",
+ help="Remove aiobotocore and upgrade botocore and boto to the latest version.",
+ is_flag=True,
+ envvar="UPGRADE_BOTO",
+)
+option_downgrade_sqlalchemy = click.option(
+ "--downgrade-sqlalchemy",
+ help="Downgrade SQLAlchemy to minimum supported version.",
+ is_flag=True,
+ envvar="DOWNGRADE_SQLALCHEMY",
+)
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index df6047c322..b0a5697b18 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -600,6 +600,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
set_value_to_default_if_not_set(env, "TEST_TYPE", "")
set_value_to_default_if_not_set(env, "TEST_TIMEOUT", "60")
set_value_to_default_if_not_set(env, "UPGRADE_BOTO", "false")
+ set_value_to_default_if_not_set(env, "DOWNGRADE_SQLALCHEMY", "false")
set_value_to_default_if_not_set(env, "UPGRADE_TO_NEWER_DEPENDENCIES", "false")
set_value_to_default_if_not_set(env, "USE_PACKAGES_FROM_DIST", "false")
set_value_to_default_if_not_set(env, "VERBOSE", "false")
@@ -647,6 +648,7 @@ DERIVE_ENV_VARIABLES_FROM_ATTRIBUTES = {
"SQLITE_URL": "sqlite_url",
"START_AIRFLOW": "start_airflow",
"UPGRADE_BOTO": "upgrade_boto",
+ "DOWNGRADE_SQLALCHEMY": "downgrade_sqlalchemy",
"USE_AIRFLOW_VERSION": "use_airflow_version",
"USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
"VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt
index b15e25524a..09b6278bf9 100644
--- a/images/breeze/output-commands-hash.txt
+++ b/images/breeze/output-commands-hash.txt
@@ -62,11 +62,11 @@ setup:regenerate-command-images:d5e29ec6acb1a6af7d83772c2962f89d
setup:self-upgrade:4af905a147fcd6670a0e33d3d369a94b
setup:version:be116d90a21c2afe01087f7609774e1e
setup:8de3ed2645928e8f9f89c58f0ccd2a60
-shell:30a9271ff59a0fe756dda402cc165683
+shell:1e901a677a6df6ba1d64d3fe79b42587
start-airflow:ff0f63e20b9ff454e5d3c7b9ba9080d7
static-checks:d319b1c7972a6623bc8ee1b174cacb48
testing:docker-compose-tests:0c810047fc66a0cfe91119e2d08b3507
testing:helm-tests:8e491da2e01ebd815322c37562059d77
testing:integration-tests:486e4d91449ecdb7630ef2a470d705a3
-testing:tests:3c202e65824e405269e78f58936980e0
-testing:68a089bc30e0e60f834f205df1e9f086
+testing:tests:b3b921fd5a7d3435a0ad34e90b75cb2f
+testing:13325e047fc32d9e40b51bfd15212a91
diff --git a/images/breeze/output_shell.svg b/images/breeze/output_shell.svg
index a8dd2ecd2e..7c8b47ed11 100644
--- a/images/breeze/output_shell.svg
+++ b/images/breeze/output_shell.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 1611.6" xmlns="http://www.w3.org/2000/svg">
+<svg class="rich-terminal" viewBox="0 0 1482 1709.1999999999998" xmlns="http://www.w3.org/2000/svg">
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -43,7 +43,7 @@
<defs>
<clipPath id="breeze-shell-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="1560.6" />
+ <rect x="0" y="0" width="1463.0" height="1658.1999999999998" />
</clipPath>
<clipPath id="breeze-shell-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -234,9 +234,21 @@
<clipPath id="breeze-shell-line-62">
<rect x="0" y="1514.3" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-shell-line-63">
+ <rect x="0" y="1538.7" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-shell-line-64">
+ <rect x="0" y="1563.1" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-shell-line-65">
+ <rect x="0" y="1587.5" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-shell-line-66">
+ <rect x="0" y="1611.9" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="1609.6" rx="8"/><text class="breeze-shell-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: shell</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="1707.2" rx="8"/><text class="breeze-shell-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: shell</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -304,12 +316,16 @@
</text><text class="breeze-shell-r5" x="0" y="1362" textLength="12.2" clip-path="url(#breeze-shell-line-55)">│</text><text class="breeze-shell-r4" x="24.4" y="1362" textLength="12.2" clip-path="url(#breeze-shell-line-55)">-</text><text class="breeze-shell-r4" x="36.6" y="1362" textLength="97.6" clip-path="url(#breeze-shell-line-55)">-package</text><text class="breeze-shell-r4" x="134.2" y="1362" textLength="85.4" clip-path="url(#breeze-shell-line-55)">-format</text><text class="breeze-sh [...]
</text><text class="breeze-shell-r5" x="0" y="1386.4" textLength="12.2" clip-path="url(#breeze-shell-line-56)">│</text><text class="breeze-shell-r5" x="451.4" y="1386.4" textLength="658.8" clip-path="url(#breeze-shell-line-56)">[default: wheel]                                      </text><text cl [...]
</text><text class="breeze-shell-r5" x="0" y="1410.8" textLength="1464" clip-path="url(#breeze-shell-line-57)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-shell-r2" x="1464" y="1410.8" textLength="12.2" clip-path="url(#breeze-shell-line-57)">
-</text><text class="breeze-shell-r5" x="0" y="1435.2" textLength="24.4" clip-path="url(#breeze-shell-line-58)">╭─</text><text class="breeze-shell-r5" x="24.4" y="1435.2" textLength="195.2" clip-path="url(#breeze-shell-line-58)"> Common options </text><text class="breeze-shell-r5" x="219.6" y="1435.2" textLength="1220" clip-path="url(#breeze-shell-line-58)">────────────────────────────────────────────────────────────────────────────────────────────────────</text><text class [...]
-</text><text class="breeze-shell-r5" x="0" y="1459.6" textLength="12.2" clip-path="url(#breeze-shell-line-59)">│</text><text class="breeze-shell-r4" x="24.4" y="1459.6" textLength="12.2" clip-path="url(#breeze-shell-line-59)">-</text><text class="breeze-shell-r4" x="36.6" y="1459.6" textLength="97.6" clip-path="url(#breeze-shell-line-59)">-verbose</text><text class="breeze-shell-r6" x="158.6" y="1459.6" textLength="24.4" clip-path="url(#breeze-shell-line-59)">-v</text><text class="breeze [...]
-</text><text class="breeze-shell-r5" x="0" y="1484" textLength="12.2" clip-path="url(#breeze-shell-line-60)">│</text><text class="breeze-shell-r4" x="24.4" y="1484" textLength="12.2" clip-path="url(#breeze-shell-line-60)">-</text><text class="breeze-shell-r4" x="36.6" y="1484" textLength="48.8" clip-path="url(#breeze-shell-line-60)">-dry</text><text class="breeze-shell-r4" x="85.4" y="1484" textLength="48.8" clip-path="url(#breeze-shell-line-60)">-run</text><text class="breeze-shell-r6" [...]
-</text><text class="breeze-shell-r5" x="0" y="1508.4" textLength="12.2" clip-path="url(#breeze-shell-line-61)">│</text><text class="breeze-shell-r4" x="24.4" y="1508.4" textLength="12.2" clip-path="url(#breeze-shell-line-61)">-</text><text class="breeze-shell-r4" x="36.6" y="1508.4" textLength="85.4" clip-path="url(#breeze-shell-line-61)">-answer</text><text class="breeze-shell-r6" x="158.6" y="1508.4" textLength="24.4" clip-path="url(#breeze-shell-line-61)">-a</text><text class="breeze- [...]
-</text><text class="breeze-shell-r5" x="0" y="1532.8" textLength="12.2" clip-path="url(#breeze-shell-line-62)">│</text><text class="breeze-shell-r4" x="24.4" y="1532.8" textLength="12.2" clip-path="url(#breeze-shell-line-62)">-</text><text class="breeze-shell-r4" x="36.6" y="1532.8" textLength="61" clip-path="url(#breeze-shell-line-62)">-help</text><text class="breeze-shell-r6" x="158.6" y="1532.8" textLength="24.4" clip-path="url(#breeze-shell-line-62)">-h</text><text class="breeze-shel [...]
-</text><text class="breeze-shell-r5" x="0" y="1557.2" textLength="1464" clip-path="url(#breeze-shell-line-63)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-shell-r2" x="1464" y="1557.2" textLength="12.2" clip-path="url(#breeze-shell-line-63)">
+</text><text class="breeze-shell-r5" x="0" y="1435.2" textLength="24.4" clip-path="url(#breeze-shell-line-58)">╭─</text><text class="breeze-shell-r5" x="24.4" y="1435.2" textLength="500.2" clip-path="url(#breeze-shell-line-58)"> Upgrading/downgrading selected packages </text><text class="breeze-shell-r5" x="524.6" y="1435.2" textLength="915" clip-path="url(#breeze-shell-line-58)">───────────────────────────────────────────────────────────────────────────</text><text c [...]
+</text><text class="breeze-shell-r5" x="0" y="1459.6" textLength="12.2" clip-path="url(#breeze-shell-line-59)">│</text><text class="breeze-shell-r4" x="24.4" y="1459.6" textLength="12.2" clip-path="url(#breeze-shell-line-59)">-</text><text class="breeze-shell-r4" x="36.6" y="1459.6" textLength="97.6" clip-path="url(#breeze-shell-line-59)">-upgrade</text><text class="breeze-shell-r4" x="134.2" y="1459.6" textLength="61" clip-path="url(#breeze-shell-line-59)">-boto</text><text class="breez [...]
+</text><text class="breeze-shell-r5" x="0" y="1484" textLength="12.2" clip-path="url(#breeze-shell-line-60)">│</text><text class="breeze-shell-r4" x="24.4" y="1484" textLength="12.2" clip-path="url(#breeze-shell-line-60)">-</text><text class="breeze-shell-r4" x="36.6" y="1484" textLength="122" clip-path="url(#breeze-shell-line-60)">-downgrade</text><text class="breeze-shell-r4" x="158.6" y="1484" textLength="134.2" clip-path="url(#breeze-shell-line-60)">-sqlalchemy</text><text class="bre [...]
+</text><text class="breeze-shell-r5" x="0" y="1508.4" textLength="1464" clip-path="url(#breeze-shell-line-61)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-shell-r2" x="1464" y="1508.4" textLength="12.2" clip-path="url(#breeze-shell-line-61)">
+</text><text class="breeze-shell-r5" x="0" y="1532.8" textLength="24.4" clip-path="url(#breeze-shell-line-62)">╭─</text><text class="breeze-shell-r5" x="24.4" y="1532.8" textLength="195.2" clip-path="url(#breeze-shell-line-62)"> Common options </text><text class="breeze-shell-r5" x="219.6" y="1532.8" textLength="1220" clip-path="url(#breeze-shell-line-62)">────────────────────────────────────────────────────────────────────────────────────────────────────</text><text class [...]
+</text><text class="breeze-shell-r5" x="0" y="1557.2" textLength="12.2" clip-path="url(#breeze-shell-line-63)">│</text><text class="breeze-shell-r4" x="24.4" y="1557.2" textLength="12.2" clip-path="url(#breeze-shell-line-63)">-</text><text class="breeze-shell-r4" x="36.6" y="1557.2" textLength="97.6" clip-path="url(#breeze-shell-line-63)">-verbose</text><text class="breeze-shell-r6" x="158.6" y="1557.2" textLength="24.4" clip-path="url(#breeze-shell-line-63)">-v</text><text class="breeze [...]
+</text><text class="breeze-shell-r5" x="0" y="1581.6" textLength="12.2" clip-path="url(#breeze-shell-line-64)">│</text><text class="breeze-shell-r4" x="24.4" y="1581.6" textLength="12.2" clip-path="url(#breeze-shell-line-64)">-</text><text class="breeze-shell-r4" x="36.6" y="1581.6" textLength="48.8" clip-path="url(#breeze-shell-line-64)">-dry</text><text class="breeze-shell-r4" x="85.4" y="1581.6" textLength="48.8" clip-path="url(#breeze-shell-line-64)">-run</text><text class="breeze-sh [...]
+</text><text class="breeze-shell-r5" x="0" y="1606" textLength="12.2" clip-path="url(#breeze-shell-line-65)">│</text><text class="breeze-shell-r4" x="24.4" y="1606" textLength="12.2" clip-path="url(#breeze-shell-line-65)">-</text><text class="breeze-shell-r4" x="36.6" y="1606" textLength="85.4" clip-path="url(#breeze-shell-line-65)">-answer</text><text class="breeze-shell-r6" x="158.6" y="1606" textLength="24.4" clip-path="url(#breeze-shell-line-65)">-a</text><text class="breeze-shell-r2 [...]
+</text><text class="breeze-shell-r5" x="0" y="1630.4" textLength="12.2" clip-path="url(#breeze-shell-line-66)">│</text><text class="breeze-shell-r4" x="24.4" y="1630.4" textLength="12.2" clip-path="url(#breeze-shell-line-66)">-</text><text class="breeze-shell-r4" x="36.6" y="1630.4" textLength="61" clip-path="url(#breeze-shell-line-66)">-help</text><text class="breeze-shell-r6" x="158.6" y="1630.4" textLength="24.4" clip-path="url(#breeze-shell-line-66)">-h</text><text class="breeze-shel [...]
+</text><text class="breeze-shell-r5" x="0" y="1654.8" textLength="1464" clip-path="url(#breeze-shell-line-67)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-shell-r2" x="1464" y="1654.8" textLength="12.2" clip-path="url(#breeze-shell-line-67)">
</text>
</g>
</g>
diff --git a/images/breeze/output_testing_tests.svg b/images/breeze/output_testing_tests.svg
index ec3da499fa..9cf14a8662 100644
--- a/images/breeze/output_testing_tests.svg
+++ b/images/breeze/output_testing_tests.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 1489.6" xmlns="http://www.w3.org/2000/svg">
+<svg class="rich-terminal" viewBox="0 0 1482 1514.0" xmlns="http://www.w3.org/2000/svg">
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -43,7 +43,7 @@
<defs>
<clipPath id="breeze-testing-tests-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="1438.6" />
+ <rect x="0" y="0" width="1463.0" height="1463.0" />
</clipPath>
<clipPath id="breeze-testing-tests-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -219,9 +219,12 @@
<clipPath id="breeze-testing-tests-line-57">
<rect x="0" y="1392.3" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-testing-tests-line-58">
+ <rect x="0" y="1416.7" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="1487.6" rx="8"/><text class="breeze-testing-tests-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: testing tests</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="1512" rx="8"/><text class="breeze-testing-tests-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: testing tests</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -282,14 +285,15 @@
</text><text class="breeze-testing-tests-r5" x="0" y="1191.2" textLength="12.2" clip-path="url(#breeze-testing-tests-line-48)">│</text><text class="breeze-testing-tests-r7" x="414.8" y="1191.2" textLength="1024.8" clip-path="url(#breeze-testing-tests-line-48)">(selected | all | skip | remove)                            & [...]
</text><text class="breeze-testing-tests-r5" x="0" y="1215.6" textLength="12.2" clip-path="url(#breeze-testing-tests-line-49)">│</text><text class="breeze-testing-tests-r5" x="414.8" y="1215.6" textLength="1024.8" clip-path="url(#breeze-testing-tests-line-49)">[default: selected]                                   [...]
</text><text class="breeze-testing-tests-r5" x="0" y="1240" textLength="12.2" clip-path="url(#breeze-testing-tests-line-50)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1240" textLength="12.2" clip-path="url(#breeze-testing-tests-line-50)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1240" textLength="97.6" clip-path="url(#breeze-testing-tests-line-50)">-upgrade</text><text class="breeze-testing-tests-r4" x="134.2" y="1240" textLength="61" clip-path="url(#breez [...]
-</text><text class="breeze-testing-tests-r5" x="0" y="1264.4" textLength="12.2" clip-path="url(#breeze-testing-tests-line-51)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1264.4" textLength="12.2" clip-path="url(#breeze-testing-tests-line-51)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1264.4" textLength="85.4" clip-path="url(#breeze-testing-tests-line-51)">-remove</text><text class="breeze-testing-tests-r4" x="122" y="1264.4" textLength="158.6" clip-path="ur [...]
-</text><text class="breeze-testing-tests-r5" x="0" y="1288.8" textLength="12.2" clip-path="url(#breeze-testing-tests-line-52)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1288.8" textLength="12.2" clip-path="url(#breeze-testing-tests-line-52)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1288.8" textLength="61" clip-path="url(#breeze-testing-tests-line-52)">-skip</text><text class="breeze-testing-tests-r4" x="97.6" y="1288.8" textLength="244" clip-path="url(#br [...]
-</text><text class="breeze-testing-tests-r5" x="0" y="1313.2" textLength="1464" clip-path="url(#breeze-testing-tests-line-53)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-testing-tests-r2" x="1464" y="1313.2" textLength="12.2" clip-path="url(#breeze-testing-tests-line-53)">
-</text><text class="breeze-testing-tests-r5" x="0" y="1337.6" textLength="24.4" clip-path="url(#breeze-testing-tests-line-54)">╭─</text><text class="breeze-testing-tests-r5" x="24.4" y="1337.6" textLength="195.2" clip-path="url(#breeze-testing-tests-line-54)"> Common options </text><text class="breeze-testing-tests-r5" x="219.6" y="1337.6" textLength="1220" clip-path="url(#breeze-testing-tests-line-54)">────────────────────────────────────────────────────────────────────── [...]
-</text><text class="breeze-testing-tests-r5" x="0" y="1362" textLength="12.2" clip-path="url(#breeze-testing-tests-line-55)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1362" textLength="12.2" clip-path="url(#breeze-testing-tests-line-55)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1362" textLength="97.6" clip-path="url(#breeze-testing-tests-line-55)">-verbose</text><text class="breeze-testing-tests-r6" x="158.6" y="1362" textLength="24.4" clip-path="url(#bre [...]
-</text><text class="breeze-testing-tests-r5" x="0" y="1386.4" textLength="12.2" clip-path="url(#breeze-testing-tests-line-56)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1386.4" textLength="12.2" clip-path="url(#breeze-testing-tests-line-56)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1386.4" textLength="48.8" clip-path="url(#breeze-testing-tests-line-56)">-dry</text><text class="breeze-testing-tests-r4" x="85.4" y="1386.4" textLength="48.8" clip-path="url(# [...]
-</text><text class="breeze-testing-tests-r5" x="0" y="1410.8" textLength="12.2" clip-path="url(#breeze-testing-tests-line-57)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1410.8" textLength="12.2" clip-path="url(#breeze-testing-tests-line-57)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1410.8" textLength="61" clip-path="url(#breeze-testing-tests-line-57)">-help</text><text class="breeze-testing-tests-r6" x="158.6" y="1410.8" textLength="24.4" clip-path="url(# [...]
-</text><text class="breeze-testing-tests-r5" x="0" y="1435.2" textLength="1464" clip-path="url(#breeze-testing-tests-line-58)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-testing-tests-r2" x="1464" y="1435.2" textLength="12.2" clip-path="url(#breeze-testing-tests-line-58)">
+</text><text class="breeze-testing-tests-r5" x="0" y="1264.4" textLength="12.2" clip-path="url(#breeze-testing-tests-line-51)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1264.4" textLength="12.2" clip-path="url(#breeze-testing-tests-line-51)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1264.4" textLength="122" clip-path="url(#breeze-testing-tests-line-51)">-downgrade</text><text class="breeze-testing-tests-r4" x="158.6" y="1264.4" textLength="134.2" clip-path [...]
+</text><text class="breeze-testing-tests-r5" x="0" y="1288.8" textLength="12.2" clip-path="url(#breeze-testing-tests-line-52)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1288.8" textLength="12.2" clip-path="url(#breeze-testing-tests-line-52)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1288.8" textLength="85.4" clip-path="url(#breeze-testing-tests-line-52)">-remove</text><text class="breeze-testing-tests-r4" x="122" y="1288.8" textLength="158.6" clip-path="ur [...]
+</text><text class="breeze-testing-tests-r5" x="0" y="1313.2" textLength="12.2" clip-path="url(#breeze-testing-tests-line-53)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1313.2" textLength="12.2" clip-path="url(#breeze-testing-tests-line-53)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1313.2" textLength="61" clip-path="url(#breeze-testing-tests-line-53)">-skip</text><text class="breeze-testing-tests-r4" x="97.6" y="1313.2" textLength="244" clip-path="url(#br [...]
+</text><text class="breeze-testing-tests-r5" x="0" y="1337.6" textLength="1464" clip-path="url(#breeze-testing-tests-line-54)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-testing-tests-r2" x="1464" y="1337.6" textLength="12.2" clip-path="url(#breeze-testing-tests-line-54)">
+</text><text class="breeze-testing-tests-r5" x="0" y="1362" textLength="24.4" clip-path="url(#breeze-testing-tests-line-55)">╭─</text><text class="breeze-testing-tests-r5" x="24.4" y="1362" textLength="195.2" clip-path="url(#breeze-testing-tests-line-55)"> Common options </text><text class="breeze-testing-tests-r5" x="219.6" y="1362" textLength="1220" clip-path="url(#breeze-testing-tests-line-55)">──────────────────────────────────────────────────────────────────────────── [...]
+</text><text class="breeze-testing-tests-r5" x="0" y="1386.4" textLength="12.2" clip-path="url(#breeze-testing-tests-line-56)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1386.4" textLength="12.2" clip-path="url(#breeze-testing-tests-line-56)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1386.4" textLength="97.6" clip-path="url(#breeze-testing-tests-line-56)">-verbose</text><text class="breeze-testing-tests-r6" x="158.6" y="1386.4" textLength="24.4" clip-path=" [...]
+</text><text class="breeze-testing-tests-r5" x="0" y="1410.8" textLength="12.2" clip-path="url(#breeze-testing-tests-line-57)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1410.8" textLength="12.2" clip-path="url(#breeze-testing-tests-line-57)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1410.8" textLength="48.8" clip-path="url(#breeze-testing-tests-line-57)">-dry</text><text class="breeze-testing-tests-r4" x="85.4" y="1410.8" textLength="48.8" clip-path="url(# [...]
+</text><text class="breeze-testing-tests-r5" x="0" y="1435.2" textLength="12.2" clip-path="url(#breeze-testing-tests-line-58)">│</text><text class="breeze-testing-tests-r4" x="24.4" y="1435.2" textLength="12.2" clip-path="url(#breeze-testing-tests-line-58)">-</text><text class="breeze-testing-tests-r4" x="36.6" y="1435.2" textLength="61" clip-path="url(#breeze-testing-tests-line-58)">-help</text><text class="breeze-testing-tests-r6" x="158.6" y="1435.2" textLength="24.4" clip-path="url(# [...]
+</text><text class="breeze-testing-tests-r5" x="0" y="1459.6" textLength="1464" clip-path="url(#breeze-testing-tests-line-59)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-testing-tests-r2" x="1464" y="1459.6" textLength="12.2" clip-path="url(#breeze-testing-tests-line-59)">
</text>
</g>
</g>
diff --git a/scripts/ci/docker-compose/_docker.env b/scripts/ci/docker-compose/_docker.env
index 2cd4bc0f7d..aa21804e9b 100644
--- a/scripts/ci/docker-compose/_docker.env
+++ b/scripts/ci/docker-compose/_docker.env
@@ -75,6 +75,7 @@ TEST_TIMEOUT
TEST_TYPE
UPGRADE_BOTO
UPGRADE_TO_NEWER_DEPENDENCIES
+DOWNGRADE_SQLALCHEMY
VERBOSE
VERBOSE_COMMANDS
VERSION_SUFFIX_FOR_PYPI
diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml
index 160e8feec0..3b7417eab5 100644
--- a/scripts/ci/docker-compose/base.yml
+++ b/scripts/ci/docker-compose/base.yml
@@ -85,6 +85,7 @@ services:
- TEST_TYPE=${TEST_TYPE}
- TEST_TIMEOUT=${TEST_TIMEOUT}
- UPGRADE_BOTO=${UPGRADE_BOTO}
+ - DOWNGRADE_SQLALCHEMY=${DOWNGRADE_SQLALCHEMY}
- UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES}
- VERBOSE=${VERBOSE}
- VERBOSE_COMMANDS=${VERBOSE_COMMANDS}
diff --git a/scripts/ci/docker-compose/devcontainer.env b/scripts/ci/docker-compose/devcontainer.env
index d66767fa02..f71a0e1e39 100644
--- a/scripts/ci/docker-compose/devcontainer.env
+++ b/scripts/ci/docker-compose/devcontainer.env
@@ -69,6 +69,7 @@ START_AIRFLOW="false"
SUSPENDED_PROVIDERS_FOLDERS=""
TEST_TYPE=
UPGRADE_BOTO="false"
+DOWNGRADE_SQLALCHEMY="false"
UPGRADE_TO_NEWER_DEPENDENCIES="false"
VERBOSE="false"
VERBOSE_COMMANDS="false"
diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh
index 73b8a20deb..87e573d114 100755
--- a/scripts/docker/entrypoint_ci.sh
+++ b/scripts/docker/entrypoint_ci.sh
@@ -326,6 +326,22 @@ fi
# Remove pytest.ini from the current directory if it exists. It has been removed from the source tree
# but may still be present in the local directory if the user has old breeze image
rm -f "${AIRFLOW_SOURCES}/pytest.ini"
+if [[ ${UPGRADE_BOTO=} == "true" ]]; then
+ echo
+ echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
+ echo
+ pip uninstall --root-user-action ignore aiobotocore -y || true
+ pip install --root-user-action ignore --upgrade boto3 botocore
+ pip check
+fi
+if [[ ${DOWNGRADE_SQLALCHEMY=} == "true" ]]; then
+ min_sqlalchemy_version=$(grep "sqlalchemy>=" setup.cfg | sed "s/.*>=\([0-9\.]*\).*/\1/")
+ echo
+ echo "${COLOR_BLUE}Downgrading sqlalchemy to minimum supported version: ${min_sqlalchemy_version}${COLOR_RESET}"
+ echo
+ pip install --root-user-action ignore "sqlalchemy==${min_sqlalchemy_version}"
+ pip check
+fi
set +u
# If we do not want to run tests, we simply drop into bash
@@ -558,13 +574,6 @@ else
exit 1
fi
fi
-if [[ ${UPGRADE_BOTO=} == "true" ]]; then
- echo
- echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
- echo
- pip uninstall aiobotocore -y || true
- pip install --upgrade boto3 botocore
-fi
readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \
ALL_TESTS ALL_PRESELECTED_TESTS
diff --git a/setup.cfg b/setup.cfg
index 367924d82d..26c8cb2e03 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -142,7 +142,7 @@ install_requires =
# See https://sqlalche.me/e/b8d9 for details of deprecated features
# you can set environment variable SQLALCHEMY_WARN_20=1 to show all deprecation warnings.
# The issue tracking it is https://github.com/apache/airflow/issues/28723
- sqlalchemy>=1.4.24,<2.0
+ sqlalchemy>=1.4.28,<2.0
sqlalchemy_jsonfield>=1.0
tabulate>=0.7.5
tenacity>=6.2.0,!=8.2.0