You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/09/08 19:06:57 UTC
[airflow] branch v2-4-test updated (f424e68cef -> e60d2b5b88)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch v2-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
from f424e68cef Change default branches for 2-4
new 7f6e0d7728 Fix case when SHELL variable is not set in kubernetes tests (#26235)
new 8c71dd8b13 Make breeze works with latest docker-compose (#26233)
new e60d2b5b88 Limit eager upgrade of protobuf library to < 4.21.0 (#26243)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
Dockerfile | 6 +++++-
Dockerfile.ci | 6 +++++-
dev/breeze/src/airflow_breeze/commands/developer_commands.py | 8 +++++---
dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py | 2 +-
dev/breeze/src/airflow_breeze/commands/testing_commands.py | 5 +++--
dev/breeze/src/airflow_breeze/utils/docker_command_utils.py | 6 +++++-
6 files changed, 24 insertions(+), 9 deletions(-)
[airflow] 02/03: Make breeze works with latest docker-compose (#26233)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 8c71dd8b13ccba199a77b205425c08087a7f177f
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Thu Sep 8 17:19:39 2022 +0200
Make breeze works with latest docker-compose (#26233)
The latest docker-compose dropped an alias for `docker-compose` and
we need to detect it and use "docker compose" instead.
(cherry picked from commit 7f47006effb330429fb510eb52644d22544f4fad)
---
dev/breeze/src/airflow_breeze/commands/developer_commands.py | 8 +++++---
dev/breeze/src/airflow_breeze/commands/testing_commands.py | 5 +++--
dev/breeze/src/airflow_breeze/utils/docker_command_utils.py | 6 +++++-
3 files changed, 13 insertions(+), 6 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index 3f3c894df8..3a3c95e7bc 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -69,6 +69,7 @@ from airflow_breeze.utils.confirm import set_forced_answer
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.custom_param_types import BetterChoice, NotVerifiedBetterChoice
from airflow_breeze.utils.docker_command_utils import (
+ DOCKER_COMPOSE_COMMAND,
check_docker_resources,
get_env_variables_for_docker_commands,
get_extra_docker_flags,
@@ -468,7 +469,8 @@ def compile_www_assets(
is_flag=True,
)
def stop(verbose: bool, dry_run: bool, preserve_volumes: bool):
- command_to_execute = ['docker-compose', 'down', "--remove-orphans"]
+ perform_environment_checks(verbose=verbose)
+ command_to_execute = [*DOCKER_COMPOSE_COMMAND, 'down', "--remove-orphans"]
if not preserve_volumes:
command_to_execute.append("--volumes")
shell_params = ShellParams(verbose=verbose, backend="all", include_mypy_volume=True)
@@ -544,7 +546,7 @@ def run_shell(verbose: bool, dry_run: bool, shell_params: ShellParams) -> RunCom
:param shell_params: parameters of the execution
"""
shell_params.print_badge_info()
- cmd = ['docker-compose', 'run', '--service-ports', "-e", "BREEZE", '--rm', 'airflow']
+ cmd = [*DOCKER_COMPOSE_COMMAND, 'run', '--service-ports', "-e", "BREEZE", '--rm', 'airflow']
cmd_added = shell_params.command_passed
env_variables = get_env_variables_for_docker_commands(shell_params)
if cmd_added is not None:
@@ -578,7 +580,7 @@ def find_airflow_container(verbose, dry_run) -> Optional[str]:
check_docker_resources(exec_shell_params.airflow_image_name, verbose=verbose, dry_run=dry_run)
exec_shell_params.print_badge_info()
env_variables = get_env_variables_for_docker_commands(exec_shell_params)
- cmd = ['docker-compose', 'ps', '--all', '--filter', 'status=running', 'airflow']
+ cmd = [*DOCKER_COMPOSE_COMMAND, 'ps', '--all', '--filter', 'status=running', 'airflow']
docker_compose_ps_command = run_command(
cmd, verbose=verbose, dry_run=dry_run, text=True, capture_output=True, env=env_variables, check=False
)
diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
index a727585df1..7b73ad2e8d 100644
--- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
@@ -50,6 +50,7 @@ from airflow_breeze.utils.common_options import (
from airflow_breeze.utils.console import get_console, message_type_from_return_code
from airflow_breeze.utils.custom_param_types import NotVerifiedBetterChoice
from airflow_breeze.utils.docker_command_utils import (
+ DOCKER_COMPOSE_COMMAND,
get_env_variables_for_docker_commands,
perform_environment_checks,
)
@@ -269,7 +270,7 @@ def tests(
if db_reset:
env_variables["DB_RESET"] = "true"
perform_environment_checks(verbose=verbose)
- cmd = ['docker-compose', 'run', '--service-ports', '--rm', 'airflow']
+ cmd = [*DOCKER_COMPOSE_COMMAND, 'run', '--service-ports', '--rm', 'airflow']
cmd.extend(list(extra_pytest_args))
version = (
mssql_version
@@ -326,7 +327,7 @@ def helm_tests(
env_variables['RUN_TESTS'] = "true"
env_variables['TEST_TYPE'] = 'Helm'
perform_environment_checks(verbose=verbose)
- cmd = ['docker-compose', 'run', '--service-ports', '--rm', 'airflow']
+ cmd = [*DOCKER_COMPOSE_COMMAND, 'run', '--service-ports', '--rm', 'airflow']
cmd.extend(list(extra_pytest_args))
result = run_command(cmd, verbose=verbose, dry_run=dry_run, env=env_variables, check=False)
sys.exit(result.returncode)
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index 0eb14c51cf..8172a40685 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -266,6 +266,9 @@ Please upgrade to at least {MIN_DOCKER_VERSION}[/]
)
+DOCKER_COMPOSE_COMMAND = ["docker-compose"]
+
+
def check_docker_compose_version(verbose: bool):
"""
Checks if the docker compose version is as expected, including some specific modifications done by
@@ -293,7 +296,8 @@ def check_docker_compose_version(verbose: bool):
capture_output=True,
text=True,
)
-
+ DOCKER_COMPOSE_COMMAND.clear()
+ DOCKER_COMPOSE_COMMAND.extend(['docker', 'compose'])
if docker_compose_version_result.returncode == 0:
docker_compose_version = docker_compose_version_result.stdout
version_extracted = version_pattern.search(docker_compose_version)
[airflow] 03/03: Limit eager upgrade of protobuf library to < 4.21.0 (#26243)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e60d2b5b88c6a45bddc1e671c8495ae04e32ecfe
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Thu Sep 8 21:04:42 2022 +0200
Limit eager upgrade of protobuf library to < 4.21.0 (#26243)
* Limit eager upgrade of protobuf library to < 4.21.0
Until all the Google client libraries get upgraded to >= 2.0.0, we need to
limit the protobuf version.
(cherry picked from commit c6053f52908d6f1605b5e9a0566a653e90cc2d0e)
---
Dockerfile | 6 +++++-
Dockerfile.ci | 6 +++++-
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 92dbe933f4..74d0821962 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1211,7 +1211,11 @@ ARG ADDITIONAL_PYTHON_DEPS=""
# * dill<0.3.3 required by apache-beam
# * pyarrow>=6.0.0 is because pip resolver decides for Python 3.10 to downgrade pyarrow to 5 even if it is OK
# for python 3.10 and other dependencies adding the limit helps resolver to make better decisions
-ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3 pyarrow>=6.0.0"
+# We need to limit the protobuf library to < 4.21.0 because not all google libraries we use
+# are compatible with the new protobuf version. All the google python client libraries need
+# to be upgraded to >=2.0.0 in order to able to lift that limitation
+# https://developers.google.com/protocol-buffers/docs/news/2022-05-06#python-updates
+ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3 pyarrow>=6.0.0 protobuf<4.21.0"
ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \
INSTALL_PACKAGES_FROM_CONTEXT=${INSTALL_PACKAGES_FROM_CONTEXT} \
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 722a130d77..5498a16f7c 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -1133,7 +1133,11 @@ RUN echo "Airflow version: ${AIRFLOW_VERSION}"
# * dill<0.3.3 required by apache-beam
# * pyarrow>=6.0.0 is because pip resolver decides for Python 3.10 to downgrade pyarrow to 5 even if it is OK
# for python 3.10 and other dependencies adding the limit helps resolver to make better decisions
-ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3 pyarrow>=6.0.0"
+# We need to limit the protobuf library to < 4.21.0 because not all google libraries we use
+# are compatible with the new protobuf version. All the google python client libraries need
+# to be upgraded to >= 2.0.0 in order to able to lift that limitation
+# https://developers.google.com/protocol-buffers/docs/news/2022-05-06#python-updates
+ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3 pyarrow>=6.0.0 protobuf<4.21.0"
ARG UPGRADE_TO_NEWER_DEPENDENCIES="false"
ENV EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \
UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES}
[airflow] 01/03: Fix case when SHELL variable is not set in kubernetes tests (#26235)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 7f6e0d7728f370aff3a17d104dd656502ed19581
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Thu Sep 8 15:47:32 2022 +0200
Fix case when SHELL variable is not set in kubernetes tests (#26235)
When SHELL variable is not set, kubernetes tests will fall back
to using 'bash'
(cherry picked from commit b7db311336bbe909ee7f559af2b1da7635c54823)
---
dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py
index 488be2548b..7857260e08 100644
--- a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py
@@ -1332,7 +1332,7 @@ def _run_tests(
env = get_k8s_env(python=python, kubernetes_version=kubernetes_version, executor=executor)
kubectl_cluster_name = get_kubectl_cluster_name(python=python, kubernetes_version=kubernetes_version)
get_console(output=output).print(f"\n[info]Running tests with {kubectl_cluster_name} cluster.")
- shell_binary = env['SHELL']
+ shell_binary = env.get('SHELL', shutil.which('bash'))
extra_shell_args: List[str] = []
if shell_binary.endswith("zsh"):
extra_shell_args.append('--no-rcs')