You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/06/29 08:48:13 UTC
[airflow] 06/39: Remove warnings when starting breeze (#24183)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 86c4540dedef05f0fd751f79feac5efd86cfbec3
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Sun Jun 5 11:51:04 2022 +0200
Remove warnings when starting breeze (#24183)
Breeze when started produced three warnings that were harmless,
but we should fix them to remove "false positives".
(cherry picked from commit ac8a790ab9dabfa747964b65aa9b93451b735ede)
---
dev/breeze/src/airflow_breeze/breeze.py | 6 +++++-
.../airflow_breeze/commands/developer_commands.py | 3 ++-
.../src/airflow_breeze/commands/main_command.py | 2 --
.../src/airflow_breeze/params/build_ci_params.py | 1 +
.../src/airflow_breeze/params/build_prod_params.py | 3 ++-
dev/breeze/src/airflow_breeze/params/shell_params.py | 5 +++++
.../src/airflow_breeze/utils/docker_command_utils.py | 16 ++++------------
dev/breeze/src/airflow_breeze/utils/path_utils.py | 14 ++++++++++++++
scripts/ci/docker-compose/local.yml | 7 +------
.../breeze.py => scripts/ci/docker-compose/mypy.yml | 20 +++++++++++---------
scripts/ci/pre_commit/pre_commit_flake8.py | 3 ++-
.../ci/pre_commit/pre_commit_migration_reference.py | 3 ++-
scripts/ci/pre_commit/pre_commit_mypy.py | 3 ++-
scripts/ci/pre_commit/pre_commit_ui_lint.py | 3 ++-
scripts/ci/pre_commit/pre_commit_www_lint.py | 3 ++-
15 files changed, 55 insertions(+), 37 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/breeze.py b/dev/breeze/src/airflow_breeze/breeze.py
index a596c2be49..608619ae8e 100755
--- a/dev/breeze/src/airflow_breeze/breeze.py
+++ b/dev/breeze/src/airflow_breeze/breeze.py
@@ -17,9 +17,13 @@
# under the License.
from airflow_breeze.configure_rich_click import click # isort: skip # noqa
from airflow_breeze.commands.main_command import main
-from airflow_breeze.utils.path_utils import find_airflow_sources_root_to_operate_on
+from airflow_breeze.utils.path_utils import (
+ create_directories_and_files,
+ find_airflow_sources_root_to_operate_on,
+)
find_airflow_sources_root_to_operate_on()
+create_directories_and_files()
if __name__ == '__main__':
main()
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index 1719bab5fc..e10e79d0a8 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -524,7 +524,7 @@ def stop(verbose: bool, dry_run: bool, preserve_volumes: bool):
command_to_execute = ['docker-compose', 'down', "--remove-orphans"]
if not preserve_volumes:
command_to_execute.append("--volumes")
- shell_params = ShellParams(verbose=verbose, backend="all")
+ shell_params = ShellParams(verbose=verbose, backend="all", include_mypy_volume=True)
env_variables = get_env_variables_for_docker_commands(shell_params)
run_command(command_to_execute, verbose=verbose, dry_run=dry_run, env=env_variables)
@@ -579,6 +579,7 @@ def enter_shell(**kwargs) -> RunCommandResult:
if read_from_cache_file('suppress_cheatsheet') is None:
get_console().print(CHEATSHEET, style=CHEATSHEET_STYLE)
enter_shell_params = ShellParams(**filter_out_none(**kwargs))
+ enter_shell_params.include_mypy_volume = True
rebuild_ci_image_if_needed(build_params=enter_shell_params, dry_run=dry_run, verbose=verbose)
return run_shell(verbose, dry_run, enter_shell_params)
diff --git a/dev/breeze/src/airflow_breeze/commands/main_command.py b/dev/breeze/src/airflow_breeze/commands/main_command.py
index 8ed451b0e5..c62101a44f 100644
--- a/dev/breeze/src/airflow_breeze/commands/main_command.py
+++ b/dev/breeze/src/airflow_breeze/commands/main_command.py
@@ -37,7 +37,6 @@ from airflow_breeze.utils.common_options import (
option_use_packages_from_dist,
option_verbose,
)
-from airflow_breeze.utils.path_utils import create_directories_and_files
@click.group(invoke_without_command=True, context_settings={'help_option_names': ['-h', '--help']})
@@ -64,6 +63,5 @@ from airflow_breeze.utils.path_utils import create_directories_and_files
def main(ctx: click.Context, **kwargs):
from airflow_breeze.commands.developer_commands import shell
- create_directories_and_files()
if not ctx.invoked_subcommand:
ctx.forward(shell, extra_args={})
diff --git a/dev/breeze/src/airflow_breeze/params/build_ci_params.py b/dev/breeze/src/airflow_breeze/params/build_ci_params.py
index aa9681fc95..52c242b65a 100644
--- a/dev/breeze/src/airflow_breeze/params/build_ci_params.py
+++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py
@@ -31,6 +31,7 @@ class BuildCiParams(CommonBuildParams):
"""
airflow_constraints_mode: str = "constraints-source-providers"
+ default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
airflow_constraints_reference: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
airflow_extras: str = "devel_ci"
airflow_pre_cached_pip_packages: bool = True
diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py
index e81a5d6739..c37569f85f 100644
--- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py
+++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py
@@ -21,7 +21,7 @@ import sys
from dataclasses import dataclass
from typing import List
-from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
+from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
from airflow_breeze.global_constants import (
AIRFLOW_SOURCES_FROM,
AIRFLOW_SOURCES_TO,
@@ -41,6 +41,7 @@ class BuildProdParams(CommonBuildParams):
"""
airflow_constraints_mode: str = "constraints"
+ default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
airflow_constraints_reference: str = ""
airflow_is_in_context: bool = False
cleanup_context: bool = False
diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py
index 58107b9e27..4ba4022651 100644
--- a/dev/breeze/src/airflow_breeze/params/shell_params.py
+++ b/dev/breeze/src/airflow_breeze/params/shell_params.py
@@ -48,6 +48,7 @@ class ShellParams:
"""
airflow_branch: str = AIRFLOW_BRANCH
+ default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
airflow_constraints_reference: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
airflow_extras: str = ""
answer: Optional[str] = None
@@ -64,6 +65,7 @@ class ShellParams:
github_repository: str = "apache/airflow"
github_token: str = os.environ.get('GITHUB_TOKEN', "")
image_tag: str = "latest"
+ include_mypy_volume: bool = False
install_airflow_version: str = ""
install_providers_from_sources: bool = True
integration: Tuple[str, ...] = ()
@@ -189,6 +191,7 @@ class ShellParams:
local_all_sources_docker_compose_file = f"{str(SCRIPTS_CI_DIR)}/docker-compose/local-all-sources.yml"
files_docker_compose_file = f"{str(SCRIPTS_CI_DIR)}/docker-compose/files.yml"
remove_sources_docker_compose_file = f"{str(SCRIPTS_CI_DIR)}/docker-compose/remove-sources.yml"
+ mypy_docker_compose_file = f"{str(SCRIPTS_CI_DIR)}/docker-compose/mypy.yml"
forward_credentials_docker_compose_file = (
f"{str(SCRIPTS_CI_DIR)}/docker-compose/forward-credentials.yml"
)
@@ -213,6 +216,8 @@ class ShellParams:
compose_ci_file.append(forward_credentials_docker_compose_file)
if self.use_airflow_version is not None:
compose_ci_file.append(remove_sources_docker_compose_file)
+ if self.include_mypy_volume:
+ compose_ci_file.append(mypy_docker_compose_file)
if "all" in self.integration:
integrations = AVAILABLE_INTEGRATIONS
else:
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index 05bcc11d6c..1f30642edd 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -17,7 +17,6 @@
"""Various utils to prepare docker and docker compose commands."""
import os
import re
-import subprocess
import sys
from copy import deepcopy
from random import randint
@@ -97,16 +96,6 @@ NECESSARY_HOST_VOLUMES = [
]
-def create_volume_if_missing(volume_name: str):
- res_inspect = run_command(cmd=["docker", "inspect", volume_name], stdout=subprocess.DEVNULL, check=False)
- if res_inspect.returncode != 0:
- run_command(cmd=["docker", "volume", "create", volume_name], check=True)
-
-
-def create_static_check_volumes():
- create_volume_if_missing("docker-compose_mypy-cache-volume")
-
-
def get_extra_docker_flags(mount_sources: str) -> List[str]:
"""
Returns extra docker flags based on the type of mounting we want to do for sources.
@@ -124,7 +113,7 @@ def get_extra_docker_flags(mount_sources: str) -> List[str]:
["--mount", f'type=bind,src={AIRFLOW_SOURCES_ROOT / src},dst={dst}']
)
extra_docker_flags.extend(
- ['--mount', "type=volume,src=docker-compose_mypy-cache-volume,dst=/opt/airflow/.mypy_cache"]
+ ['--mount', "type=volume,src=mypy-cache-volume,dst=/opt/airflow/.mypy_cache"]
)
else: # none
extra_docker_flags.extend(
@@ -512,6 +501,7 @@ def update_expected_environment_variables(env: Dict[str, str]) -> None:
:param env: environment variables to update with missing values if not set.
"""
set_value_to_default_if_not_set(env, 'AIRFLOW_CONSTRAINTS_MODE', "constraints-source-providers")
+ set_value_to_default_if_not_set(env, 'AIRFLOW_CONSTRAINTS_REFERENCE', "constraints-source-providers")
set_value_to_default_if_not_set(env, 'AIRFLOW_EXTRAS', "")
set_value_to_default_if_not_set(env, 'ANSWER', "")
set_value_to_default_if_not_set(env, 'BREEZE', "true")
@@ -557,7 +547,9 @@ DERIVE_ENV_VARIABLES_FROM_ATTRIBUTES = {
"AIRFLOW_CI_IMAGE": "airflow_image_name",
"AIRFLOW_CI_IMAGE_WITH_TAG": "airflow_image_name_with_tag",
"AIRFLOW_EXTRAS": "airflow_extras",
+ "DEFAULT_CONSTRAINTS_BRANCH": "default-constraints-branch",
"AIRFLOW_CONSTRAINTS_MODE": "airflow_constraints_mode",
+ "AIRFLOW_CONSTRAINTS_REFERENCE": "airflow_constraints_reference",
"AIRFLOW_IMAGE_KUBERNETES": "airflow_image_kubernetes",
"AIRFLOW_PROD_IMAGE": "airflow_image_name",
"AIRFLOW_SOURCES": "airflow_sources",
diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py
index 759c3e09a7..e4e79d5520 100644
--- a/dev/breeze/src/airflow_breeze/utils/path_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py
@@ -19,6 +19,7 @@ Useful tools for various Paths used inside Airflow Sources.
"""
import hashlib
import os
+import subprocess
import sys
import tempfile
from functools import lru_cache
@@ -249,6 +250,18 @@ OUTPUT_LOG = Path(CACHE_TMP_FILE_DIR.name, 'out.log')
BREEZE_SOURCES_ROOT = AIRFLOW_SOURCES_ROOT / "dev" / "breeze"
+def create_volume_if_missing(volume_name: str):
+ from airflow_breeze.utils.run_utils import run_command
+
+ res_inspect = run_command(cmd=["docker", "inspect", volume_name], stdout=subprocess.DEVNULL, check=False)
+ if res_inspect.returncode != 0:
+ run_command(cmd=["docker", "volume", "create", volume_name], check=True)
+
+
+def create_static_check_volumes():
+ create_volume_if_missing("mypy-cache-volume")
+
+
def create_directories_and_files() -> None:
"""
Creates all directories and files that are needed for Breeze to work via docker-compose.
@@ -264,3 +277,4 @@ def create_directories_and_files() -> None:
(AIRFLOW_SOURCES_ROOT / ".bash_aliases").touch()
(AIRFLOW_SOURCES_ROOT / ".bash_history").touch()
(AIRFLOW_SOURCES_ROOT / ".inputrc").touch()
+ create_static_check_volumes()
diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml
index e356cb8736..9e63ffa467 100644
--- a/scripts/ci/docker-compose/local.yml
+++ b/scripts/ci/docker-compose/local.yml
@@ -26,10 +26,7 @@ services:
# or those that might be useful to see in the host as output of the
# tests (such as logs)
volumes:
- - type: volume
- source: mypy-cache-volume
- target: /opt/airflow/.mypy_cache/
- # START automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py
+ # START automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py
- type: bind
source: ../../../.bash_aliases
target: /root/.bash_aliases
@@ -121,5 +118,3 @@ services:
source: ../../../metastore_browser
target: /opt/airflow/metastore_browser
# END automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py
-volumes:
- mypy-cache-volume:
diff --git a/dev/breeze/src/airflow_breeze/breeze.py b/scripts/ci/docker-compose/mypy.yml
old mode 100755
new mode 100644
similarity index 71%
copy from dev/breeze/src/airflow_breeze/breeze.py
copy to scripts/ci/docker-compose/mypy.yml
index a596c2be49..7287738690
--- a/dev/breeze/src/airflow_breeze/breeze.py
+++ b/scripts/ci/docker-compose/mypy.yml
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,11 +14,14 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from airflow_breeze.configure_rich_click import click # isort: skip # noqa
-from airflow_breeze.commands.main_command import main
-from airflow_breeze.utils.path_utils import find_airflow_sources_root_to_operate_on
-
-find_airflow_sources_root_to_operate_on()
-
-if __name__ == '__main__':
- main()
+---
+version: "3.7"
+services:
+ airflow:
+ volumes:
+ - type: volume
+ source: mypy-cache-volume
+ target: /opt/airflow/.mypy_cache/
+volumes:
+ mypy-cache-volume:
+ external: true
diff --git a/scripts/ci/pre_commit/pre_commit_flake8.py b/scripts/ci/pre_commit/pre_commit_flake8.py
index 59bd356d24..22b0997893 100755
--- a/scripts/ci/pre_commit/pre_commit_flake8.py
+++ b/scripts/ci/pre_commit/pre_commit_flake8.py
@@ -34,7 +34,8 @@ DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t"
if __name__ == '__main__':
sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src"))
from airflow_breeze.global_constants import MOUNT_SELECTED
- from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags
+ from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags
+ from airflow_breeze.utils.path_utils import create_static_check_volumes
from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command
airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN)
diff --git a/scripts/ci/pre_commit/pre_commit_migration_reference.py b/scripts/ci/pre_commit/pre_commit_migration_reference.py
index 154a795f3e..25ac796fdf 100755
--- a/scripts/ci/pre_commit/pre_commit_migration_reference.py
+++ b/scripts/ci/pre_commit/pre_commit_migration_reference.py
@@ -34,7 +34,8 @@ DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t"
if __name__ == '__main__':
sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src"))
from airflow_breeze.global_constants import MOUNT_SELECTED
- from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags
+ from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags
+ from airflow_breeze.utils.path_utils import create_static_check_volumes
from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command
airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN)
diff --git a/scripts/ci/pre_commit/pre_commit_mypy.py b/scripts/ci/pre_commit/pre_commit_mypy.py
index 74d511a65b..965bed9a74 100755
--- a/scripts/ci/pre_commit/pre_commit_mypy.py
+++ b/scripts/ci/pre_commit/pre_commit_mypy.py
@@ -35,7 +35,8 @@ DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t"
if __name__ == '__main__':
sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src"))
from airflow_breeze.global_constants import MOUNT_SELECTED
- from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags
+ from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags
+ from airflow_breeze.utils.path_utils import create_static_check_volumes
from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command
airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN)
diff --git a/scripts/ci/pre_commit/pre_commit_ui_lint.py b/scripts/ci/pre_commit/pre_commit_ui_lint.py
index 7755e3d204..5812344a45 100755
--- a/scripts/ci/pre_commit/pre_commit_ui_lint.py
+++ b/scripts/ci/pre_commit/pre_commit_ui_lint.py
@@ -33,7 +33,8 @@ DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t"
if __name__ == '__main__':
sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src"))
from airflow_breeze.global_constants import MOUNT_SELECTED
- from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags
+ from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags
+ from airflow_breeze.utils.path_utils import create_static_check_volumes
from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command
airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN)
diff --git a/scripts/ci/pre_commit/pre_commit_www_lint.py b/scripts/ci/pre_commit/pre_commit_www_lint.py
index 65553c3d81..b22750c56d 100755
--- a/scripts/ci/pre_commit/pre_commit_www_lint.py
+++ b/scripts/ci/pre_commit/pre_commit_www_lint.py
@@ -34,7 +34,8 @@ DRY_RUN = os.environ.get('DRY_RUN', "false")[0].lower() == "t"
if __name__ == '__main__':
sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src"))
from airflow_breeze.global_constants import MOUNT_SELECTED
- from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags
+ from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags
+ from airflow_breeze.utils.path_utils import create_static_check_volumes
from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command
airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN)