You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/05/19 16:22:20 UTC
[airflow] 05/05: Further speed up fixing ownership in CI (#23782)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit ec92aa9707aa88f3482e86ed5964efd6c1e1b56c
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Thu May 19 17:43:26 2022 +0200
Further speed up fixing ownership in CI (#23782)
After #23775 I noticed that there is yet another small improvement
area in the CI buld speed. Currently build-ci-image builds and push
only "commit-tagged" images, but "fix-ownership" requires
the "latest" image to run.
This PR adds --tag-as-latest option also to build-image and
build-prod-image commands - similarly as for the pull-image and
pull-prod-image. This will retag the "commit" images as latest in the
build-ci-images step and allow to save 1m on pulling the latest image
before fix-ownership (bringing it back to 1s overhead)
(cherry picked from commit 252ef66438ecda87a8aac4beed1f689f14ee8bec)
---
.github/workflows/build-images.yml | 3 +-
.github/workflows/ci.yml | 3 +-
.../airflow_breeze/commands/ci_image_commands.py | 15 ++++--
.../configuration_and_maintenance_commands.py | 5 +-
.../airflow_breeze/commands/developer_commands.py | 16 ++++---
.../commands/production_image_commands.py | 7 ++-
.../commands/release_management_commands.py | 7 ++-
.../airflow_breeze/params/_common_build_params.py | 1 +
.../src/airflow_breeze/utils/common_options.py | 2 +-
.../airflow_breeze/utils/docker_command_utils.py | 9 ++--
dev/breeze/src/airflow_breeze/utils/image.py | 55 ++++++++++++++--------
dev/breeze/src/airflow_breeze/utils/run_utils.py | 4 +-
images/breeze/output-commands-hash.txt | 2 +-
13 files changed, 81 insertions(+), 48 deletions(-)
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index f95b404e7f..9970a82e6c 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -227,7 +227,7 @@ jobs:
- name: "Free space"
run: breeze free-space
- name: Build & Push CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- run: breeze build-image --push-image
+ run: breeze build-image --push-image --tag-as-latest
env:
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
@@ -341,6 +341,7 @@ jobs:
- name: Build & Push PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: >
breeze build-prod-image
+ --tag-as-latest
--push-image
--install-packages-from-context
--disable-airflow-repo-cache
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3456c43f1e..cb24723b10 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -332,7 +332,7 @@ jobs:
run: breeze free-space
if: needs.build-info.outputs.inWorkflowBuild == 'true'
- name: Build & Push CI image ${{ matrix.python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- run: breeze build-image --push-image
+ run: breeze build-image --push-image --tag-as-latest
env:
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
@@ -427,6 +427,7 @@ jobs:
- name: Build & Push PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: >
breeze build-prod-image
+ --tag-as-latest
--push-image
--install-packages-from-context
--disable-airflow-repo-cache
diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
index 1bf3bb461d..7a04670c52 100644
--- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py
@@ -73,7 +73,7 @@ from airflow_breeze.utils.docker_command_utils import (
prepare_docker_build_command,
prepare_empty_docker_build_command,
)
-from airflow_breeze.utils.image import run_pull_image, run_pull_in_parallel
+from airflow_breeze.utils.image import run_pull_image, run_pull_in_parallel, tag_image_as_latest
from airflow_breeze.utils.mark_image_as_refreshed import mark_image_as_refreshed
from airflow_breeze.utils.md5_build_check import md5sum_check_if_build_is_needed
from airflow_breeze.utils.parallel import check_async_run_results
@@ -107,6 +107,7 @@ CI_IMAGE_TOOLS_PARAMETERS = {
"--upgrade-to-newer-dependencies",
"--debian-version",
"--image-tag",
+ "--tag-as-latest",
"--docker-cache",
"--force-build",
],
@@ -218,6 +219,7 @@ CI_IMAGE_TOOLS_PARAMETERS = {
@option_runtime_apt_deps
@option_force_build
@option_airflow_constraints_mode_ci
+@option_tag_as_latest
def build_image(
verbose: bool,
dry_run: bool,
@@ -259,8 +261,8 @@ def build_image(
@option_github_token
@option_verify_image
@option_wait_for_image
-@option_tag_as_latest
@option_image_tag
+@option_tag_as_latest
@click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED)
def pull_image(
verbose: bool,
@@ -483,7 +485,14 @@ def build_ci_image(verbose: bool, dry_run: bool, ci_image_params: BuildCiParams)
if not ci_image_params.prepare_buildx_cache:
if not dry_run:
if build_command_result.returncode == 0:
- mark_image_as_refreshed(ci_image_params)
+ if ci_image_params.tag_as_latest:
+ build_command_result = tag_image_as_latest(ci_image_params, dry_run, verbose)
+ if (
+ ci_image_params.airflow_image_name == ci_image_params.airflow_image_name_with_tag
+ or ci_image_params.tag_as_latest
+ and build_command_result.returncode == 0
+ ):
+ mark_image_as_refreshed(ci_image_params)
else:
get_console().print("[error]Error when building image![/]")
return (
diff --git a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py
index c1ba93b868..032b5f0380 100644
--- a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py
@@ -437,11 +437,12 @@ def command_hash_export(verbose: bool, output: IO):
@main.command(name="fix-ownership", help="Fix ownership of source files to be same as host user.")
+@option_github_repository
@option_verbose
@option_dry_run
-def fix_ownership(verbose: bool, dry_run: bool):
+def fix_ownership(github_repository: str, verbose: bool, dry_run: bool):
perform_environment_checks(verbose=verbose)
- shell_params = find_available_ci_image(dry_run, verbose)
+ shell_params = find_available_ci_image(github_repository, dry_run, verbose)
extra_docker_flags = get_extra_docker_flags(MOUNT_ALL)
env = get_env_variables_for_docker_commands(shell_params)
cmd = [
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index 311fd3f726..1cc9fba59c 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -16,9 +16,8 @@
# under the License.
import os
-import subprocess
import sys
-from typing import Optional, Tuple, Union
+from typing import Optional, Tuple
import rich_click as click
@@ -67,7 +66,12 @@ from airflow_breeze.utils.docker_command_utils import (
perform_environment_checks,
)
from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
-from airflow_breeze.utils.run_utils import assert_pre_commit_installed, filter_out_none, run_command
+from airflow_breeze.utils.run_utils import (
+ RunCommandResult,
+ assert_pre_commit_installed,
+ filter_out_none,
+ run_command,
+)
from airflow_breeze.utils.visuals import ASCIIART, ASCIIART_STYLE, CHEATSHEET, CHEATSHEET_STYLE
DEVELOPER_COMMANDS = {
@@ -554,7 +558,7 @@ def exec(verbose: bool, dry_run: bool, exec_args: Tuple):
sys.exit(process.returncode)
-def enter_shell(**kwargs) -> Union[subprocess.CompletedProcess, subprocess.CalledProcessError]:
+def enter_shell(**kwargs) -> RunCommandResult:
"""
Executes entering shell using the parameters passed as kwargs:
@@ -578,9 +582,7 @@ def enter_shell(**kwargs) -> Union[subprocess.CompletedProcess, subprocess.Calle
return run_shell(verbose, dry_run, enter_shell_params)
-def run_shell(
- verbose: bool, dry_run: bool, shell_params: ShellParams
-) -> Union[subprocess.CompletedProcess, subprocess.CalledProcessError]:
+def run_shell(verbose: bool, dry_run: bool, shell_params: ShellParams) -> RunCommandResult:
"""
Executes a shell command built from params passed.
* prints information about the build
diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py
index 5fced6abda..53349e97c2 100644
--- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py
@@ -71,7 +71,7 @@ from airflow_breeze.utils.docker_command_utils import (
prepare_docker_build_command,
prepare_empty_docker_build_command,
)
-from airflow_breeze.utils.image import run_pull_image, run_pull_in_parallel
+from airflow_breeze.utils.image import run_pull_image, run_pull_in_parallel, tag_image_as_latest
from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, DOCKER_CONTEXT_DIR
from airflow_breeze.utils.python_versions import get_python_version_list
from airflow_breeze.utils.registry import login_to_github_docker_registry
@@ -96,6 +96,7 @@ PRODUCTION_IMAGE_TOOLS_PARAMETERS = {
"--upgrade-to-newer-dependencies",
"--debian-version",
"--image-tag",
+ "--tag-as-latest",
"--docker-cache",
],
},
@@ -259,6 +260,7 @@ PRODUCTION_IMAGE_TOOLS_PARAMETERS = {
@option_dev_apt_deps
@option_runtime_apt_command
@option_runtime_apt_deps
+@option_tag_as_latest
def build_prod_image(
verbose: bool,
dry_run: bool,
@@ -520,5 +522,8 @@ def build_production_image(
build_command_result = build_cache(
image_params=prod_image_params, dry_run=dry_run, verbose=verbose
)
+ else:
+ if prod_image_params.tag_as_latest:
+ build_command_result = tag_image_as_latest(prod_image_params, dry_run, verbose)
return build_command_result.returncode, f"Image build: {prod_image_params.python}"
diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index e5851d0c92..1e5b8286ff 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -20,8 +20,7 @@ import sys
import time
from copy import deepcopy
from re import match
-from subprocess import CalledProcessError, CompletedProcess
-from typing import IO, Dict, List, Optional, Tuple, Union
+from typing import IO, Dict, List, Optional, Tuple
import click
@@ -70,7 +69,7 @@ from airflow_breeze.utils.docker_command_utils import (
from airflow_breeze.utils.find_newer_dependencies import find_newer_dependencies
from airflow_breeze.utils.parallel import check_async_run_results
from airflow_breeze.utils.python_versions import get_python_version_list
-from airflow_breeze.utils.run_utils import run_command
+from airflow_breeze.utils.run_utils import RunCommandResult, run_command
RELEASE_MANAGEMENT_PARAMETERS = {
"breeze prepare-airflow-package": [
@@ -182,7 +181,7 @@ def run_with_debug(
dry_run: bool,
debug: bool,
enable_input: bool = False,
-) -> Union[CompletedProcess, CalledProcessError]:
+) -> RunCommandResult:
env_variables = get_env_variables_for_docker_commands(params)
extra_docker_flags = get_extra_docker_flags(mount_sources=params.mount_sources)
if enable_input or debug:
diff --git a/dev/breeze/src/airflow_breeze/params/_common_build_params.py b/dev/breeze/src/airflow_breeze/params/_common_build_params.py
index 9b510a06f8..2fda00fbd5 100644
--- a/dev/breeze/src/airflow_breeze/params/_common_build_params.py
+++ b/dev/breeze/src/airflow_breeze/params/_common_build_params.py
@@ -64,6 +64,7 @@ class _CommonBuildParams:
python: str = "3.7"
runtime_apt_command: str = ""
runtime_apt_deps: str = ""
+ tag_as_latest: bool = False
upgrade_to_newer_dependencies: bool = False
@property
diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py
index 615b7a2565..4d0fcfa21b 100644
--- a/dev/breeze/src/airflow_breeze/utils/common_options.py
+++ b/dev/breeze/src/airflow_breeze/utils/common_options.py
@@ -310,7 +310,7 @@ option_wait_for_image = click.option(
option_tag_as_latest = click.option(
'--tag-as-latest',
help='Tags the image as latest and update checksum of all files after pulling. '
- 'Used in CI to pull the image built in another job.',
+ 'Useful when you build or pull image with --image-tag.',
is_flag=True,
envvar='TAG_AS_LATEST',
)
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index 1efee4cf1c..85361c8d2a 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -56,6 +56,7 @@ from airflow_breeze.global_constants import (
)
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.run_utils import (
+ RunCommandResult,
commit_sha,
prepare_base_build_command,
prepare_build_cache_command,
@@ -118,9 +119,7 @@ def get_extra_docker_flags(mount_sources: str) -> List[str]:
return extra_docker_flags
-def check_docker_resources(
- airflow_image_name: str, verbose: bool, dry_run: bool
-) -> Union[CompletedProcess, CalledProcessError]:
+def check_docker_resources(airflow_image_name: str, verbose: bool, dry_run: bool) -> RunCommandResult:
"""
Check if we have enough resources to run docker. This is done via running script embedded in our image.
:param verbose: print commands when running
@@ -418,9 +417,7 @@ def prepare_empty_docker_build_command(
return ["docker", "build", "-t", image_params.airflow_image_name_with_tag, "-"]
-def build_cache(
- image_params: _CommonBuildParams, dry_run: bool, verbose: bool
-) -> Union[CompletedProcess, CalledProcessError]:
+def build_cache(image_params: _CommonBuildParams, dry_run: bool, verbose: bool) -> RunCommandResult:
build_command_result: Union[CompletedProcess, CalledProcessError] = CompletedProcess(
args=[], returncode=0
)
diff --git a/dev/breeze/src/airflow_breeze/utils/image.py b/dev/breeze/src/airflow_breeze/utils/image.py
index c5ec0cde66..4131b1c0bb 100644
--- a/dev/breeze/src/airflow_breeze/utils/image.py
+++ b/dev/breeze/src/airflow_breeze/utils/image.py
@@ -34,7 +34,7 @@ from airflow_breeze.utils.mark_image_as_refreshed import mark_image_as_refreshed
from airflow_breeze.utils.parallel import check_async_run_results
from airflow_breeze.utils.registry import login_to_github_docker_registry
from airflow_breeze.utils.run_tests import verify_an_image
-from airflow_breeze.utils.run_utils import run_command
+from airflow_breeze.utils.run_utils import RunCommandResult, run_command
def run_pull_in_parallel(
@@ -137,18 +137,7 @@ def run_pull_image(
f"Image Python {image_params.python}",
)
if tag_as_latest:
- command_result = run_command(
- [
- "docker",
- "tag",
- image_params.airflow_image_name_with_tag,
- image_params.airflow_image_name,
- ],
- capture_output=True,
- verbose=verbose,
- dry_run=dry_run,
- check=False,
- )
+ command_result = tag_image_as_latest(image_params, dry_run, verbose)
if command_result.returncode == 0 and isinstance(image_params, BuildCiParams):
mark_image_as_refreshed(image_params)
return command_result.returncode, f"Image Python {image_params.python}"
@@ -167,6 +156,26 @@ def run_pull_image(
return command_result.returncode, f"Image Python {image_params.python}"
+def tag_image_as_latest(image_params: _CommonBuildParams, dry_run: bool, verbose: bool) -> RunCommandResult:
+ if image_params.airflow_image_name_with_tag == image_params.airflow_image_name:
+ get_console().print(
+ f"[info]Skip tagging {image_params.airflow_image_name} " "as latest as it is already 'latest'[/]"
+ )
+ return subprocess.CompletedProcess(returncode=0, args=[])
+ return run_command(
+ [
+ "docker",
+ "tag",
+ image_params.airflow_image_name_with_tag,
+ image_params.airflow_image_name,
+ ],
+ capture_output=True,
+ verbose=verbose,
+ dry_run=dry_run,
+ check=False,
+ )
+
+
def run_pull_and_verify_image(
image_params: _CommonBuildParams,
dry_run: bool,
@@ -193,12 +202,13 @@ def run_pull_and_verify_image(
def just_pull_ci_image(
- python_version: str, dry_run: bool, verbose: bool
-) -> Tuple[ShellParams, Union[subprocess.CompletedProcess, subprocess.CalledProcessError]]:
+ github_repository, python_version: str, dry_run: bool, verbose: bool
+) -> Tuple[ShellParams, RunCommandResult]:
shell_params = ShellParams(
verbose=verbose,
mount_sources=MOUNT_ALL,
python=python_version,
+ github_repository=github_repository,
skip_environment_initialization=True,
)
get_console().print(f"[info]Pulling {shell_params.airflow_image_name_with_tag}.[/]")
@@ -212,12 +222,13 @@ def just_pull_ci_image(
def check_if_ci_image_available(
- python_version: str, dry_run: bool, verbose: bool
-) -> Tuple[ShellParams, Union[subprocess.CompletedProcess, subprocess.CalledProcessError]]:
+ github_repository: str, python_version: str, dry_run: bool, verbose: bool
+) -> Tuple[ShellParams, RunCommandResult]:
shell_params = ShellParams(
verbose=verbose,
mount_sources=MOUNT_ALL,
python=python_version,
+ github_repository=github_repository,
skip_environment_initialization=True,
)
inspect_command_result = run_command(
@@ -233,13 +244,17 @@ def check_if_ci_image_available(
)
-def find_available_ci_image(dry_run: bool, verbose: bool) -> ShellParams:
+def find_available_ci_image(github_repository: str, dry_run: bool, verbose: bool) -> ShellParams:
for python_version in ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS:
- shell_params, inspect_command_result = check_if_ci_image_available(python_version, dry_run, verbose)
+ shell_params, inspect_command_result = check_if_ci_image_available(
+ github_repository, python_version, dry_run, verbose
+ )
if inspect_command_result.returncode == 0:
get_console().print(
"[info]Running fix_ownership " f"with {shell_params.airflow_image_name_with_tag}.[/]"
)
return shell_params
- shell_params, _ = just_pull_ci_image(DEFAULT_PYTHON_MAJOR_MINOR_VERSION, dry_run, verbose)
+ shell_params, _ = just_pull_ci_image(
+ github_repository, DEFAULT_PYTHON_MAJOR_MINOR_VERSION, dry_run, verbose
+ )
return shell_params
diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py
index d0d0d535d5..241407a1af 100644
--- a/dev/breeze/src/airflow_breeze/utils/run_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py
@@ -32,6 +32,8 @@ from airflow_breeze.utils.ci_group import ci_group
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
+RunCommandResult = Union[subprocess.CompletedProcess, subprocess.CalledProcessError]
+
def run_command(
cmd: List[str],
@@ -45,7 +47,7 @@ def run_command(
cwd: Optional[Path] = None,
input: Optional[str] = None,
**kwargs,
-) -> Union[subprocess.CompletedProcess, subprocess.CalledProcessError]:
+) -> RunCommandResult:
"""
Runs command passed as list of strings with some extra functionality over POpen (kwargs from PoPen can
be used in this command even if not explicitly specified).
diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt
index eddf757dea..9eeb12fd47 100644
--- a/images/breeze/output-commands-hash.txt
+++ b/images/breeze/output-commands-hash.txt
@@ -1 +1 @@
-8beb1fd4bb08181701933ace04eb0911
+bc274910e868b80f476c1048dea1f957