You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/03/20 20:54:40 UTC
[airflow] branch master updated: Adds resource check when running
Breeze (#14908)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 3dd42a5 Adds resource check when running Breeze (#14908)
3dd42a5 is described below
commit 3dd42a5a3f4620fdf4db3355d29500bfd08c17d7
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sat Mar 20 21:54:30 2021 +0100
Adds resource check when running Breeze (#14908)
When Breeze is run, it requires some resources in the docker
engine, otherwise it will produce strange errors.
This PR adds resource check when running breeze - it will print
human-friendly size of CPU/Memory/Disk available for docker
engine and red error (still allowing Docker to run) when the
resources are not enough.
Fixes: #14899
---
BREEZE.rst | 16 ++++-
breeze | 12 ++++
scripts/ci/libraries/_all_libs.sh | 2 +
scripts/ci/libraries/_docker_engine_resources.sh | 75 ++++++++++++++++++++++++
scripts/ci/testing/ci_run_airflow_testing.sh | 10 ++++
5 files changed, 114 insertions(+), 1 deletion(-)
diff --git a/BREEZE.rst b/BREEZE.rst
index af57c64..6e71868 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -173,11 +173,13 @@ Let's confirm that ``getopt`` and ``gstat`` utilities are successfully installed
Written by Michael Meskes.
+Resources required
+==================
Memory
------
-Minimum 4GB RAM is required to run the full Breeze environment.
+Minimum 4GB RAM for Docker Engine is required to run the full Breeze environment.
On macOS, 2GB of RAM are available for your Docker containers by default, but more memory is recommended
(4GB should be comfortable). For details see
@@ -185,6 +187,18 @@ On macOS, 2GB of RAM are available for your Docker containers by default, but mo
On Windows WSL 2 expect the Linux Distro and Docker containers to use 7 - 8 GB of RAM.
+Disk
+----
+
+Minimum 40GB free disk space is required for your Docker Containers.
+
+On Mac OS This might deteriorate over time so you might need to increase it or run ``docker system --prune``
+periodically. For details see
+`Docker for Mac - Advanced tab <https://docs.docker.com/v17.12/docker-for-mac/#advanced-tab>`_.
+
+On WSL2 you might want to increase your Virtual Hard Disk by following:
+`Expanding the size of your WSL 2 Virtual Hard Disk <https://docs.microsoft.com/en-us/windows/wsl/compare-versions#expanding-the-size-of-your-wsl-2-virtual-hard-disk>`_
+
Cleaning the environment
------------------------
diff --git a/breeze b/breeze
index a45f224..c17bcb4 100755
--- a/breeze
+++ b/breeze
@@ -3430,8 +3430,10 @@ function breeze::run_breeze_command() {
else
dc_run_file="${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}"
fi
+
case "${command_to_run}" in
enter_breeze)
+ docker_engine::check_all_resources
if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
${run_command} "${dc_run_file}" run --service-ports --rm airflow "${@}"
${run_command} "${SCRIPTS_CI_DIR}/tools/ci_fix_ownership.sh"
@@ -3440,6 +3442,7 @@ function breeze::run_breeze_command() {
fi
;;
run_exec)
+ docker_engine::check_all_resources
# Unfortunately `docker-compose exec` does not support exec'ing into containers started with run :(
# so we have to find it manually
set +e
@@ -3451,16 +3454,19 @@ function breeze::run_breeze_command() {
"/opt/airflow/scripts/in_container/entrypoint_exec.sh" "${@}"
;;
run_tests)
+ docker_engine::check_all_resources
export RUN_TESTS="true"
readonly RUN_TESTS
${run_command} "${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}" run --service-ports --rm airflow "$@"
;;
run_docker_compose)
+ docker_engine::check_all_resources
set +u
${run_command} "${dc_run_file}" "${docker_compose_command}" "${EXTRA_DC_OPTIONS[@]}" "$@"
set -u
;;
perform_static_checks)
+ docker_engine::check_all_resources
breeze::make_sure_precommit_is_installed
breeze::run_static_checks "${@}"
;;
@@ -3470,15 +3476,19 @@ function breeze::run_breeze_command() {
breeze::remove_images
;;
perform_generate_constraints)
+ docker_engine::check_all_resources
runs::run_generate_constraints
;;
perform_prepare_airflow_packages)
+ docker_engine::check_all_resources
build_airflow_packages::build_airflow_packages
;;
perform_prepare_provider_packages)
+ docker_engine::check_all_resources
runs::run_prepare_provider_packages "${@}"
;;
perform_prepare_provider_documentation)
+ docker_engine::check_all_resources
runs::run_prepare_provider_documentation "${@}"
;;
perform_push_image)
@@ -3495,11 +3505,13 @@ function breeze::run_breeze_command() {
breeze::setup_autocomplete
;;
manage_kind_cluster)
+ docker_engine::check_all_resources
kind::make_sure_kubernetes_tools_are_installed
kind::get_kind_cluster_name
kind::perform_kind_cluster_operation "${KIND_CLUSTER_OPERATION}"
;;
build_docs)
+ docker_engine::check_all_resources
runs::run_docs "${@}"
;;
toggle_suppress_cheatsheet)
diff --git a/scripts/ci/libraries/_all_libs.sh b/scripts/ci/libraries/_all_libs.sh
index a8aedae..aa48552 100755
--- a/scripts/ci/libraries/_all_libs.sh
+++ b/scripts/ci/libraries/_all_libs.sh
@@ -28,6 +28,8 @@ readonly SCRIPTS_CI_DIR
. "${LIBRARIES_DIR}"/_traps.sh
# shellcheck source=scripts/ci/libraries/_initialization.sh
. "${LIBRARIES_DIR}"/_initialization.sh
+# shellcheck source=scripts/ci/libraries/_docker_engine_resources.sh
+. "${LIBRARIES_DIR}"/_docker_engine_resources.sh
# shellcheck source=scripts/ci/libraries/_repeats.sh
. "${LIBRARIES_DIR}"/_repeats.sh
# shellcheck source=scripts/ci/libraries/_sanity_checks.sh
diff --git a/scripts/ci/libraries/_docker_engine_resources.sh b/scripts/ci/libraries/_docker_engine_resources.sh
new file mode 100644
index 0000000..a3bd12a
--- /dev/null
+++ b/scripts/ci/libraries/_docker_engine_resources.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+function docker_engine::get_available_memory_in_docker() {
+ MEMORY_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash \
+ "${AIRFLOW_CI_IMAGE}" -c \
+ 'echo $(($(getconf _PHYS_PAGES) * $(getconf PAGE_SIZE) / (1024 * 1024)))')
+ echo "${COLOR_BLUE}Memory available for Docker${COLOR_RESET}: $(numfmt --to iec $((MEMORY_AVAILABLE_FOR_DOCKER * 1024 * 1024)))"
+ export MEMORY_AVAILABLE_FOR_DOCKER
+}
+
+function docker_engine::get_available_cpus_in_docker() {
+ CPUS_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash \
+ "${AIRFLOW_CI_IMAGE}" -c \
+ 'grep -cE "cpu[0-9]+" </proc/stat')
+ echo "${COLOR_BLUE}CPUS available for Docker${COLOR_RESET}: ${CPUS_AVAILABLE_FOR_DOCKER}"
+ export CPUS_AVAILABLE_FOR_DOCKER
+}
+
+function docker_engine::get_available_disk_space_in_docker() {
+ DISK_SPACE_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash \
+ "${AIRFLOW_CI_IMAGE}" -c \
+ 'df / | tail -1 | awk '\''{print $4}'\')
+ echo "${COLOR_BLUE}Disk space available for Docker${COLOR_RESET}: $(numfmt --to iec $((DISK_SPACE_AVAILABLE_FOR_DOCKER * 1024)))"
+ export DISK_SPACE_AVAILABLE_FOR_DOCKER
+}
+
+function docker_engine::check_enough_resources() {
+ local successful_resource_check="true"
+ if (( MEMORY_AVAILABLE_FOR_DOCKER < 4000 )) ; then
+ successful_resource_check="false"
+ echo
+ echo "${COLOR_RED}WARNING! Not enough memory to use breeze. At least 4GB memory is required for Docker engine to run Breeze${COLOR_RESET}"
+ fi
+
+ if (( CPUS_AVAILABLE_FOR_DOCKER < 2 )) ; then
+ successful_resource_check="false"
+ echo
+ echo "${COLOR_RED}WARNING! Not enough CPUs to use breeze. At least 2 CPUS are required for Docker engine to run Breeze.${COLOR_RESET}"
+ fi
+
+ if (( DISK_SPACE_AVAILABLE_FOR_DOCKER < 40000000 )) ; then
+ successful_resource_check="false"
+ echo
+ echo "${COLOR_RED}WARNING! Not enough disk space to use breeze. At least 40GB are required for Docker engine to run Breeze.${COLOR_RESET}"
+ fi
+
+ if [[ ${successful_resource_check} != "true" ]];then
+ echo
+ echo "${COLOR_RED}Please check https://github.com/apache/airflow/blob/master/BREEZE.rst#resources-required for details${COLOR_RESET}"
+ echo
+ fi
+}
+
+function docker_engine::check_all_resources() {
+ docker_engine::get_available_memory_in_docker
+ docker_engine::get_available_cpus_in_docker
+ docker_engine::get_available_disk_space_in_docker
+ docker_engine::check_enough_resources
+}
diff --git a/scripts/ci/testing/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh
index e1f4a04..7ea01d7 100755
--- a/scripts/ci/testing/ci_run_airflow_testing.sh
+++ b/scripts/ci/testing/ci_run_airflow_testing.sh
@@ -140,6 +140,11 @@ build_images::rebuild_ci_image_if_needed_with_group
prepare_tests_to_run
+docker_engine::get_available_memory_in_docker
+docker_engine::get_available_cpus_in_docker
+docker_engine::get_available_disk_space_in_docker
+
+
for TEST_TYPE in ${TEST_TYPES}
do
start_end::group_start "Running tests ${TEST_TYPE}"
@@ -170,5 +175,10 @@ do
echo "**********************************************************************************************"
run_airflow_testing_in_docker "${@}"
+
+ docker_engine::get_available_memory_in_docker
+ docker_engine::get_available_cpus_in_docker
+ docker_engine::get_available_disk_space_in_docker
+
start_end::group_end
done