You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/12/24 13:48:34 UTC
[airflow] branch main updated: Wait for asset compilation to finish before starting airflow in Breeze (#28575)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 979a72a7a3 Wait for asset compilation to finish before starting airflow in Breeze (#28575)
979a72a7a3 is described below
commit 979a72a7a3bff4e6cb8132360408584f65f2b203
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sat Dec 24 14:48:14 2022 +0100
Wait for asset compilation to finish before starting airflow in Breeze (#28575)
Asset compilation is performed in background and especially if it is run
for the first time it might take some time. At the same time database
intialization is done so usually it is not a problem but it some
cases database might be initialized faster and webserver would start
without assets compiled leading to nasty output.
This change adds waiting for the compilation - tmux will not
split screens and run webserver before it is done.
---
Dockerfile.ci | 30 ++++++++++++
dev/breeze/README.md | 2 +-
dev/breeze/setup.cfg | 1 +
.../airflow_breeze/utils/docker_command_utils.py | 1 +
dev/breeze/src/airflow_breeze/utils/path_utils.py | 3 ++
dev/breeze/src/airflow_breeze/utils/run_utils.py | 53 +++++++++++++++++-----
scripts/ci/docker-compose/local.yml | 3 ++
.../ci/pre_commit/pre_commit_compile_www_assets.py | 4 +-
.../pre_commit_compile_www_assets_dev.py | 2 +-
scripts/docker/entrypoint_ci.sh | 30 ++++++++++++
scripts/in_container/bin/run_tmux | 1 -
11 files changed, 113 insertions(+), 17 deletions(-)
diff --git a/Dockerfile.ci b/Dockerfile.ci
index a0623de882..b59bb0066c 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -598,6 +598,35 @@ export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}}
: "${AIRFLOW_SOURCES:?"ERROR: AIRFLOW_SOURCES not set !!!!"}"
+function wait_for_asset_compilation() {
+ if [[ -f "${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock" ]]; then
+ echo
+ echo "${COLOR_YELLOW}Waiting for asset compilation to complete in the background.${COLOR_RESET}"
+ echo
+ local counter=0
+ while [[ -f "${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock" ]]; do
+ echo "${COLOR_BLUE}Still waiting .....${COLOR_RESET}"
+ sleep 1
+ ((counter=counter+1))
+ if [[ ${counter} == "30" ]]; then
+ echo
+ echo "${COLOR_YELLOW}The asset compilation is taking too long.${COLOR_YELLOW}"
+ echo """
+If it does not complete soon, you might want to stop it and remove file lock:
+ * press Ctrl-C
+ * run 'rm ${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock'
+"""
+ fi
+ if [[ ${counter} == "60" ]]; then
+ echo
+ echo "${COLOR_RED}The asset compilation is taking too long. Exiting.${COLOR_RED}"
+ echo
+ exit 1
+ fi
+ done
+ fi
+}
+
if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then
@@ -785,6 +814,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then
export AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS}
export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES}
+ wait_for_asset_compilation
# shellcheck source=scripts/in_container/bin/run_tmux
exec run_tmux
fi
diff --git a/dev/breeze/README.md b/dev/breeze/README.md
index 0c86259305..50c4df000a 100644
--- a/dev/breeze/README.md
+++ b/dev/breeze/README.md
@@ -52,6 +52,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT.
---------------------------------------------------------------------------------------------------------
-Package config hash: f28f0d555b81a0f48d6b29b3cf8bba132b8c6a8f3d290a25ad4fd62019a9adbf86c0dc913c474e23ae110f3f433db0214bf46b21000f0d2bdd0884134923ae91
+Package config hash: 303fcf07d47a537ae4bf82f30328bb6a30440b11fc8e9dfa7f5188398c18103c4927129b245420ab7fd65e9bc16d359528f74e2d616f8b9f92c15e0e9a9b052c
---------------------------------------------------------------------------------------------------------
diff --git a/dev/breeze/setup.cfg b/dev/breeze/setup.cfg
index c6c6d8dee9..fe4fef0225 100644
--- a/dev/breeze/setup.cfg
+++ b/dev/breeze/setup.cfg
@@ -55,6 +55,7 @@ packages = find:
install_requires =
cached_property>=1.5.0;python_version<="3.7"
click
+ filelock
inputimeout
importlib-metadata>=4.4; python_version < "3.8"
jinja2
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index fe6239c04b..7670b5bbdd 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -74,6 +74,7 @@ from airflow_breeze.utils.run_utils import (
VOLUMES_FOR_SELECTED_MOUNTS = [
(".bash_aliases", "/root/.bash_aliases"),
(".bash_history", "/root/.bash_history"),
+ (".build", "/opt/airflow/.build"),
(".coveragerc", "/opt/airflow/.coveragerc"),
(".dockerignore", "/opt/airflow/.dockerignore"),
(".flake8", "/opt/airflow/.flake8"),
diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py
index 7138d24c36..33142072dc 100644
--- a/dev/breeze/src/airflow_breeze/utils/path_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py
@@ -263,6 +263,9 @@ def find_airflow_sources_root_to_operate_on() -> Path:
AIRFLOW_SOURCES_ROOT = find_airflow_sources_root_to_operate_on().resolve()
BUILD_CACHE_DIR = AIRFLOW_SOURCES_ROOT / ".build"
+WWW_CACHE_DIR = BUILD_CACHE_DIR / "www"
+WWW_ASSET_COMPILE_LOCK = WWW_CACHE_DIR / ".asset_compile.lock"
+WWW_ASSET_OUT_FILE = WWW_CACHE_DIR / "asset_compile.out"
DAGS_DIR = AIRFLOW_SOURCES_ROOT / "dags"
FILES_DIR = AIRFLOW_SOURCES_ROOT / "files"
HOOKS_DIR = AIRFLOW_SOURCES_ROOT / "hooks"
diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py
index 506aff693f..70cf89687e 100644
--- a/dev/breeze/src/airflow_breeze/utils/run_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py
@@ -36,7 +36,7 @@ from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
from airflow_breeze.global_constants import APACHE_AIRFLOW_GITHUB_REPOSITORY
from airflow_breeze.utils.ci_group import ci_group
from airflow_breeze.utils.console import Output, get_console
-from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
+from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, WWW_ASSET_COMPILE_LOCK, WWW_ASSET_OUT_FILE
from airflow_breeze.utils.shared_options import get_dry_run, get_verbose
RunCommandResult = Union[subprocess.CompletedProcess, subprocess.CalledProcessError]
@@ -393,16 +393,43 @@ def get_ci_image_for_pre_commits() -> str:
return airflow_image
-def _run_compile_internally(command_to_execute: list[str]):
+def _run_compile_internally(command_to_execute: list[str], dev: bool) -> RunCommandResult:
+ from filelock import SoftFileLock, Timeout
+
env = os.environ.copy()
- compile_www_assets_result = run_command(
- command_to_execute,
- check=False,
- no_output_dump_on_exception=True,
- text=True,
- env=env,
- )
- return compile_www_assets_result
+ if dev:
+ return run_command(
+ command_to_execute,
+ check=False,
+ no_output_dump_on_exception=True,
+ text=True,
+ env=env,
+ )
+ else:
+ WWW_ASSET_COMPILE_LOCK.parent.mkdir(parents=True, exist_ok=True)
+ try:
+ WWW_ASSET_COMPILE_LOCK.unlink()
+ except FileNotFoundError:
+ pass
+ try:
+ with SoftFileLock(WWW_ASSET_COMPILE_LOCK, timeout=5):
+ with open(WWW_ASSET_OUT_FILE, "w") as f:
+ return run_command(
+ command_to_execute,
+ check=False,
+ no_output_dump_on_exception=True,
+ text=True,
+ env=env,
+ stderr=subprocess.STDOUT,
+ stdout=f,
+ )
+ except Timeout:
+ get_console().print("[error]Another asset compilation is running. Exiting[/]\n")
+ get_console().print("[warning]If you are sure there is no other compilation,[/]")
+ get_console().print("[warning]Remove the lock file and re-run compilation:[/]")
+ get_console().print(WWW_ASSET_COMPILE_LOCK)
+ get_console().print()
+ sys.exit(1)
def run_compile_www_assets(
@@ -425,9 +452,11 @@ def run_compile_www_assets(
"manual",
"compile-www-assets-dev" if dev else "compile-www-assets",
"--all-files",
+ "--verbose",
]
+ get_console().print(f"[info] The output of the asset compilation is stored in: [/]{WWW_ASSET_OUT_FILE}\n")
if run_in_background:
- thread = Thread(daemon=True, target=_run_compile_internally, args=(command_to_execute,))
+ thread = Thread(daemon=True, target=_run_compile_internally, args=(command_to_execute, dev))
thread.start()
else:
- return _run_compile_internally(command_to_execute)
+ return _run_compile_internally(command_to_execute, dev)
diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml
index 0b0c8461a7..45a18acceb 100644
--- a/scripts/ci/docker-compose/local.yml
+++ b/scripts/ci/docker-compose/local.yml
@@ -33,6 +33,9 @@ services:
- type: bind
source: ../../../.bash_history
target: /root/.bash_history
+ - type: bind
+ source: ../../../.build
+ target: /opt/airflow/.build
- type: bind
source: ../../../.coveragerc
target: /opt/airflow/.coveragerc
diff --git a/scripts/ci/pre_commit/pre_commit_compile_www_assets.py b/scripts/ci/pre_commit/pre_commit_compile_www_assets.py
index 26f0fb8dc3..a2b4723b93 100755
--- a/scripts/ci/pre_commit/pre_commit_compile_www_assets.py
+++ b/scripts/ci/pre_commit/pre_commit_compile_www_assets.py
@@ -26,7 +26,7 @@ sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure common_pre
from common_precommit_utils import get_directory_hash # isort: skip # noqa
AIRFLOW_SOURCES_PATH = Path(__file__).parents[3].resolve()
-WWW_HASH_FILE = AIRFLOW_SOURCES_PATH / ".build" / "www_dir_hash.txt"
+WWW_HASH_FILE = AIRFLOW_SOURCES_PATH / ".build" / "www" / "hash.txt"
if __name__ not in ("__main__", "__mp_main__"):
raise SystemExit(
@@ -42,8 +42,8 @@ if __name__ == "__main__":
if new_hash == old_hash:
print("The WWW directory has not changed! Skip regeneration.")
sys.exit(0)
- WWW_HASH_FILE.write_text(new_hash)
env = os.environ.copy()
env["FORCE_COLOR"] = "true"
subprocess.check_call(["yarn", "install", "--frozen-lockfile"], cwd=str(www_directory))
subprocess.check_call(["yarn", "run", "build"], cwd=str(www_directory), env=env)
+ WWW_HASH_FILE.write_text(new_hash)
diff --git a/scripts/ci/pre_commit/pre_commit_compile_www_assets_dev.py b/scripts/ci/pre_commit/pre_commit_compile_www_assets_dev.py
index bcd906eb15..778e8d67d1 100755
--- a/scripts/ci/pre_commit/pre_commit_compile_www_assets_dev.py
+++ b/scripts/ci/pre_commit/pre_commit_compile_www_assets_dev.py
@@ -28,7 +28,7 @@ if __name__ not in ("__main__", "__mp_main__"):
)
AIRFLOW_SOURCES_PATH = Path(__file__).parents[3].resolve()
-WWW_HASH_FILE = AIRFLOW_SOURCES_PATH / ".build" / "www_dir_hash.txt"
+WWW_HASH_FILE = AIRFLOW_SOURCES_PATH / ".build" / "www" / "hash.txt"
if __name__ == "__main__":
www_directory = Path("airflow") / "www"
diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh
index 1a057ae75a..190995590d 100755
--- a/scripts/docker/entrypoint_ci.sh
+++ b/scripts/docker/entrypoint_ci.sh
@@ -45,6 +45,35 @@ export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}}
: "${AIRFLOW_SOURCES:?"ERROR: AIRFLOW_SOURCES not set !!!!"}"
+function wait_for_asset_compilation() {
+ if [[ -f "${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock" ]]; then
+ echo
+ echo "${COLOR_YELLOW}Waiting for asset compilation to complete in the background.${COLOR_RESET}"
+ echo
+ local counter=0
+ while [[ -f "${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock" ]]; do
+ echo "${COLOR_BLUE}Still waiting .....${COLOR_RESET}"
+ sleep 1
+ ((counter=counter+1))
+ if [[ ${counter} == "30" ]]; then
+ echo
+ echo "${COLOR_YELLOW}The asset compilation is taking too long.${COLOR_YELLOW}"
+ echo """
+If it does not complete soon, you might want to stop it and remove file lock:
+ * press Ctrl-C
+ * run 'rm ${AIRFLOW_SOURCES}/.build/www/.asset_compile.lock'
+"""
+ fi
+ if [[ ${counter} == "60" ]]; then
+ echo
+ echo "${COLOR_RED}The asset compilation is taking too long. Exiting.${COLOR_RED}"
+ echo
+ exit 1
+ fi
+ done
+ fi
+}
+
if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then
@@ -232,6 +261,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then
export AIRFLOW__DATABASE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS}
export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES}
+ wait_for_asset_compilation
# shellcheck source=scripts/in_container/bin/run_tmux
exec run_tmux
fi
diff --git a/scripts/in_container/bin/run_tmux b/scripts/in_container/bin/run_tmux
index 8811863b4c..a877411d28 100755
--- a/scripts/in_container/bin/run_tmux
+++ b/scripts/in_container/bin/run_tmux
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
if [ ! -e /usr/local/bin/stop_airflow ]; then
ln -s "/opt/airflow/scripts/in_container/stop_tmux_airflow.sh" /usr/local/bin/stop_airflow || true
fi