(airflow) branch main updated: Add local airflow + configuration cleanup (#37644)

[po...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 2f4dda95a8 Add local airflow + configuration cleanup (#37644)
2f4dda95a8 is described below

commit 2f4dda95a840baa3662bada9ebd50d282092336f
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Fri Feb 23 08:05:31 2024 +0100

    Add local airflow + configuration cleanup (#37644)
    
    Our CI sometimes reuses same runner for several jobs (this is until
    we switch to --ephemeral runners) and this might lead to some
    side-effects between runners. When the two runners are run for
    different branches, and airflow is locally installed, it might
    lead to configuration being written by one version and used by
    another. This - for example - made
    https://github.com/apache/airflow/actions/runs/8011411765/job/21884630103
    to fail.
    
    This PR adds extra cleanup to `breeze ci free-space` command and
    add extra step in `breeze cleanup` to remove installed airflow
    and the whole airflow home.
---
 dev/breeze/src/airflow_breeze/commands/ci_commands.py  | 6 +++++-
 dev/breeze/src/airflow_breeze/commands/main_command.py | 9 ++++++++-
 dev/breeze/src/airflow_breeze/utils/path_utils.py      | 1 +
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/dev/breeze/src/airflow_breeze/commands/ci_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_commands.py
index 150ca16d76..12587e55d6 100644
--- a/dev/breeze/src/airflow_breeze/commands/ci_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/ci_commands.py
@@ -21,6 +21,7 @@ import json
 import os
 import platform
 import re
+import shutil
 import subprocess
 import sys
 import tempfile
@@ -54,7 +55,7 @@ from airflow_breeze.utils.docker_command_utils import (
     fix_ownership_using_docker,
     perform_environment_checks,
 )
-from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
+from airflow_breeze.utils.path_utils import AIRFLOW_HOME_DIR, AIRFLOW_SOURCES_ROOT
 from airflow_breeze.utils.run_utils import run_command
 
 
@@ -82,6 +83,9 @@ def free_space():
         run_command(["docker", "system", "prune", "--all", "--force", "--volumes"])
         run_command(["df", "-h"])
         run_command(["docker", "logout", "ghcr.io"], check=False)
+        shutil.rmtree(AIRFLOW_HOME_DIR, ignore_errors=True)
+        AIRFLOW_HOME_DIR.mkdir(exist_ok=True, parents=True)
+        run_command(["pip", "uninstall", "apache-airflow", "--yes"], check=False)
 
 
 @ci_group.command(name="resource-check", help="Check if available docker resources are enough.")
diff --git a/dev/breeze/src/airflow_breeze/commands/main_command.py b/dev/breeze/src/airflow_breeze/commands/main_command.py
index daeb1dd2fb..a3098473f3 100644
--- a/dev/breeze/src/airflow_breeze/commands/main_command.py
+++ b/dev/breeze/src/airflow_breeze/commands/main_command.py
@@ -49,7 +49,7 @@ from airflow_breeze.utils.click_utils import BreezeGroup
 from airflow_breeze.utils.confirm import Answer, user_confirm
 from airflow_breeze.utils.console import get_console
 from airflow_breeze.utils.docker_command_utils import remove_docker_networks
-from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR
+from airflow_breeze.utils.path_utils import AIRFLOW_HOME_DIR, BUILD_CACHE_DIR
 from airflow_breeze.utils.run_utils import run_command
 from airflow_breeze.utils.shared_options import get_dry_run
 
@@ -288,5 +288,12 @@ def cleanup(all: bool):
     if given_answer == Answer.YES:
         if not get_dry_run():
             shutil.rmtree(BUILD_CACHE_DIR, ignore_errors=True)
+    get_console().print("Uninstalling airflow and removing configuration")
+    given_answer = user_confirm("Are you sure with the uninstall / remove?")
+    if given_answer == Answer.YES:
+        if not get_dry_run():
+            shutil.rmtree(AIRFLOW_HOME_DIR, ignore_errors=True)
+            AIRFLOW_HOME_DIR.mkdir(exist_ok=True, parents=True)
+            run_command(["pip", "uninstall", "apache-airflow", "--yes"], check=False)
     elif given_answer == Answer.QUIT:
         sys.exit(0)
diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py
index 2ffdb6d24b..380b1a4d52 100644
--- a/dev/breeze/src/airflow_breeze/utils/path_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py
@@ -308,6 +308,7 @@ DOCKER_CONTEXT_DIR = AIRFLOW_SOURCES_ROOT / "docker-context-files"
 CACHE_TMP_FILE_DIR = tempfile.TemporaryDirectory()
 OUTPUT_LOG = Path(CACHE_TMP_FILE_DIR.name, "out.log")
 BREEZE_SOURCES_ROOT = AIRFLOW_SOURCES_ROOT / "dev" / "breeze"
+AIRFLOW_HOME_DIR = os.environ.get("AIRFLOW_HOME", Path.home() / "airflow")
 
 
 def create_volume_if_missing(volume_name: str):