[airflow] 02/04: Add "generated" folder to volumes mounted when "MOUNT_SELECTED" used (#24818)

[po...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit cd6987f7cb4d076660a867a3789815cf854486ff
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Mon Jul 4 12:36:08 2022 +0200

    Add "generated" folder to volumes mounted when "MOUNT_SELECTED" used (#24818)
    
    (cherry picked from commit 31578583124c5badcd8d3b7411315b1b02d4df68)
---
 dev/breeze/src/airflow_breeze/utils/docker_command_utils.py | 12 ++++++++++--
 scripts/ci/docker-compose/local.yml                         |  4 ++--
 scripts/ci/pre_commit/pre_commit_local_yml_mounts.py        |  9 +++++----
 3 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index 7b5e37f3b0..c49c0f183f 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -60,7 +60,14 @@ from airflow_breeze.utils.run_utils import (
     run_command,
 )
 
-NECESSARY_HOST_VOLUMES = [
+# Those are volumes that are mounted when MOUNT_SELECTED is chosen (which is the default when
+# entering Breeze. MOUNT_SELECTED prevents to mount the files that you can have accidentally added
+# in your sources (or they were added automatically by setup.py etc.) to be mounted to container.
+# This is important to get a "clean" environment for different python versions and to avoid
+# unnecessary slow-downs when you are mounting files on MacOS (which has very slow filesystem)
+# Any time you add a top-level folder in airflow that should also be added to container you should
+# add it here.
+VOLUMES_FOR_SELECTED_MOUNTS = [
     (".bash_aliases", "/root/.bash_aliases"),
     (".bash_history", "/root/.bash_history"),
     (".coveragerc", "/opt/airflow/.coveragerc"),
@@ -78,6 +85,7 @@ NECESSARY_HOST_VOLUMES = [
     ("dags", "/opt/airflow/dags"),
     ("dev", "/opt/airflow/dev"),
     ("docs", "/opt/airflow/docs"),
+    ("generated", "/opt/airflow/generated"),
     ("hooks", "/opt/airflow/hooks"),
     ("logs", "/root/airflow/logs"),
     ("pyproject.toml", "/opt/airflow/pyproject.toml"),
@@ -105,7 +113,7 @@ def get_extra_docker_flags(mount_sources: str) -> List[str]:
     if mount_sources == MOUNT_ALL:
         extra_docker_flags.extend(["--mount", f"type=bind,src={AIRFLOW_SOURCES_ROOT},dst=/opt/airflow/"])
     elif mount_sources == MOUNT_SELECTED:
-        for (src, dst) in NECESSARY_HOST_VOLUMES:
+        for (src, dst) in VOLUMES_FOR_SELECTED_MOUNTS:
             if (AIRFLOW_SOURCES_ROOT / src).exists():
                 extra_docker_flags.extend(
                     ["--mount", f'type=bind,src={AIRFLOW_SOURCES_ROOT / src},dst={dst}']
diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml
index 9e63ffa467..7df00f7462 100644
--- a/scripts/ci/docker-compose/local.yml
+++ b/scripts/ci/docker-compose/local.yml
@@ -26,7 +26,7 @@ services:
     # or those that might be useful to see in the host as output of the
     # tests (such as logs)
     volumes:
-      # START automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py
+      # START automatically generated volumes from VOLUMES_FOR_SELECTED_MOUNTS in docker_command_utils.py
       - type: bind
         source: ../../../.bash_aliases
         target: /root/.bash_aliases
@@ -117,4 +117,4 @@ services:
       - type: bind
         source: ../../../metastore_browser
         target: /opt/airflow/metastore_browser
-        # END automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py
+        # END automatically generated volumes from VOLUMES_FOR_SELECTED_MOUNTS in docker_command_utils.py
diff --git a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py
index 391214fa29..ceaf5a4db8 100755
--- a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py
+++ b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py
@@ -33,19 +33,20 @@ sys.path.append(str(AIRFLOW_SOURCES_DIR))
 
 MOUNTS_HEADER = (
     '        # START automatically generated volumes from '
-    'NECESSARY_HOST_VOLUMES in docker_command_utils.py'
+    'VOLUMES_FOR_SELECTED_MOUNTS in docker_command_utils.py'
 )
 MOUNTS_FOOTER = (
-    '        # END automatically generated volumes from ' 'NECESSARY_HOST_VOLUMES in docker_command_utils.py'
+    '        # END automatically generated volumes from '
+    'VOLUMES_FOR_SELECTED_MOUNTS in docker_command_utils.py'
 )
 
 if __name__ == '__main__':
-    from airflow_breeze.utils.docker_command_utils import NECESSARY_HOST_VOLUMES
+    from airflow_breeze.utils.docker_command_utils import VOLUMES_FOR_SELECTED_MOUNTS
 
     local_mount_file_path = AIRFLOW_SOURCES_DIR / 'scripts' / 'ci' / 'docker-compose' / 'local.yml'
     PREFIX = '      '
     volumes = []
-    for (src, dest) in NECESSARY_HOST_VOLUMES:
+    for (src, dest) in VOLUMES_FOR_SELECTED_MOUNTS:
         volumes.extend(
             [
                 PREFIX + "- type: bind\n",