[airflow] 42/44: Support rootless mode for docker. (#34537)

[ep...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

ephraimanierobi pushed a commit to branch v2-7-test
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit d2436e004685333cd395bc32a59cd13a2639838f
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Fri Sep 22 05:12:11 2023 -0400

    Support rootless mode for docker. (#34537)
    
    In case docker is run in rootless mode, the host UID is mapped to root
    user automatically and host user id is mapped to 100999 (unknown) so
    changing ownership for created files in rootless mode is problematic
    as it makes the generated files inaccessible
    
    (cherry picked from commit 0631af86525ad98e90cdc0bf120df7192ea2e912)
---
 dev/breeze/src/airflow_breeze/utils/docker_command_utils.py | 13 +++++++++++++
 scripts/ci/docker-compose/_docker.env                       |  1 +
 scripts/ci/docker-compose/base.yml                          |  1 +
 scripts/ci/docker-compose/devcontainer.env                  |  1 +
 scripts/in_container/_in_container_utils.sh                 |  4 ++++
 5 files changed, 20 insertions(+)

diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index b0a5697b18..b7b8041cb5 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -146,6 +146,16 @@ def get_extra_docker_flags(mount_sources: str, include_mypy_volume: bool = False
     return extra_docker_flags
 
 
+def is_docker_rootless():
+    response = run_command(
+        ["docker", "info", "-f", "{{println .SecurityOptions}}"], capture_output=True, check=True, text=True
+    )
+    if "rootless" in response.stdout.strip():
+        get_console().print("[info]Docker is running in rootless mode.[/]\n")
+        return True
+    return False
+
+
 def check_docker_resources(airflow_image_name: str) -> RunCommandResult:
     """
     Check if we have enough resources to run docker. This is done via running script embedded in our image.
@@ -575,6 +585,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
     set_value_to_default_if_not_set(env, "COLLECT_ONLY", "false")
     set_value_to_default_if_not_set(env, "DB_RESET", "false")
     set_value_to_default_if_not_set(env, "DEFAULT_BRANCH", AIRFLOW_BRANCH)
+    set_value_to_default_if_not_set(env, "DOCKER_IS_ROOTLESS", "false")
     set_value_to_default_if_not_set(env, "ENABLED_SYSTEMS", "")
     set_value_to_default_if_not_set(env, "ENABLE_TEST_COVERAGE", "false")
     set_value_to_default_if_not_set(env, "HELM_TEST_PACKAGE", "")
@@ -710,6 +721,8 @@ def prepare_broker_url(params, env_variables):
 def perform_environment_checks():
     check_docker_is_running()
     check_docker_version()
+    if is_docker_rootless():
+        os.environ["DOCKER_IS_ROOTLESS"] = "true"
     check_docker_compose_version()
 
 
diff --git a/scripts/ci/docker-compose/_docker.env b/scripts/ci/docker-compose/_docker.env
index aa21804e9b..f0efac7768 100644
--- a/scripts/ci/docker-compose/_docker.env
+++ b/scripts/ci/docker-compose/_docker.env
@@ -37,6 +37,7 @@ DB_RESET
 DEFAULT_BRANCH
 DEFAULT_CONSTRAINTS_BRANCH
 DEV_MODE
+DOCKER_IS_ROOTLESS
 ENABLED_SYSTEMS
 ENABLE_TEST_COVERAGE
 GITHUB_ACTIONS
diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml
index 3b7417eab5..f3b4742a31 100644
--- a/scripts/ci/docker-compose/base.yml
+++ b/scripts/ci/docker-compose/base.yml
@@ -48,6 +48,7 @@ services:
       - DEFAULT_BRANCH=${DEFAULT_BRANCH}
       - DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}
       - DEV_MODE=${DEV_MODE}
+      - DOCKER_IS_ROOTLESS=${DOCKER_IS_ROOTLESS}
       - ENABLED_SYSTEMS=${ENABLED_SYSTEMS}
       - ENABLE_TEST_COVERAGE=${ENABLE_TEST_COVERAGE}
       - GITHUB_ACTIONS=${GITHUB_ACTIONS}
diff --git a/scripts/ci/docker-compose/devcontainer.env b/scripts/ci/docker-compose/devcontainer.env
index f71a0e1e39..a297d7579c 100644
--- a/scripts/ci/docker-compose/devcontainer.env
+++ b/scripts/ci/docker-compose/devcontainer.env
@@ -35,6 +35,7 @@ DB_RESET="false"
 DEFAULT_BRANCH="main"
 DEFAULT_CONSTRAINTS_BRANCH="constraints-main"
 DEV_MODE="true"
+DOCKER_IS_ROOTLESS="false"
 ENABLED_SYSTEMS=
 ENABLE_TEST_COVERAGE="false"
 GITHUB_ACTIONS="false"
diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh
index 2ed267dd54..c962856827 100644
--- a/scripts/in_container/_in_container_utils.sh
+++ b/scripts/in_container/_in_container_utils.sh
@@ -66,6 +66,10 @@ function in_container_script_start() {
 #
 function in_container_fix_ownership() {
     if [[ ${HOST_OS:=} == "linux" ]]; then
+        if [[ ${DOCKER_IS_ROOTLESS=} == "true" ]]; then
+             echo "${COLOR_YELLOW}Skip fixing ownership of generated files: Docker is rootless${COLOR_RESET}"
+             return
+        fi
         DIRECTORIES_TO_FIX=(
             "/dist"
             "/files"