[airflow] branch main updated: Mark Airflow directory in container as safe for git commands (#29386)

[po...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 2e1635a9c1 Mark Airflow directory in container as safe for git commands (#29386)
2e1635a9c1 is described below

commit 2e1635a9c1d6faaa9d5cd5cfd1a364091cd62bde
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Mon Feb 6 17:02:08 2023 +0100

    Mark Airflow directory in container as safe for git commands (#29386)
    
    There is a new setting/version of git in GitHub Actions that started
    checking the ownership of the Git repository. Since in case of the
    provider commands we run them inside docker image as root user
    (this is in order to isolate the provider package building from
    the main CI environment), the owner of such directory is different
    (runner user) than the user that runs the git command (root).
    
    This change marks the current git directory for such commands as
    safe, regardles from the discrepancy.
    
    This config is global and run inside the image, so it is safe to
    leave it after methods complete as containers are torn-down after
    completing package preparation.
    
    This PR also improves diagnostics. Previously the `git remote add`
    output was redirected to dev null as there was no way it could fail,
    but this turned to be false - the output of the `git remote add`
    commnd is now also printed for diagnostics.
---
 dev/provider_packages/prepare_provider_packages.py | 32 ++++++++++++++++++++--
 .../in_container/run_prepare_airflow_packages.sh   |  7 +++++
 2 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py
index 1d4ec06e75..e14238e529 100755
--- a/dev/provider_packages/prepare_provider_packages.py
+++ b/dev/provider_packages/prepare_provider_packages.py
@@ -670,6 +670,31 @@ def get_cross_provider_dependent_packages(provider_package_id: str) -> list[str]
     return ALL_DEPENDENCIES[provider_package_id][CROSS_PROVIDERS_DEPS]
 
 
+def make_current_directory_safe(verbose: bool):
+    """
+    Makes current directory safe for Git.
+
+    New git checks if git ownership for the folder is not manipulated with. We are running this command
+    only inside the container where the directory is mounted from "regular" user to "root" user which is
+    used inside the container, so this is quite ok to assume the directory it is used is safe.
+
+    It's also ok to leave it as safe - it is a global option inside the container so it will disappear
+    when we exit.
+
+    :param verbose: whether to print commands being executed
+    :return:
+    """
+    safe_dir_remove_command = ["git", "config", "--global", "--unset-all", "safe.directory"]
+    if verbose:
+        console.print(f"Running command: '{' '.join(safe_dir_remove_command)}'")
+    # we ignore result of this call
+    subprocess.call(safe_dir_remove_command)
+    safe_dir_add_command = ["git", "config", "--global", "--add", "safe.directory", "/opt/airflow"]
+    if verbose:
+        console.print(f"Running command: '{' '.join(safe_dir_add_command)}'")
+    subprocess.check_call(safe_dir_add_command)
+
+
 def make_sure_remote_apache_exists_and_fetch(git_update: bool, verbose: bool):
     """
     Make sure that apache remote exist in git. We need to take a log from the apache
@@ -678,6 +703,7 @@ def make_sure_remote_apache_exists_and_fetch(git_update: bool, verbose: bool):
     Also, the local repo might be shallow, so we need to un-shallow it.
 
     This will:
+    * mark current directory as safe for ownership (it is run in the container)
     * check if the remote exists and add if it does not
     * check if the local repo is shallow, mark it to un-shallow in this case
     * fetch from the remote including all tags and overriding local tags in case they are set differently
@@ -685,6 +711,8 @@ def make_sure_remote_apache_exists_and_fetch(git_update: bool, verbose: bool):
     :param git_update: If the git remote already exists, should we try to update it
     :param verbose: print verbose messages while fetching
     """
+
+    make_current_directory_safe(verbose)
     try:
         check_remote_command = ["git", "remote", "get-url", HTTPS_REMOTE]
         if verbose:
@@ -710,10 +738,8 @@ def make_sure_remote_apache_exists_and_fetch(git_update: bool, verbose: bool):
             if verbose:
                 console.print(f"Running command: '{' '.join(remote_add_command)}'")
             try:
-                subprocess.check_output(
+                subprocess.check_call(
                     remote_add_command,
-                    stderr=subprocess.STDOUT,
-                    text=True,
                 )
             except subprocess.CalledProcessError as ex:
                 console.print("[red]Error: when adding remote:[/]", ex)
diff --git a/scripts/in_container/run_prepare_airflow_packages.sh b/scripts/in_container/run_prepare_airflow_packages.sh
index 68f9dc92be..afcceedbf0 100755
--- a/scripts/in_container/run_prepare_airflow_packages.sh
+++ b/scripts/in_container/run_prepare_airflow_packages.sh
@@ -78,8 +78,15 @@ function prepare_airflow_packages() {
     echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
 }
 
+function mark_directory_as_safe() {
+    git config --global --unset-all safe.directory || true
+    git config --global --add safe.directory /opt/airflow
+}
+
 install_supported_pip_version
 
+mark_directory_as_safe
+
 prepare_airflow_packages
 
 echo