You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/06/09 09:04:31 UTC
[airflow] 18/36: Enable configurable git sync depth (#9094)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 64194eec4efdcbdc151e47979f6ae64ed4820ed8
Author: mgorsk1 <go...@gmail.com>
AuthorDate: Tue Jun 2 06:36:03 2020 +0000
Enable configurable git sync depth (#9094)
Enable configurable git sync depth
(cherry picked from commit 93e924d3fab6137fda097716a9078705621eb358)
---
airflow/config_templates/config.yml | 8 ++++++++
airflow/config_templates/default_airflow.cfg | 4 ++++
airflow/executors/kubernetes_executor.py | 2 ++
airflow/kubernetes/worker_configuration.py | 2 +-
.../kubernetes/app/templates/configmaps.template.yaml | 1 +
5 files changed, 16 insertions(+), 1 deletion(-)
diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml
index 26cfae8..a7fa7a6 100644
--- a/airflow/config_templates/config.yml
+++ b/airflow/config_templates/config.yml
@@ -1893,6 +1893,14 @@
type: string
example: ~
default: ""
+ - name: git_sync_depth
+ description: |
+ Use a shallow clone with a history truncated to the specified number of commits.
+ 0 - do not use shallow clone.
+ version_added: 1.10.11
+ type: string
+ example: ~
+ default: "1"
- name: git_subpath
description: ~
version_added: ~
diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg
index 5f240de..27fe92a 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -881,6 +881,10 @@ env_from_secret_ref =
# Git credentials and repository for DAGs mounted via Git (mutually exclusive with volume claim)
git_repo =
git_branch =
+
+# Use a shallow clone with a history truncated to the specified number of commits.
+# 0 - do not use shallow clone.
+git_sync_depth = 1
git_subpath =
# The specific rev or hash the git_sync init container will checkout
diff --git a/airflow/executors/kubernetes_executor.py b/airflow/executors/kubernetes_executor.py
index 6944274..cf40e43 100644
--- a/airflow/executors/kubernetes_executor.py
+++ b/airflow/executors/kubernetes_executor.py
@@ -101,6 +101,8 @@ class KubeConfig:
self.git_repo = conf.get(self.kubernetes_section, 'git_repo')
# The branch of the repository to be checked out
self.git_branch = conf.get(self.kubernetes_section, 'git_branch')
+ # Clone depth for git sync
+ self.git_sync_depth = conf.get(self.kubernetes_section, 'git_sync_depth')
# Optionally, the directory in the git repository containing the dags
self.git_subpath = conf.get(self.kubernetes_section, 'git_subpath')
# Optionally, the root directory for git operations
diff --git a/airflow/kubernetes/worker_configuration.py b/airflow/kubernetes/worker_configuration.py
index 41a65b3..bed1ac2 100644
--- a/airflow/kubernetes/worker_configuration.py
+++ b/airflow/kubernetes/worker_configuration.py
@@ -68,7 +68,7 @@ class WorkerConfiguration(LoggingMixin):
value=self.kube_config.git_sync_dest
), k8s.V1EnvVar(
name='GIT_SYNC_DEPTH',
- value='1'
+ value=self.kube_config.git_sync_depth
), k8s.V1EnvVar(
name='GIT_SYNC_ONE_TIME',
value='true'
diff --git a/scripts/ci/in_container/kubernetes/app/templates/configmaps.template.yaml b/scripts/ci/in_container/kubernetes/app/templates/configmaps.template.yaml
index 868da7c..bb942bc 100644
--- a/scripts/ci/in_container/kubernetes/app/templates/configmaps.template.yaml
+++ b/scripts/ci/in_container/kubernetes/app/templates/configmaps.template.yaml
@@ -185,6 +185,7 @@ data:
dags_in_image = False
git_repo = https://github.com/{{CONFIGMAP_GIT_REPO}}.git
git_branch = {{CONFIGMAP_BRANCH}}
+ git_sync_depth = "1"
git_subpath = airflow/contrib/example_dags/
git_user =
git_password =