You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by di...@apache.org on 2020/11/20 15:40:41 UTC
[airflow] branch merge-invalid-k8s created (now c8a0547)
This is an automated email from the ASF dual-hosted git repository.
dimberman pushed a change to branch merge-invalid-k8s
in repository https://gitbox.apache.org/repos/asf/airflow.git.
at c8a0547 merge invalid_kubernetes_config rule and pod_template_file rule
This branch includes the following new commits:
new c8a0547 merge invalid_kubernetes_config rule and pod_template_file rule
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[airflow] 01/01: merge invalid_kubernetes_config rule and
pod_template_file rule
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimberman pushed a commit to branch merge-invalid-k8s
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit c8a054792bff9d56bb27e6c6aabfb732dfd80ff8
Author: Daniel Imberman <da...@gmail.com>
AuthorDate: Fri Nov 20 07:19:10 2020 -0800
merge invalid_kubernetes_config rule and pod_template_file rule
(cherry picked from commit 5fac1e15c702faa9606c22390bf9b143193dfc79)
---
.../upgrade/rules/invalid_kubernetes_configs.py | 69 ----------------------
airflow/upgrade/rules/pod_template_file_rule.py | 38 +++++++++++-
2 files changed, 37 insertions(+), 70 deletions(-)
diff --git a/airflow/upgrade/rules/invalid_kubernetes_configs.py b/airflow/upgrade/rules/invalid_kubernetes_configs.py
deleted file mode 100644
index 0850501..0000000
--- a/airflow/upgrade/rules/invalid_kubernetes_configs.py
+++ /dev/null
@@ -1,69 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from __future__ import absolute_import
-
-from airflow.upgrade.rules.base_rule import BaseRule
-from airflow.configuration import conf
-
-invalid_keys = {"airflow_configmap",
- "airflow_local_settings_configmap",
- "dags_in_image",
- "dags_volume_subpath",
- "dags_volume_mount_point", "dags_volume_claim",
- "logs_volume_subpath", "logs_volume_claim",
- "dags_volume_host", "logs_volume_host",
- "env_from_configmap_ref", "env_from_secret_ref", "git_repo",
- "git_branch", "git_sync_depth", "git_subpath",
- "git_sync_rev", "git_user", "git_password",
- "git_sync_root", "git_sync_dest",
- "git_dags_folder_mount_point", "git_ssh_key_secret_name",
- "git_ssh_known_hosts_configmap_name", "git_sync_credentials_secret",
- "git_sync_container_repository",
- "git_sync_container_tag", "git_sync_init_container_name",
- "git_sync_run_as_user",
- "worker_service_account_name", "image_pull_secrets",
- "gcp_service_account_keys", "affinity",
- "tolerations", "run_as_user", "fs_group"}
-
-
-class InvalidKubernetesConfigRule(BaseRule):
-
- title = "Users must delete deprecated configs for KubernetesExecutor"
-
- description = """\
-In Airflow 2.0, KubernetesExecutor Users need to set a pod_template_file as a base
-value for all pods launched by the KubernetesExecutor. Many Kubernetes configs are no longer
-needed once this pod_template_file has been generated.
-"""
-
- def check(self):
- conf_dict = conf.as_dict(display_sensitive=True)
- kube_conf = conf_dict['kubernetes']
- keys = kube_conf.keys()
- resp = [k for k in keys if k in invalid_keys]
- if conf_dict['kubernetes_labels']:
- resp.append("kubernetes_labels")
- if conf_dict['kubernetes_secrets']:
- resp.append("kubernetes_secrets")
-
- if resp:
- resp_string = "\n".join(resp)
- return "The following invalid keys were found in your airflow.cfg: \
- \n\n{resp_string}\n\n \
- Please generate a pod_template_file by running `airflow generate_pod_template` \
- and delete these keys.".format(resp_string=resp_string)
diff --git a/airflow/upgrade/rules/pod_template_file_rule.py b/airflow/upgrade/rules/pod_template_file_rule.py
index 21363d3..5b9c795 100644
--- a/airflow/upgrade/rules/pod_template_file_rule.py
+++ b/airflow/upgrade/rules/pod_template_file_rule.py
@@ -20,6 +20,25 @@ from __future__ import absolute_import
from airflow.upgrade.rules.base_rule import BaseRule
from airflow.configuration import conf
+invalid_config_keys = {"airflow_configmap",
+ "airflow_local_settings_configmap",
+ "dags_in_image",
+ "dags_volume_subpath",
+ "dags_volume_mount_point", "dags_volume_claim",
+ "logs_volume_subpath", "logs_volume_claim",
+ "dags_volume_host", "logs_volume_host",
+ "env_from_configmap_ref", "env_from_secret_ref", "git_repo",
+ "git_branch", "git_sync_depth", "git_subpath",
+ "git_sync_rev", "git_user", "git_password",
+ "git_sync_root", "git_sync_dest",
+ "git_dags_folder_mount_point", "git_ssh_key_secret_name",
+ "git_ssh_known_hosts_configmap_name", "git_sync_credentials_secret",
+ "git_sync_container_repository",
+ "git_sync_container_tag", "git_sync_init_container_name",
+ "git_sync_run_as_user",
+ "worker_service_account_name", "image_pull_secrets",
+ "gcp_service_account_keys", "affinity",
+ "tolerations", "run_as_user", "fs_group"}
class PodTemplateFileRule(BaseRule):
@@ -27,7 +46,8 @@ class PodTemplateFileRule(BaseRule):
description = """\
In Airflow 2.0, KubernetesExecutor Users need to set a pod_template_file as a base
-value for all pods launched by the KubernetesExecutor
+value for all pods launched by the KubernetesExecutor. Many Kubernetes configs are no longer
+needed once this pod_template_file has been generated.
"""
def check(self):
@@ -37,3 +57,19 @@ value for all pods launched by the KubernetesExecutor
"Please create a pod_template_file by running `airflow generate_pod_template`.\n"
"This will generate a pod using your aiflow.cfg settings"
)
+
+ conf_dict = conf.as_dict(display_sensitive=True)
+ kube_conf = conf_dict['kubernetes']
+ keys = kube_conf.keys()
+ resp = [k for k in keys if k in invalid_config_keys]
+ if conf_dict['kubernetes_labels']:
+ resp.append("kubernetes_labels")
+ if conf_dict['kubernetes_secrets']:
+ resp.append("kubernetes_secrets")
+
+ if resp:
+ resp_string = "\n".join(resp)
+ return "The following invalid keys were found in your airflow.cfg: \
+ \n\n{resp_string}\n\n \
+ Now that you have a pod_template_file, these keys no longer do anything.\n\
+ Please delete these keys.".format(resp_string=resp_string)