You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by di...@apache.org on 2020/11/20 15:40:42 UTC
[airflow] 01/01: merge invalid_kubernetes_config rule and
pod_template_file rule
This is an automated email from the ASF dual-hosted git repository.
dimberman pushed a commit to branch merge-invalid-k8s
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit c8a054792bff9d56bb27e6c6aabfb732dfd80ff8
Author: Daniel Imberman <da...@gmail.com>
AuthorDate: Fri Nov 20 07:19:10 2020 -0800
merge invalid_kubernetes_config rule and pod_template_file rule
(cherry picked from commit 5fac1e15c702faa9606c22390bf9b143193dfc79)
---
.../upgrade/rules/invalid_kubernetes_configs.py | 69 ----------------------
airflow/upgrade/rules/pod_template_file_rule.py | 38 +++++++++++-
2 files changed, 37 insertions(+), 70 deletions(-)
diff --git a/airflow/upgrade/rules/invalid_kubernetes_configs.py b/airflow/upgrade/rules/invalid_kubernetes_configs.py
deleted file mode 100644
index 0850501..0000000
--- a/airflow/upgrade/rules/invalid_kubernetes_configs.py
+++ /dev/null
@@ -1,69 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from __future__ import absolute_import
-
-from airflow.upgrade.rules.base_rule import BaseRule
-from airflow.configuration import conf
-
-invalid_keys = {"airflow_configmap",
- "airflow_local_settings_configmap",
- "dags_in_image",
- "dags_volume_subpath",
- "dags_volume_mount_point", "dags_volume_claim",
- "logs_volume_subpath", "logs_volume_claim",
- "dags_volume_host", "logs_volume_host",
- "env_from_configmap_ref", "env_from_secret_ref", "git_repo",
- "git_branch", "git_sync_depth", "git_subpath",
- "git_sync_rev", "git_user", "git_password",
- "git_sync_root", "git_sync_dest",
- "git_dags_folder_mount_point", "git_ssh_key_secret_name",
- "git_ssh_known_hosts_configmap_name", "git_sync_credentials_secret",
- "git_sync_container_repository",
- "git_sync_container_tag", "git_sync_init_container_name",
- "git_sync_run_as_user",
- "worker_service_account_name", "image_pull_secrets",
- "gcp_service_account_keys", "affinity",
- "tolerations", "run_as_user", "fs_group"}
-
-
-class InvalidKubernetesConfigRule(BaseRule):
-
- title = "Users must delete deprecated configs for KubernetesExecutor"
-
- description = """\
-In Airflow 2.0, KubernetesExecutor Users need to set a pod_template_file as a base
-value for all pods launched by the KubernetesExecutor. Many Kubernetes configs are no longer
-needed once this pod_template_file has been generated.
-"""
-
- def check(self):
- conf_dict = conf.as_dict(display_sensitive=True)
- kube_conf = conf_dict['kubernetes']
- keys = kube_conf.keys()
- resp = [k for k in keys if k in invalid_keys]
- if conf_dict['kubernetes_labels']:
- resp.append("kubernetes_labels")
- if conf_dict['kubernetes_secrets']:
- resp.append("kubernetes_secrets")
-
- if resp:
- resp_string = "\n".join(resp)
- return "The following invalid keys were found in your airflow.cfg: \
- \n\n{resp_string}\n\n \
- Please generate a pod_template_file by running `airflow generate_pod_template` \
- and delete these keys.".format(resp_string=resp_string)
diff --git a/airflow/upgrade/rules/pod_template_file_rule.py b/airflow/upgrade/rules/pod_template_file_rule.py
index 21363d3..5b9c795 100644
--- a/airflow/upgrade/rules/pod_template_file_rule.py
+++ b/airflow/upgrade/rules/pod_template_file_rule.py
@@ -20,6 +20,25 @@ from __future__ import absolute_import
from airflow.upgrade.rules.base_rule import BaseRule
from airflow.configuration import conf
+invalid_config_keys = {"airflow_configmap",
+ "airflow_local_settings_configmap",
+ "dags_in_image",
+ "dags_volume_subpath",
+ "dags_volume_mount_point", "dags_volume_claim",
+ "logs_volume_subpath", "logs_volume_claim",
+ "dags_volume_host", "logs_volume_host",
+ "env_from_configmap_ref", "env_from_secret_ref", "git_repo",
+ "git_branch", "git_sync_depth", "git_subpath",
+ "git_sync_rev", "git_user", "git_password",
+ "git_sync_root", "git_sync_dest",
+ "git_dags_folder_mount_point", "git_ssh_key_secret_name",
+ "git_ssh_known_hosts_configmap_name", "git_sync_credentials_secret",
+ "git_sync_container_repository",
+ "git_sync_container_tag", "git_sync_init_container_name",
+ "git_sync_run_as_user",
+ "worker_service_account_name", "image_pull_secrets",
+ "gcp_service_account_keys", "affinity",
+ "tolerations", "run_as_user", "fs_group"}
class PodTemplateFileRule(BaseRule):
@@ -27,7 +46,8 @@ class PodTemplateFileRule(BaseRule):
description = """\
In Airflow 2.0, KubernetesExecutor Users need to set a pod_template_file as a base
-value for all pods launched by the KubernetesExecutor
+value for all pods launched by the KubernetesExecutor. Many Kubernetes configs are no longer
+needed once this pod_template_file has been generated.
"""
def check(self):
@@ -37,3 +57,19 @@ value for all pods launched by the KubernetesExecutor
"Please create a pod_template_file by running `airflow generate_pod_template`.\n"
"This will generate a pod using your aiflow.cfg settings"
)
+
+ conf_dict = conf.as_dict(display_sensitive=True)
+ kube_conf = conf_dict['kubernetes']
+ keys = kube_conf.keys()
+ resp = [k for k in keys if k in invalid_config_keys]
+ if conf_dict['kubernetes_labels']:
+ resp.append("kubernetes_labels")
+ if conf_dict['kubernetes_secrets']:
+ resp.append("kubernetes_secrets")
+
+ if resp:
+ resp_string = "\n".join(resp)
+ return "The following invalid keys were found in your airflow.cfg: \
+ \n\n{resp_string}\n\n \
+ Now that you have a pod_template_file, these keys no longer do anything.\n\
+ Please delete these keys.".format(resp_string=resp_string)