You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ds...@apache.org on 2022/10/12 21:14:32 UTC
[airflow] branch main updated: Rename kubernetes config section to kubernetes_executor (#26873)
This is an automated email from the ASF dual-hosted git repository.
dstandish pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 21e550ea71 Rename kubernetes config section to kubernetes_executor (#26873)
21e550ea71 is described below
commit 21e550ea71509f8d23bc2269c58345561674f9c2
Author: Daniel Standish <15...@users.noreply.github.com>
AuthorDate: Wed Oct 12 14:14:13 2022 -0700
Rename kubernetes config section to kubernetes_executor (#26873)
Now that KPO does not consider any core k8s config params, this section truly is just about kubernetes executor. Renaming it reduces potential for confusion.
---
airflow/cli/cli_parser.py | 2 +-
airflow/config_templates/config.yml | 2 +-
airflow/config_templates/default_airflow.cfg | 2 +-
airflow/configuration.py | 27 ++++++++++++++++++++++
.../example_dags/example_kubernetes_executor.py | 4 ++--
.../example_local_kubernetes_executor.py | 4 ++--
airflow/kubernetes/kube_client.py | 16 ++++++-------
airflow/kubernetes/kube_config.py | 2 +-
airflow/utils/log/file_task_handler.py | 4 ++--
docs/apache-airflow/executor/kubernetes.rst | 2 +-
docs/apache-airflow/upgrading-from-1-10/index.rst | 5 +++-
newsfragments/26873.significant.rst | 3 +++
tests/core/test_config_templates.py | 2 +-
tests/kubernetes/test_client.py | 2 +-
14 files changed, 55 insertions(+), 22 deletions(-)
diff --git a/airflow/cli/cli_parser.py b/airflow/cli/cli_parser.py
index 301166a6e6..bf08fe2d74 100644
--- a/airflow/cli/cli_parser.py
+++ b/airflow/cli/cli_parser.py
@@ -889,7 +889,7 @@ ARG_OPTION = Arg(
# kubernetes cleanup-pods
ARG_NAMESPACE = Arg(
("--namespace",),
- default=conf.get('kubernetes', 'namespace'),
+ default=conf.get('kubernetes_executor', 'namespace'),
help="Kubernetes Namespace. Default value is `[kubernetes] namespace` in configuration.",
)
diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml
index b20fea7ff2..cea65765ae 100644
--- a/airflow/config_templates/config.yml
+++ b/airflow/config_templates/config.yml
@@ -2311,7 +2311,7 @@
type: string
example: ~
default: "True"
-- name: kubernetes
+- name: kubernetes_executor
description: ~
options:
- name: pod_template_file
diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg
index 174f33dbc8..a4c1c2626a 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -1153,7 +1153,7 @@ offset_field = offset
use_ssl = False
verify_certs = True
-[kubernetes]
+[kubernetes_executor]
# Path to the YAML pod file that forms the basis for KubernetesExecutor workers.
pod_template_file =
diff --git a/airflow/configuration.py b/airflow/configuration.py
index 905564ac8d..f3e1a01464 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -227,6 +227,33 @@ class AirflowConfigParser(ConfigParser):
('database', 'sql_alchemy_connect_args'): ('core', 'sql_alchemy_connect_args', '2.3.0'),
('database', 'load_default_connections'): ('core', 'load_default_connections', '2.3.0'),
('database', 'max_db_retries'): ('core', 'max_db_retries', '2.3.0'),
+ **{
+ ('kubernetes_executor', x): ('kubernetes', x, '2.4.2')
+ for x in (
+ 'pod_template_file',
+ 'worker_container_repository',
+ 'worker_container_tag',
+ 'namespace',
+ 'delete_worker_pods',
+ 'delete_worker_pods_on_failure',
+ 'worker_pods_creation_batch_size',
+ 'multi_namespace_mode',
+ 'in_cluster',
+ 'cluster_context',
+ 'config_file',
+ 'kube_client_request_args',
+ 'delete_option_kwargs',
+ 'enable_tcp_keepalive',
+ 'tcp_keep_idle',
+ 'tcp_keep_intvl',
+ 'tcp_keep_cnt',
+ 'verify_ssl',
+ 'worker_pods_pending_timeout',
+ 'worker_pods_pending_timeout_check_interval',
+ 'worker_pods_queued_check_interval',
+ 'worker_pods_pending_timeout_batch_size',
+ )
+ },
}
# A mapping of old default values that we want to change and warn the user
diff --git a/airflow/example_dags/example_kubernetes_executor.py b/airflow/example_dags/example_kubernetes_executor.py
index 81e335c54a..49b2101321 100644
--- a/airflow/example_dags/example_kubernetes_executor.py
+++ b/airflow/example_dags/example_kubernetes_executor.py
@@ -32,8 +32,8 @@ from airflow.example_dags.libs.helper import print_stuff
log = logging.getLogger(__name__)
-worker_container_repository = conf.get('kubernetes', 'worker_container_repository')
-worker_container_tag = conf.get('kubernetes', 'worker_container_tag')
+worker_container_repository = conf.get('kubernetes_executor', 'worker_container_repository')
+worker_container_tag = conf.get('kubernetes_executor', 'worker_container_tag')
try:
from kubernetes.client import models as k8s
diff --git a/airflow/example_dags/example_local_kubernetes_executor.py b/airflow/example_dags/example_local_kubernetes_executor.py
index cd1fd6a0c0..420b277c02 100644
--- a/airflow/example_dags/example_local_kubernetes_executor.py
+++ b/airflow/example_dags/example_local_kubernetes_executor.py
@@ -30,8 +30,8 @@ from airflow.example_dags.libs.helper import print_stuff
log = logging.getLogger(__name__)
-worker_container_repository = conf.get('kubernetes', 'worker_container_repository')
-worker_container_tag = conf.get('kubernetes', 'worker_container_tag')
+worker_container_repository = conf.get('kubernetes_executor', 'worker_container_repository')
+worker_container_tag = conf.get('kubernetes_executor', 'worker_container_tag')
try:
from kubernetes.client import models as k8s
diff --git a/airflow/kubernetes/kube_client.py b/airflow/kubernetes/kube_client.py
index 76f0b26f01..2719e15e7e 100644
--- a/airflow/kubernetes/kube_client.py
+++ b/airflow/kubernetes/kube_client.py
@@ -59,9 +59,9 @@ def _enable_tcp_keepalive() -> None:
from urllib3.connection import HTTPConnection, HTTPSConnection
- tcp_keep_idle = conf.getint('kubernetes', 'tcp_keep_idle')
- tcp_keep_intvl = conf.getint('kubernetes', 'tcp_keep_intvl')
- tcp_keep_cnt = conf.getint('kubernetes', 'tcp_keep_cnt')
+ tcp_keep_idle = conf.getint('kubernetes_executor', 'tcp_keep_idle')
+ tcp_keep_intvl = conf.getint('kubernetes_executor', 'tcp_keep_intvl')
+ tcp_keep_cnt = conf.getint('kubernetes_executor', 'tcp_keep_cnt')
socket_options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
@@ -85,7 +85,7 @@ def _enable_tcp_keepalive() -> None:
def get_kube_client(
- in_cluster: bool = conf.getboolean('kubernetes', 'in_cluster'),
+ in_cluster: bool = conf.getboolean('kubernetes_executor', 'in_cluster'),
cluster_context: str | None = None,
config_file: str | None = None,
) -> client.CoreV1Api:
@@ -101,19 +101,19 @@ def get_kube_client(
if not has_kubernetes:
raise _import_err
- if conf.getboolean('kubernetes', 'enable_tcp_keepalive'):
+ if conf.getboolean('kubernetes_executor', 'enable_tcp_keepalive'):
_enable_tcp_keepalive()
if in_cluster:
config.load_incluster_config()
else:
if cluster_context is None:
- cluster_context = conf.get('kubernetes', 'cluster_context', fallback=None)
+ cluster_context = conf.get('kubernetes_executor', 'cluster_context', fallback=None)
if config_file is None:
- config_file = conf.get('kubernetes', 'config_file', fallback=None)
+ config_file = conf.get('kubernetes_executor', 'config_file', fallback=None)
config.load_kube_config(config_file=config_file, context=cluster_context)
- if not conf.getboolean('kubernetes', 'verify_ssl'):
+ if not conf.getboolean('kubernetes_executor', 'verify_ssl'):
_disable_verify_ssl()
return client.CoreV1Api()
diff --git a/airflow/kubernetes/kube_config.py b/airflow/kubernetes/kube_config.py
index 322a6a9a77..e61657fa18 100644
--- a/airflow/kubernetes/kube_config.py
+++ b/airflow/kubernetes/kube_config.py
@@ -25,7 +25,7 @@ class KubeConfig:
"""Configuration for Kubernetes"""
core_section = 'core'
- kubernetes_section = 'kubernetes'
+ kubernetes_section = 'kubernetes_executor'
logging_section = 'logging'
def __init__(self):
diff --git a/airflow/utils/log/file_task_handler.py b/airflow/utils/log/file_task_handler.py
index 1c9c559bd1..0b4763bfa4 100644
--- a/airflow/utils/log/file_task_handler.py
+++ b/airflow/utils/log/file_task_handler.py
@@ -184,7 +184,7 @@ class FileTaskHandler(logging.Handler):
# Kubernetes takes the pod name and truncates it for the hostname. This truncated hostname
# is returned for the fqdn to comply with the 63 character limit imposed by DNS standards
# on any label of a FQDN.
- pod_list = kube_client.list_namespaced_pod(conf.get('kubernetes', 'namespace'))
+ pod_list = kube_client.list_namespaced_pod(conf.get('kubernetes_executor', 'namespace'))
matches = [
pod.metadata.name
for pod in pod_list.items
@@ -198,7 +198,7 @@ class FileTaskHandler(logging.Handler):
res = kube_client.read_namespaced_pod_log(
name=ti.hostname,
- namespace=conf.get('kubernetes', 'namespace'),
+ namespace=conf.get('kubernetes_executor', 'namespace'),
container='base',
follow=False,
tail_lines=100,
diff --git a/docs/apache-airflow/executor/kubernetes.rst b/docs/apache-airflow/executor/kubernetes.rst
index e24582c7df..429fb4066e 100644
--- a/docs/apache-airflow/executor/kubernetes.rst
+++ b/docs/apache-airflow/executor/kubernetes.rst
@@ -59,7 +59,7 @@ pod_template_file
~~~~~~~~~~~~~~~~~
To customize the pod used for k8s executor worker processes, you may create a pod template file. You must provide
-the path to the template file in the ``pod_template_file`` option in the ``kubernetes`` section of ``airflow.cfg``.
+the path to the template file in the ``pod_template_file`` option in the ``kubernetes_executor`` section of ``airflow.cfg``.
Airflow has two strict requirements for pod template files: base image and pod name.
diff --git a/docs/apache-airflow/upgrading-from-1-10/index.rst b/docs/apache-airflow/upgrading-from-1-10/index.rst
index c6bc04c83d..081a902d19 100644
--- a/docs/apache-airflow/upgrading-from-1-10/index.rst
+++ b/docs/apache-airflow/upgrading-from-1-10/index.rst
@@ -73,9 +73,12 @@ simply run the following command:
airflow generate_pod_template -o <output file path>
-Once you have performed this step, simply write out the file path to this file in the ``pod_template_file`` config of the ``kubernetes``
+Once you have performed this step, simply write out the file path to this file in the ``pod_template_file`` config of the ``kubernetes_executor``
section of your ``airflow.cfg``
+.. note::
+ Prior to airflow version 2.4.2, the ``kubernetes_executor`` section was called ``kubernetes``.
+
Step 3: Run the Upgrade check scripts
'''''''''''''''''''''''''''''''''''''
diff --git a/newsfragments/26873.significant.rst b/newsfragments/26873.significant.rst
new file mode 100644
index 0000000000..15d920a2f5
--- /dev/null
+++ b/newsfragments/26873.significant.rst
@@ -0,0 +1,3 @@
+Airflow config section ``kubernetes`` renamed to ``kubernetes_executor``
+
+KubernetesPodOperator no longer considers any core kubernetes config params, so this section now only applies to kubernetes executor. Renaming it reduces potential for confusion.
diff --git a/tests/core/test_config_templates.py b/tests/core/test_config_templates.py
index ecef25a4a4..29528cf257 100644
--- a/tests/core/test_config_templates.py
+++ b/tests/core/test_config_templates.py
@@ -52,7 +52,7 @@ DEFAULT_AIRFLOW_SECTIONS = [
'kerberos',
'elasticsearch',
'elasticsearch_configs',
- 'kubernetes',
+ 'kubernetes_executor',
'sensors',
]
diff --git a/tests/kubernetes/test_client.py b/tests/kubernetes/test_client.py
index 13fa8a3417..16cfc3ca7d 100644
--- a/tests/kubernetes/test_client.py
+++ b/tests/kubernetes/test_client.py
@@ -43,7 +43,7 @@ class TestClient:
def test_load_config_disable_ssl(self, conf, config):
conf.getboolean.return_value = False
get_kube_client(in_cluster=False)
- conf.getboolean.assert_called_with('kubernetes', 'verify_ssl')
+ conf.getboolean.assert_called_with('kubernetes_executor', 'verify_ssl')
# Support wide range of kube client libraries
if hasattr(Configuration, 'get_default_copy'):
configuration = Configuration.get_default_copy()