You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2021/01/27 12:06:51 UTC
[airflow] branch master updated: Add deprecated config options to
docs (#13883)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 65e49fc Add deprecated config options to docs (#13883)
65e49fc is described below
commit 65e49fc56f32b3e815fdf4a17be6b4e1c1e43c11
Author: Vladimir Mikhaylov <38...@users.noreply.github.com>
AuthorDate: Wed Jan 27 15:06:35 2021 +0300
Add deprecated config options to docs (#13883)
closes: #12772
---
airflow/configuration.py | 68 ++++++++++++++++--------------
docs/apache-airflow/configurations-ref.rst | 14 ++++++
docs/conf.py | 13 ++++--
tests/core/test_configuration.py | 6 +--
4 files changed, 64 insertions(+), 37 deletions(-)
diff --git a/airflow/configuration.py b/airflow/configuration.py
index 30acb35..5b765de 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -132,39 +132,43 @@ class AirflowConfigParser(ConfigParser): # pylint: disable=too-many-ancestors
('webserver', 'secret_key'),
}
- # A mapping of (new option -> old option). where option is a tuple of section name and key.
+ # A mapping of (new section, new option) -> (old section, old option, since_version).
# When reading new option, the old option will be checked to see if it exists. If it does a
# DeprecationWarning will be issued and the old option will be used instead
deprecated_options = {
- ('celery', 'worker_precheck'): ('core', 'worker_precheck'),
- ('logging', 'base_log_folder'): ('core', 'base_log_folder'),
- ('logging', 'remote_logging'): ('core', 'remote_logging'),
- ('logging', 'remote_log_conn_id'): ('core', 'remote_log_conn_id'),
- ('logging', 'remote_base_log_folder'): ('core', 'remote_base_log_folder'),
- ('logging', 'encrypt_s3_logs'): ('core', 'encrypt_s3_logs'),
- ('logging', 'logging_level'): ('core', 'logging_level'),
- ('logging', 'fab_logging_level'): ('core', 'fab_logging_level'),
- ('logging', 'logging_config_class'): ('core', 'logging_config_class'),
- ('logging', 'colored_console_log'): ('core', 'colored_console_log'),
- ('logging', 'colored_log_format'): ('core', 'colored_log_format'),
- ('logging', 'colored_formatter_class'): ('core', 'colored_formatter_class'),
- ('logging', 'log_format'): ('core', 'log_format'),
- ('logging', 'simple_log_format'): ('core', 'simple_log_format'),
- ('logging', 'task_log_prefix_template'): ('core', 'task_log_prefix_template'),
- ('logging', 'log_filename_template'): ('core', 'log_filename_template'),
- ('logging', 'log_processor_filename_template'): ('core', 'log_processor_filename_template'),
- ('logging', 'dag_processor_manager_log_location'): ('core', 'dag_processor_manager_log_location'),
- ('logging', 'task_log_reader'): ('core', 'task_log_reader'),
- ('metrics', 'statsd_on'): ('scheduler', 'statsd_on'),
- ('metrics', 'statsd_host'): ('scheduler', 'statsd_host'),
- ('metrics', 'statsd_port'): ('scheduler', 'statsd_port'),
- ('metrics', 'statsd_prefix'): ('scheduler', 'statsd_prefix'),
- ('metrics', 'statsd_allow_list'): ('scheduler', 'statsd_allow_list'),
- ('metrics', 'stat_name_handler'): ('scheduler', 'stat_name_handler'),
- ('metrics', 'statsd_datadog_enabled'): ('scheduler', 'statsd_datadog_enabled'),
- ('metrics', 'statsd_datadog_tags'): ('scheduler', 'statsd_datadog_tags'),
- ('metrics', 'statsd_custom_client_path'): ('scheduler', 'statsd_custom_client_path'),
- ('scheduler', 'parsing_processes'): ('scheduler', 'max_threads'),
+ ('celery', 'worker_precheck'): ('core', 'worker_precheck', '2.0.0'),
+ ('logging', 'base_log_folder'): ('core', 'base_log_folder', '2.0.0'),
+ ('logging', 'remote_logging'): ('core', 'remote_logging', '2.0.0'),
+ ('logging', 'remote_log_conn_id'): ('core', 'remote_log_conn_id', '2.0.0'),
+ ('logging', 'remote_base_log_folder'): ('core', 'remote_base_log_folder', '2.0.0'),
+ ('logging', 'encrypt_s3_logs'): ('core', 'encrypt_s3_logs', '2.0.0'),
+ ('logging', 'logging_level'): ('core', 'logging_level', '2.0.0'),
+ ('logging', 'fab_logging_level'): ('core', 'fab_logging_level', '2.0.0'),
+ ('logging', 'logging_config_class'): ('core', 'logging_config_class', '2.0.0'),
+ ('logging', 'colored_console_log'): ('core', 'colored_console_log', '2.0.0'),
+ ('logging', 'colored_log_format'): ('core', 'colored_log_format', '2.0.0'),
+ ('logging', 'colored_formatter_class'): ('core', 'colored_formatter_class', '2.0.0'),
+ ('logging', 'log_format'): ('core', 'log_format', '2.0.0'),
+ ('logging', 'simple_log_format'): ('core', 'simple_log_format', '2.0.0'),
+ ('logging', 'task_log_prefix_template'): ('core', 'task_log_prefix_template', '2.0.0'),
+ ('logging', 'log_filename_template'): ('core', 'log_filename_template', '2.0.0'),
+ ('logging', 'log_processor_filename_template'): ('core', 'log_processor_filename_template', '2.0.0'),
+ ('logging', 'dag_processor_manager_log_location'): (
+ 'core',
+ 'dag_processor_manager_log_location',
+ '2.0.0',
+ ),
+ ('logging', 'task_log_reader'): ('core', 'task_log_reader', '2.0.0'),
+ ('metrics', 'statsd_on'): ('scheduler', 'statsd_on', '2.0.0'),
+ ('metrics', 'statsd_host'): ('scheduler', 'statsd_host', '2.0.0'),
+ ('metrics', 'statsd_port'): ('scheduler', 'statsd_port', '2.0.0'),
+ ('metrics', 'statsd_prefix'): ('scheduler', 'statsd_prefix', '2.0.0'),
+ ('metrics', 'statsd_allow_list'): ('scheduler', 'statsd_allow_list', '2.0.0'),
+ ('metrics', 'stat_name_handler'): ('scheduler', 'stat_name_handler', '2.0.0'),
+ ('metrics', 'statsd_datadog_enabled'): ('scheduler', 'statsd_datadog_enabled', '2.0.0'),
+ ('metrics', 'statsd_datadog_tags'): ('scheduler', 'statsd_datadog_tags', '2.0.0'),
+ ('metrics', 'statsd_custom_client_path'): ('scheduler', 'statsd_custom_client_path', '2.0.0'),
+ ('scheduler', 'parsing_processes'): ('scheduler', 'max_threads', '1.10.14'),
}
# A mapping of old default values that we want to change and warn the user
@@ -321,7 +325,9 @@ class AirflowConfigParser(ConfigParser): # pylint: disable=too-many-ancestors
section = str(section).lower()
key = str(key).lower()
- deprecated_section, deprecated_key = self.deprecated_options.get((section, key), (None, None))
+ deprecated_section, deprecated_key, _ = self.deprecated_options.get(
+ (section, key), (None, None, None)
+ )
option = self._get_environment_variables(deprecated_key, deprecated_section, key, section)
if option is not None:
diff --git a/docs/apache-airflow/configurations-ref.rst b/docs/apache-airflow/configurations-ref.rst
index c92a205..d9f7690 100644
--- a/docs/apache-airflow/configurations-ref.rst
+++ b/docs/apache-airflow/configurations-ref.rst
@@ -79,4 +79,18 @@ can set in ``airflow.cfg`` file or using environment variables.
{% endif %}
{% endfor %}
+
+ {% if section["name"] in deprecated_options %}
+
+ {% for deprecated_option_name, (new_section_name, new_option_name, since_version) in deprecated_options[section["name"]].items() %}
+ .. _config:{{ section["name"] }}__{{ deprecated_option_name }}:
+
+ {{ deprecated_option_name }} (Deprecated)
+ {{ "-" * (deprecated_option_name + " (Deprecated)")|length }}
+
+ .. deprecated:: {{ since_version }}
+ The option has been moved to :ref:`{{ new_section_name }}.{{ new_option_name }} <config:{{ new_section_name }}__{{ new_option_name }}>`
+ {% endfor %}
+ {% endif %}
+
{% endfor %}
diff --git a/docs/conf.py b/docs/conf.py
index 188a4b6..411796c 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -34,12 +34,13 @@
import glob
import os
import sys
-from typing import Any, Dict, List, Optional
+from collections import defaultdict
+from typing import Any, Dict, List, Optional, Tuple
import yaml
import airflow
-from airflow.configuration import default_config_yaml
+from airflow.configuration import AirflowConfigParser, default_config_yaml
from docs.exts.docs_build.third_party_inventories import ( # pylint: disable=no-name-in-module,wrong-import-order
THIRD_PARTY_INDEXES,
)
@@ -308,8 +309,14 @@ html_context = {
# Jinja context
if PACKAGE_NAME == 'apache-airflow':
+ deprecated_options: Dict[str, Dict[str, Tuple[str, str, str]]] = defaultdict(dict)
+ for (section, key), (
+ (deprecated_section, deprecated_key, since_version)
+ ) in AirflowConfigParser.deprecated_options.items():
+ deprecated_options[deprecated_section][deprecated_key] = section, key, since_version
+
jinja_contexts = {
- 'config_ctx': {"configs": default_config_yaml()},
+ 'config_ctx': {"configs": default_config_yaml(), "deprecated_options": deprecated_options},
'quick_start_ctx': {
'doc_root_url': (f'https://airflow.apache.org/docs/apache-airflow/{PACKAGE_VERSION}/')
if FOR_PRODUCTION
diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py
index a279837..4e32c5e 100644
--- a/tests/core/test_configuration.py
+++ b/tests/core/test_configuration.py
@@ -424,7 +424,7 @@ AIRFLOW_HOME = /root/airflow
# Guarantee we have a deprecated setting, so we test the deprecation
# lookup even if we remove this explicit fallback
conf.deprecated_options = {
- ('celery', 'worker_concurrency'): ('celery', 'celeryd_concurrency'),
+ ('celery', 'worker_concurrency'): ('celery', 'celeryd_concurrency', '2.0.0'),
}
# Remove it so we are sure we use the right setting
@@ -447,7 +447,7 @@ AIRFLOW_HOME = /root/airflow
# Guarantee we have a deprecated setting, so we test the deprecation
# lookup even if we remove this explicit fallback
conf.deprecated_options = {
- ('logging', 'logging_level'): ('core', 'logging_level'),
+ ('logging', 'logging_level'): ('core', 'logging_level', '2.0.0'),
}
# Remove it so we are sure we use the right setting
@@ -471,7 +471,7 @@ AIRFLOW_HOME = /root/airflow
def test_deprecated_options_cmd(self):
# Guarantee we have a deprecated setting, so we test the deprecation
# lookup even if we remove this explicit fallback
- conf.deprecated_options[('celery', "result_backend")] = ('celery', 'celery_result_backend')
+ conf.deprecated_options[('celery', "result_backend")] = 'celery', 'celery_result_backend', '2.0.0'
conf.sensitive_config_values.add(('celery', 'celery_result_backend'))
conf.remove_option('celery', 'result_backend')