You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by as...@apache.org on 2022/06/17 13:20:20 UTC

[airflow] branch logging-settings-is-not-a-template created (now a7064f295f)

This is an automated email from the ASF dual-hosted git repository.

ash pushed a change to branch logging-settings-is-not-a-template
in repository https://gitbox.apache.org/repos/asf/airflow.git


      at a7064f295f airflow_local_settings is not a "template" file

This branch includes the following new commits:

     new a7064f295f airflow_local_settings is not a "template" file

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



[airflow] 01/01: airflow_local_settings is not a "template" file

Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

ash pushed a commit to branch logging-settings-is-not-a-template
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit a7064f295feab056eebd094327bc1573a2b83103
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Mon Jun 1 13:53:50 2020 +0100

    airflow_local_settings is not a "template" file
    
    All of the other files in the config_templates are that -- templates,
    but airflow_local_settings as it was was the actual file used by running
    installs.
    
    For the sake of clarity let's move it
---
 airflow/config_templates/airflow_local_settings.py | 283 +--------------------
 airflow/configuration.py                           |  12 +-
 airflow/logging_config.py                          | 282 +++++++++++++++++++-
 .../logging-monitoring/logging-tasks.rst           |   2 +-
 newsfragments/9092.significant.rst                 |   8 +
 tests/api_connexion/endpoints/test_log_endpoint.py |   2 +-
 tests/dag_processing/test_manager.py               |   2 +-
 .../task/task_runner/test_standard_task_runner.py  |   2 +-
 tests/utils/log/test_log_reader.py                 |   2 +-
 tests/utils/test_log_handlers.py                   |   2 +-
 .../test_task_handler_with_custom_formatter.py     |   2 +-
 tests/www/views/test_views_log.py                  |   2 +-
 12 files changed, 314 insertions(+), 287 deletions(-)

diff --git a/airflow/config_templates/airflow_local_settings.py b/airflow/config_templates/airflow_local_settings.py
index 6684fd18e5..bcb604d7fd 100644
--- a/airflow/config_templates/airflow_local_settings.py
+++ b/airflow/config_templates/airflow_local_settings.py
@@ -15,283 +15,14 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-"""Airflow logging settings"""
+"""This module is deprecated. Please use `airflow.logging_config` instead"""
 
-import os
-from pathlib import Path
-from typing import Any, Dict, Optional, Union
-from urllib.parse import urlparse
+import warnings
 
-from airflow.configuration import conf
-from airflow.exceptions import AirflowException
+from airflow.logging_config import *  # noqa
 
-# TODO: Logging format and level should be configured
-# in this file instead of from airflow.cfg. Currently
-# there are other log format and level configurations in
-# settings.py and cli.py. Please see AIRFLOW-1455.
-LOG_LEVEL: str = conf.get_mandatory_value('logging', 'LOGGING_LEVEL').upper()
-
-
-# Flask appbuilder's info level log is very verbose,
-# so it's set to 'WARN' by default.
-FAB_LOG_LEVEL: str = conf.get_mandatory_value('logging', 'FAB_LOGGING_LEVEL').upper()
-
-LOG_FORMAT: str = conf.get_mandatory_value('logging', 'LOG_FORMAT')
-
-COLORED_LOG_FORMAT: str = conf.get_mandatory_value('logging', 'COLORED_LOG_FORMAT')
-
-COLORED_LOG: bool = conf.getboolean('logging', 'COLORED_CONSOLE_LOG')
-
-COLORED_FORMATTER_CLASS: str = conf.get_mandatory_value('logging', 'COLORED_FORMATTER_CLASS')
-
-BASE_LOG_FOLDER: str = conf.get_mandatory_value('logging', 'BASE_LOG_FOLDER')
-
-PROCESSOR_LOG_FOLDER: str = conf.get_mandatory_value('scheduler', 'CHILD_PROCESS_LOG_DIRECTORY')
-
-DAG_PROCESSOR_MANAGER_LOG_LOCATION: str = conf.get_mandatory_value(
-    'logging', 'DAG_PROCESSOR_MANAGER_LOG_LOCATION'
+warnings.warn(
+    "This module is deprecated. Please use `airflow.logging_config`.",
+    DeprecationWarning,
+    stacklevel=2,
 )
-
-FILENAME_TEMPLATE: str = conf.get_mandatory_value('logging', 'LOG_FILENAME_TEMPLATE')
-
-PROCESSOR_FILENAME_TEMPLATE: str = conf.get_mandatory_value('logging', 'LOG_PROCESSOR_FILENAME_TEMPLATE')
-
-DEFAULT_LOGGING_CONFIG: Dict[str, Any] = {
-    'version': 1,
-    'disable_existing_loggers': False,
-    'formatters': {
-        'airflow': {'format': LOG_FORMAT},
-        'airflow_coloured': {
-            'format': COLORED_LOG_FORMAT if COLORED_LOG else LOG_FORMAT,
-            'class': COLORED_FORMATTER_CLASS if COLORED_LOG else 'logging.Formatter',
-        },
-    },
-    'filters': {
-        'mask_secrets': {
-            '()': 'airflow.utils.log.secrets_masker.SecretsMasker',
-        },
-    },
-    'handlers': {
-        'console': {
-            'class': 'airflow.utils.log.logging_mixin.RedirectStdHandler',
-            'formatter': 'airflow_coloured',
-            'stream': 'sys.stdout',
-            'filters': ['mask_secrets'],
-        },
-        'task': {
-            'class': 'airflow.utils.log.file_task_handler.FileTaskHandler',
-            'formatter': 'airflow',
-            'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER),
-            'filters': ['mask_secrets'],
-        },
-        'processor': {
-            'class': 'airflow.utils.log.file_processor_handler.FileProcessorHandler',
-            'formatter': 'airflow',
-            'base_log_folder': os.path.expanduser(PROCESSOR_LOG_FOLDER),
-            'filename_template': PROCESSOR_FILENAME_TEMPLATE,
-            'filters': ['mask_secrets'],
-        },
-    },
-    'loggers': {
-        'airflow.processor': {
-            'handlers': ['processor'],
-            'level': LOG_LEVEL,
-            'propagate': False,
-        },
-        'airflow.task': {
-            'handlers': ['task'],
-            'level': LOG_LEVEL,
-            'propagate': False,
-            'filters': ['mask_secrets'],
-        },
-        'flask_appbuilder': {
-            'handlers': ['console'],
-            'level': FAB_LOG_LEVEL,
-            'propagate': True,
-        },
-    },
-    'root': {
-        'handlers': ['console'],
-        'level': LOG_LEVEL,
-        'filters': ['mask_secrets'],
-    },
-}
-
-EXTRA_LOGGER_NAMES: Optional[str] = conf.get('logging', 'EXTRA_LOGGER_NAMES', fallback=None)
-if EXTRA_LOGGER_NAMES:
-    new_loggers = {
-        logger_name.strip(): {
-            'handlers': ['console'],
-            'level': LOG_LEVEL,
-            'propagate': True,
-        }
-        for logger_name in EXTRA_LOGGER_NAMES.split(",")
-    }
-    DEFAULT_LOGGING_CONFIG['loggers'].update(new_loggers)
-
-DEFAULT_DAG_PARSING_LOGGING_CONFIG: Dict[str, Dict[str, Dict[str, Any]]] = {
-    'handlers': {
-        'processor_manager': {
-            'class': 'logging.handlers.RotatingFileHandler',
-            'formatter': 'airflow',
-            'filename': DAG_PROCESSOR_MANAGER_LOG_LOCATION,
-            'mode': 'a',
-            'maxBytes': 104857600,  # 100MB
-            'backupCount': 5,
-        }
-    },
-    'loggers': {
-        'airflow.processor_manager': {
-            'handlers': ['processor_manager'],
-            'level': LOG_LEVEL,
-            'propagate': False,
-        }
-    },
-}
-
-# Only update the handlers and loggers when CONFIG_PROCESSOR_MANAGER_LOGGER is set.
-# This is to avoid exceptions when initializing RotatingFileHandler multiple times
-# in multiple processes.
-if os.environ.get('CONFIG_PROCESSOR_MANAGER_LOGGER') == 'True':
-    DEFAULT_LOGGING_CONFIG['handlers'].update(DEFAULT_DAG_PARSING_LOGGING_CONFIG['handlers'])
-    DEFAULT_LOGGING_CONFIG['loggers'].update(DEFAULT_DAG_PARSING_LOGGING_CONFIG['loggers'])
-
-    # Manually create log directory for processor_manager handler as RotatingFileHandler
-    # will only create file but not the directory.
-    processor_manager_handler_config: Dict[str, Any] = DEFAULT_DAG_PARSING_LOGGING_CONFIG['handlers'][
-        'processor_manager'
-    ]
-    directory: str = os.path.dirname(processor_manager_handler_config['filename'])
-    Path(directory).mkdir(parents=True, exist_ok=True, mode=0o755)
-
-##################
-# Remote logging #
-##################
-
-REMOTE_LOGGING: bool = conf.getboolean('logging', 'remote_logging')
-
-if REMOTE_LOGGING:
-
-    ELASTICSEARCH_HOST: Optional[str] = conf.get('elasticsearch', 'HOST')
-
-    # Storage bucket URL for remote logging
-    # S3 buckets should start with "s3://"
-    # Cloudwatch log groups should start with "cloudwatch://"
-    # GCS buckets should start with "gs://"
-    # WASB buckets should start with "wasb"
-    # just to help Airflow select correct handler
-    REMOTE_BASE_LOG_FOLDER: str = conf.get_mandatory_value('logging', 'REMOTE_BASE_LOG_FOLDER')
-
-    if REMOTE_BASE_LOG_FOLDER.startswith('s3://'):
-        S3_REMOTE_HANDLERS: Dict[str, Dict[str, str]] = {
-            'task': {
-                'class': 'airflow.providers.amazon.aws.log.s3_task_handler.S3TaskHandler',
-                'formatter': 'airflow',
-                'base_log_folder': str(os.path.expanduser(BASE_LOG_FOLDER)),
-                's3_log_folder': REMOTE_BASE_LOG_FOLDER,
-                'filename_template': FILENAME_TEMPLATE,
-            },
-        }
-
-        DEFAULT_LOGGING_CONFIG['handlers'].update(S3_REMOTE_HANDLERS)
-    elif REMOTE_BASE_LOG_FOLDER.startswith('cloudwatch://'):
-        url_parts = urlparse(REMOTE_BASE_LOG_FOLDER)
-        CLOUDWATCH_REMOTE_HANDLERS: Dict[str, Dict[str, str]] = {
-            'task': {
-                'class': 'airflow.providers.amazon.aws.log.cloudwatch_task_handler.CloudwatchTaskHandler',
-                'formatter': 'airflow',
-                'base_log_folder': str(os.path.expanduser(BASE_LOG_FOLDER)),
-                'log_group_arn': url_parts.netloc + url_parts.path,
-                'filename_template': FILENAME_TEMPLATE,
-            },
-        }
-
-        DEFAULT_LOGGING_CONFIG['handlers'].update(CLOUDWATCH_REMOTE_HANDLERS)
-    elif REMOTE_BASE_LOG_FOLDER.startswith('gs://'):
-        key_path = conf.get_mandatory_value('logging', 'GOOGLE_KEY_PATH', fallback=None)
-        GCS_REMOTE_HANDLERS: Dict[str, Dict[str, str]] = {
-            'task': {
-                'class': 'airflow.providers.google.cloud.log.gcs_task_handler.GCSTaskHandler',
-                'formatter': 'airflow',
-                'base_log_folder': str(os.path.expanduser(BASE_LOG_FOLDER)),
-                'gcs_log_folder': REMOTE_BASE_LOG_FOLDER,
-                'filename_template': FILENAME_TEMPLATE,
-                'gcp_key_path': key_path,
-            },
-        }
-
-        DEFAULT_LOGGING_CONFIG['handlers'].update(GCS_REMOTE_HANDLERS)
-    elif REMOTE_BASE_LOG_FOLDER.startswith('wasb'):
-        WASB_REMOTE_HANDLERS: Dict[str, Dict[str, Union[str, bool]]] = {
-            'task': {
-                'class': 'airflow.providers.microsoft.azure.log.wasb_task_handler.WasbTaskHandler',
-                'formatter': 'airflow',
-                'base_log_folder': str(os.path.expanduser(BASE_LOG_FOLDER)),
-                'wasb_log_folder': REMOTE_BASE_LOG_FOLDER,
-                'wasb_container': 'airflow-logs',
-                'filename_template': FILENAME_TEMPLATE,
-                'delete_local_copy': False,
-            },
-        }
-
-        DEFAULT_LOGGING_CONFIG['handlers'].update(WASB_REMOTE_HANDLERS)
-    elif REMOTE_BASE_LOG_FOLDER.startswith('stackdriver://'):
-        key_path = conf.get_mandatory_value('logging', 'GOOGLE_KEY_PATH', fallback=None)
-        # stackdriver:///airflow-tasks => airflow-tasks
-        log_name = urlparse(REMOTE_BASE_LOG_FOLDER).path[1:]
-        STACKDRIVER_REMOTE_HANDLERS = {
-            'task': {
-                'class': 'airflow.providers.google.cloud.log.stackdriver_task_handler.StackdriverTaskHandler',
-                'formatter': 'airflow',
-                'name': log_name,
-                'gcp_key_path': key_path,
-            }
-        }
-
-        DEFAULT_LOGGING_CONFIG['handlers'].update(STACKDRIVER_REMOTE_HANDLERS)
-    elif REMOTE_BASE_LOG_FOLDER.startswith('oss://'):
-        OSS_REMOTE_HANDLERS = {
-            'task': {
-                'class': 'airflow.providers.alibaba.cloud.log.oss_task_handler.OSSTaskHandler',
-                'formatter': 'airflow',
-                'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER),
-                'oss_log_folder': REMOTE_BASE_LOG_FOLDER,
-                'filename_template': FILENAME_TEMPLATE,
-            },
-        }
-        DEFAULT_LOGGING_CONFIG['handlers'].update(OSS_REMOTE_HANDLERS)
-    elif ELASTICSEARCH_HOST:
-        ELASTICSEARCH_LOG_ID_TEMPLATE: str = conf.get_mandatory_value('elasticsearch', 'LOG_ID_TEMPLATE')
-        ELASTICSEARCH_END_OF_LOG_MARK: str = conf.get_mandatory_value('elasticsearch', 'END_OF_LOG_MARK')
-        ELASTICSEARCH_FRONTEND: str = conf.get_mandatory_value('elasticsearch', 'frontend')
-        ELASTICSEARCH_WRITE_STDOUT: bool = conf.getboolean('elasticsearch', 'WRITE_STDOUT')
-        ELASTICSEARCH_JSON_FORMAT: bool = conf.getboolean('elasticsearch', 'JSON_FORMAT')
-        ELASTICSEARCH_JSON_FIELDS: str = conf.get_mandatory_value('elasticsearch', 'JSON_FIELDS')
-        ELASTICSEARCH_HOST_FIELD: str = conf.get_mandatory_value('elasticsearch', 'HOST_FIELD')
-        ELASTICSEARCH_OFFSET_FIELD: str = conf.get_mandatory_value('elasticsearch', 'OFFSET_FIELD')
-
-        ELASTIC_REMOTE_HANDLERS: Dict[str, Dict[str, Union[str, bool]]] = {
-            'task': {
-                'class': 'airflow.providers.elasticsearch.log.es_task_handler.ElasticsearchTaskHandler',
-                'formatter': 'airflow',
-                'base_log_folder': str(os.path.expanduser(BASE_LOG_FOLDER)),
-                'log_id_template': ELASTICSEARCH_LOG_ID_TEMPLATE,
-                'filename_template': FILENAME_TEMPLATE,
-                'end_of_log_mark': ELASTICSEARCH_END_OF_LOG_MARK,
-                'host': ELASTICSEARCH_HOST,
-                'frontend': ELASTICSEARCH_FRONTEND,
-                'write_stdout': ELASTICSEARCH_WRITE_STDOUT,
-                'json_format': ELASTICSEARCH_JSON_FORMAT,
-                'json_fields': ELASTICSEARCH_JSON_FIELDS,
-                'host_field': ELASTICSEARCH_HOST_FIELD,
-                'offset_field': ELASTICSEARCH_OFFSET_FIELD,
-            },
-        }
-
-        DEFAULT_LOGGING_CONFIG['handlers'].update(ELASTIC_REMOTE_HANDLERS)
-    else:
-        raise AirflowException(
-            "Incorrect remote log configuration. Please check the configuration of option 'host' in "
-            "section 'elasticsearch' if you are using Elasticsearch. In the other case, "
-            "'remote_base_log_folder' option in the 'logging' section."
-        )
diff --git a/airflow/configuration.py b/airflow/configuration.py
index 84df81812c..813f1a39eb 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -228,6 +228,11 @@ class AirflowConfigParser(ConfigParser):
     deprecated_values: Dict[str, Dict[str, Tuple[Pattern, str, str]]] = {
         'core': {
             'hostname_callable': (re.compile(r':'), r'.', '2.1'),
+            'logging_config_class': (
+                re.compile(r'\Aairflow.config_templates.airflow_local_settings.DEFAULT_LOGGING_CONFIG\Z'),
+                'airflow.logging_config.DEFAULT_LOGGING_CONFIG',
+                '3.0',
+            ),
         },
         'webserver': {
             'navbar_color': (re.compile(r'\A#007A87\Z', re.IGNORECASE), '#fff', '2.1'),
@@ -246,6 +251,11 @@ class AirflowConfigParser(ConfigParser):
                 "XX-set-after-default-config-loaded-XX",
                 '3.0',
             ),
+            'logging_config_class': (
+                re.compile(r'\Aairflow.config_templates.airflow_local_settings.DEFAULT_LOGGING_CONFIG\Z'),
+                'airflow.logging_config.DEFAULT_LOGGING_CONFIG',
+                '3.0',
+            ),
         },
         'api': {
             'auth_backends': (
@@ -422,7 +432,7 @@ class AirflowConfigParser(ConfigParser):
                 )
 
     def _using_old_value(self, old: Pattern, current_value: str) -> bool:
-        return old.search(current_value) is not None
+        return current_value is not None and old.search(current_value) is not None
 
     def _update_env_var(self, section: str, name: str, new_value: Union[str]):
         env_var = self._env_var_name(section, name)
diff --git a/airflow/logging_config.py b/airflow/logging_config.py
index 645e53eb3e..0b35eade9b 100644
--- a/airflow/logging_config.py
+++ b/airflow/logging_config.py
@@ -17,8 +17,12 @@
 # under the License.
 #
 import logging
+import os
 import warnings
 from logging.config import dictConfig
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
+from urllib.parse import urlparse
 
 from airflow.configuration import conf
 from airflow.exceptions import AirflowConfigException
@@ -48,8 +52,8 @@ def configure_logging():
             # Import default logging configurations.
             raise ImportError(f'Unable to load custom logging from {logging_class_path} due to {err}')
     else:
-        logging_class_path = 'airflow.config_templates.airflow_local_settings.DEFAULT_LOGGING_CONFIG'
-        logging_config = import_string(logging_class_path)
+        logging_config = DEFAULT_LOGGING_CONFIG
+        logging_class_path = 'airflow.logging_config.DEFAULT_LOGGING_CONFIG'
         log.debug('Unable to load custom logging, using default config instead')
 
     try:
@@ -82,6 +86,12 @@ def validate_logging_config(logging_config):
     # Now lets validate the other logging-related settings
     task_log_reader = conf.get('logging', 'task_log_reader')
 
+    if REMOTE_LOGGING and logging_config['handlers']['task'] is __stock_task_handler:
+        raise AirflowConfigException(
+            "[logging] remote_logging config was enabled, but no suitable backend was detected. Please check"
+            " the 'remote_base_log_folder' setting"
+        )
+
     logger = logging.getLogger('airflow.task')
 
     def _get_handler(name):
@@ -102,3 +112,271 @@ def validate_logging_config(logging_config):
                 f"Configured task_log_reader {task_log_reader!r} was not a handler of "
                 f"the 'airflow.task' logger."
             )
+
+
+# TODO: Logging format and level should be configured
+# in this file instead of from airflow.cfg. Currently
+# there are other log format and level configurations in
+# settings.py and cli.py. Please see AIRFLOW-1455.
+LOG_LEVEL: str = conf.get_mandatory_value('logging', 'LOGGING_LEVEL').upper()
+
+
+# Flask appbuilder's info level log is very verbose,
+# so it's set to 'WARN' by default.
+FAB_LOG_LEVEL: str = conf.get_mandatory_value('logging', 'FAB_LOGGING_LEVEL').upper()
+
+LOG_FORMAT: str = conf.get_mandatory_value('logging', 'LOG_FORMAT')
+
+COLORED_LOG_FORMAT: str = conf.get_mandatory_value('logging', 'COLORED_LOG_FORMAT')
+
+COLORED_LOG: bool = conf.getboolean('logging', 'COLORED_CONSOLE_LOG')
+
+COLORED_FORMATTER_CLASS: str = conf.get_mandatory_value('logging', 'COLORED_FORMATTER_CLASS')
+
+BASE_LOG_FOLDER: str = conf.get_mandatory_value('logging', 'BASE_LOG_FOLDER')
+
+PROCESSOR_LOG_FOLDER: str = conf.get_mandatory_value('scheduler', 'CHILD_PROCESS_LOG_DIRECTORY')
+
+DAG_PROCESSOR_MANAGER_LOG_LOCATION: str = conf.get_mandatory_value(
+    'logging', 'DAG_PROCESSOR_MANAGER_LOG_LOCATION'
+)
+
+FILENAME_TEMPLATE: str = conf.get_mandatory_value('logging', 'LOG_FILENAME_TEMPLATE')
+
+PROCESSOR_FILENAME_TEMPLATE: str = conf.get_mandatory_value('logging', 'LOG_PROCESSOR_FILENAME_TEMPLATE')
+
+DEFAULT_LOGGING_CONFIG: Dict[str, Any] = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'airflow': {'format': LOG_FORMAT},
+        'airflow_coloured': {
+            'format': COLORED_LOG_FORMAT if COLORED_LOG else LOG_FORMAT,
+            'class': COLORED_FORMATTER_CLASS if COLORED_LOG else 'logging.Formatter',
+        },
+    },
+    'filters': {
+        'mask_secrets': {
+            '()': 'airflow.utils.log.secrets_masker.SecretsMasker',
+        },
+    },
+    'handlers': {
+        'console': {
+            'class': 'airflow.utils.log.logging_mixin.RedirectStdHandler',
+            'formatter': 'airflow_coloured',
+            'stream': 'sys.stdout',
+            'filters': ['mask_secrets'],
+        },
+        'task': {
+            'class': 'airflow.utils.log.file_task_handler.FileTaskHandler',
+            'formatter': 'airflow',
+            'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER),
+            'filters': ['mask_secrets'],
+        },
+        'processor': {
+            'class': 'airflow.utils.log.file_processor_handler.FileProcessorHandler',
+            'formatter': 'airflow',
+            'base_log_folder': os.path.expanduser(PROCESSOR_LOG_FOLDER),
+            'filename_template': PROCESSOR_FILENAME_TEMPLATE,
+            'filters': ['mask_secrets'],
+        },
+    },
+    'loggers': {
+        'airflow.processor': {
+            'handlers': ['processor'],
+            'level': LOG_LEVEL,
+            'propagate': False,
+        },
+        'airflow.task': {
+            'handlers': ['task'],
+            'level': LOG_LEVEL,
+            'propagate': False,
+            'filters': ['mask_secrets'],
+        },
+        'flask_appbuilder': {
+            'handlers': ['console'],
+            'level': FAB_LOG_LEVEL,
+            'propagate': True,
+        },
+    },
+    'root': {
+        'handlers': ['console'],
+        'level': LOG_LEVEL,
+        'filters': ['mask_secrets'],
+    },
+}
+
+__stock_task_handler = DEFAULT_LOGGING_CONFIG['handlers']['task']
+
+EXTRA_LOGGER_NAMES: Optional[str] = conf.get('logging', 'EXTRA_LOGGER_NAMES', fallback=None)
+if EXTRA_LOGGER_NAMES:
+    new_loggers = {
+        logger_name.strip(): {
+            'handlers': ['console'],
+            'level': LOG_LEVEL,
+            'propagate': True,
+        }
+        for logger_name in EXTRA_LOGGER_NAMES.split(",")
+    }
+    DEFAULT_LOGGING_CONFIG['loggers'].update(new_loggers)
+
+DEFAULT_DAG_PARSING_LOGGING_CONFIG: Dict[str, Dict[str, Dict[str, Any]]] = {
+    'handlers': {
+        'processor_manager': {
+            'class': 'logging.handlers.RotatingFileHandler',
+            'formatter': 'airflow',
+            'filename': DAG_PROCESSOR_MANAGER_LOG_LOCATION,
+            'mode': 'a',
+            'maxBytes': 104857600,  # 100MB
+            'backupCount': 5,
+        }
+    },
+    'loggers': {
+        'airflow.processor_manager': {
+            'handlers': ['processor_manager'],
+            'level': LOG_LEVEL,
+            'propagate': False,
+        }
+    },
+}
+
+# Only update the handlers and loggers when CONFIG_PROCESSOR_MANAGER_LOGGER is set.
+# This is to avoid exceptions when initializing RotatingFileHandler multiple times
+# in multiple processes.
+if os.environ.get('CONFIG_PROCESSOR_MANAGER_LOGGER') == 'True':
+    DEFAULT_LOGGING_CONFIG['handlers'].update(DEFAULT_DAG_PARSING_LOGGING_CONFIG['handlers'])
+    DEFAULT_LOGGING_CONFIG['loggers'].update(DEFAULT_DAG_PARSING_LOGGING_CONFIG['loggers'])
+
+    # Manually create log directory for processor_manager handler as RotatingFileHandler
+    # will only create file but not the directory.
+    processor_manager_handler_config: Dict[str, Any] = DEFAULT_DAG_PARSING_LOGGING_CONFIG['handlers'][
+        'processor_manager'
+    ]
+    directory: str = os.path.dirname(processor_manager_handler_config['filename'])
+    Path(directory).mkdir(parents=True, exist_ok=True, mode=0o755)
+
+##################
+# Remote logging #
+##################
+
+REMOTE_LOGGING: bool = conf.getboolean('logging', 'remote_logging')
+
+if REMOTE_LOGGING:
+
+    ELASTICSEARCH_HOST: Optional[str] = conf.get('elasticsearch', 'HOST')
+
+    # Storage bucket URL for remote logging
+    # S3 buckets should start with "s3://"
+    # Cloudwatch log groups should start with "cloudwatch://"
+    # GCS buckets should start with "gs://"
+    # WASB buckets should start with "wasb"
+    # just to help Airflow select correct handler
+    REMOTE_BASE_LOG_FOLDER: str = conf.get_mandatory_value('logging', 'REMOTE_BASE_LOG_FOLDER')
+
+    if REMOTE_BASE_LOG_FOLDER.startswith('s3://'):
+        S3_REMOTE_HANDLERS: Dict[str, Dict[str, str]] = {
+            'task': {
+                'class': 'airflow.providers.amazon.aws.log.s3_task_handler.S3TaskHandler',
+                'formatter': 'airflow',
+                'base_log_folder': str(os.path.expanduser(BASE_LOG_FOLDER)),
+                's3_log_folder': REMOTE_BASE_LOG_FOLDER,
+                'filename_template': FILENAME_TEMPLATE,
+            },
+        }
+
+        DEFAULT_LOGGING_CONFIG['handlers'].update(S3_REMOTE_HANDLERS)
+    elif REMOTE_BASE_LOG_FOLDER.startswith('cloudwatch://'):
+        url_parts = urlparse(REMOTE_BASE_LOG_FOLDER)
+        CLOUDWATCH_REMOTE_HANDLERS: Dict[str, Dict[str, str]] = {
+            'task': {
+                'class': 'airflow.providers.amazon.aws.log.cloudwatch_task_handler.CloudwatchTaskHandler',
+                'formatter': 'airflow',
+                'base_log_folder': str(os.path.expanduser(BASE_LOG_FOLDER)),
+                'log_group_arn': url_parts.netloc + url_parts.path,
+                'filename_template': FILENAME_TEMPLATE,
+            },
+        }
+
+        DEFAULT_LOGGING_CONFIG['handlers'].update(CLOUDWATCH_REMOTE_HANDLERS)
+    elif REMOTE_BASE_LOG_FOLDER.startswith('gs://'):
+        key_path = conf.get_mandatory_value('logging', 'GOOGLE_KEY_PATH', fallback=None)
+        GCS_REMOTE_HANDLERS: Dict[str, Dict[str, str]] = {
+            'task': {
+                'class': 'airflow.providers.google.cloud.log.gcs_task_handler.GCSTaskHandler',
+                'formatter': 'airflow',
+                'base_log_folder': str(os.path.expanduser(BASE_LOG_FOLDER)),
+                'gcs_log_folder': REMOTE_BASE_LOG_FOLDER,
+                'filename_template': FILENAME_TEMPLATE,
+                'gcp_key_path': key_path,
+            },
+        }
+
+        DEFAULT_LOGGING_CONFIG['handlers'].update(GCS_REMOTE_HANDLERS)
+    elif REMOTE_BASE_LOG_FOLDER.startswith('wasb'):
+        WASB_REMOTE_HANDLERS: Dict[str, Dict[str, Union[str, bool]]] = {
+            'task': {
+                'class': 'airflow.providers.microsoft.azure.log.wasb_task_handler.WasbTaskHandler',
+                'formatter': 'airflow',
+                'base_log_folder': str(os.path.expanduser(BASE_LOG_FOLDER)),
+                'wasb_log_folder': REMOTE_BASE_LOG_FOLDER,
+                'wasb_container': 'airflow-logs',
+                'filename_template': FILENAME_TEMPLATE,
+                'delete_local_copy': False,
+            },
+        }
+
+        DEFAULT_LOGGING_CONFIG['handlers'].update(WASB_REMOTE_HANDLERS)
+    elif REMOTE_BASE_LOG_FOLDER.startswith('stackdriver://'):
+        key_path = conf.get_mandatory_value('logging', 'GOOGLE_KEY_PATH', fallback=None)
+        # stackdriver:///airflow-tasks => airflow-tasks
+        log_name = urlparse(REMOTE_BASE_LOG_FOLDER).path[1:]
+        STACKDRIVER_REMOTE_HANDLERS = {
+            'task': {
+                'class': 'airflow.providers.google.cloud.log.stackdriver_task_handler.StackdriverTaskHandler',
+                'formatter': 'airflow',
+                'name': log_name,
+                'gcp_key_path': key_path,
+            }
+        }
+
+        DEFAULT_LOGGING_CONFIG['handlers'].update(STACKDRIVER_REMOTE_HANDLERS)
+    elif REMOTE_BASE_LOG_FOLDER.startswith('oss://'):
+        OSS_REMOTE_HANDLERS = {
+            'task': {
+                'class': 'airflow.providers.alibaba.cloud.log.oss_task_handler.OSSTaskHandler',
+                'formatter': 'airflow',
+                'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER),
+                'oss_log_folder': REMOTE_BASE_LOG_FOLDER,
+                'filename_template': FILENAME_TEMPLATE,
+            },
+        }
+        DEFAULT_LOGGING_CONFIG['handlers'].update(OSS_REMOTE_HANDLERS)
+    elif ELASTICSEARCH_HOST:
+        ELASTICSEARCH_LOG_ID_TEMPLATE: str = conf.get_mandatory_value('elasticsearch', 'LOG_ID_TEMPLATE')
+        ELASTICSEARCH_END_OF_LOG_MARK: str = conf.get_mandatory_value('elasticsearch', 'END_OF_LOG_MARK')
+        ELASTICSEARCH_FRONTEND: str = conf.get_mandatory_value('elasticsearch', 'frontend')
+        ELASTICSEARCH_WRITE_STDOUT: bool = conf.getboolean('elasticsearch', 'WRITE_STDOUT')
+        ELASTICSEARCH_JSON_FORMAT: bool = conf.getboolean('elasticsearch', 'JSON_FORMAT')
+        ELASTICSEARCH_JSON_FIELDS: str = conf.get_mandatory_value('elasticsearch', 'JSON_FIELDS')
+        ELASTICSEARCH_HOST_FIELD: str = conf.get_mandatory_value('elasticsearch', 'HOST_FIELD')
+        ELASTICSEARCH_OFFSET_FIELD: str = conf.get_mandatory_value('elasticsearch', 'OFFSET_FIELD')
+
+        ELASTIC_REMOTE_HANDLERS: Dict[str, Dict[str, Union[str, bool]]] = {
+            'task': {
+                'class': 'airflow.providers.elasticsearch.log.es_task_handler.ElasticsearchTaskHandler',
+                'formatter': 'airflow',
+                'base_log_folder': str(os.path.expanduser(BASE_LOG_FOLDER)),
+                'log_id_template': ELASTICSEARCH_LOG_ID_TEMPLATE,
+                'filename_template': FILENAME_TEMPLATE,
+                'end_of_log_mark': ELASTICSEARCH_END_OF_LOG_MARK,
+                'host': ELASTICSEARCH_HOST,
+                'frontend': ELASTICSEARCH_FRONTEND,
+                'write_stdout': ELASTICSEARCH_WRITE_STDOUT,
+                'json_format': ELASTICSEARCH_JSON_FORMAT,
+                'json_fields': ELASTICSEARCH_JSON_FIELDS,
+                'host_field': ELASTICSEARCH_HOST_FIELD,
+                'offset_field': ELASTICSEARCH_OFFSET_FIELD,
+            },
+        }
+
+        DEFAULT_LOGGING_CONFIG['handlers'].update(ELASTIC_REMOTE_HANDLERS)
diff --git a/docs/apache-airflow/logging-monitoring/logging-tasks.rst b/docs/apache-airflow/logging-monitoring/logging-tasks.rst
index e5b0335651..6078b06179 100644
--- a/docs/apache-airflow/logging-monitoring/logging-tasks.rst
+++ b/docs/apache-airflow/logging-monitoring/logging-tasks.rst
@@ -98,7 +98,7 @@ Follow the steps below to enable custom logging config class:
     .. code-block:: python
 
       from copy import deepcopy
-      from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG
+      from airflow.logging_config import DEFAULT_LOGGING_CONFIG
 
       LOGGING_CONFIG = deepcopy(DEFAULT_LOGGING_CONFIG)
 
diff --git a/newsfragments/9092.significant.rst b/newsfragments/9092.significant.rst
new file mode 100644
index 0000000000..47ffc753aa
--- /dev/null
+++ b/newsfragments/9092.significant.rst
@@ -0,0 +1,8 @@
+Default logging config has moved to a new module
+
+The default logging config dict has moved from
+``airflow.config_templates.airflow_local_settings.DEFAULT_LOGGING_CONFIG`` to
+``airflow.logging_config.DEFAULT_LOGGING_CONFIG``.
+
+Most users will not need to take any action as a result of this change. (If you have a custom logging config
+which imports the base one you will receive a deprecation warning.)
diff --git a/tests/api_connexion/endpoints/test_log_endpoint.py b/tests/api_connexion/endpoints/test_log_endpoint.py
index 1b226be96f..d432540662 100644
--- a/tests/api_connexion/endpoints/test_log_endpoint.py
+++ b/tests/api_connexion/endpoints/test_log_endpoint.py
@@ -25,7 +25,7 @@ from itsdangerous.url_safe import URLSafeSerializer
 
 from airflow import DAG
 from airflow.api_connexion.exceptions import EXCEPTIONS_LINK_MAP
-from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG
+from airflow.logging_config import DEFAULT_LOGGING_CONFIG
 from airflow.operators.empty import EmptyOperator
 from airflow.security import permissions
 from airflow.utils import timezone
diff --git a/tests/dag_processing/test_manager.py b/tests/dag_processing/test_manager.py
index 891df4ec64..bb60999deb 100644
--- a/tests/dag_processing/test_manager.py
+++ b/tests/dag_processing/test_manager.py
@@ -37,7 +37,6 @@ from freezegun import freeze_time
 from sqlalchemy import func
 
 from airflow.callbacks.callback_requests import CallbackRequest, DagCallbackRequest, SlaCallbackRequest
-from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG
 from airflow.configuration import conf
 from airflow.dag_processing.manager import (
     DagFileProcessorAgent,
@@ -47,6 +46,7 @@ from airflow.dag_processing.manager import (
     DagParsingStat,
 )
 from airflow.dag_processing.processor import DagFileProcessorProcess
+from airflow.logging_config import DEFAULT_LOGGING_CONFIG
 from airflow.models import DagBag, DagModel, DbCallbackRequest, errors
 from airflow.models.dagcode import DagCode
 from airflow.models.serialized_dag import SerializedDagModel
diff --git a/tests/task/task_runner/test_standard_task_runner.py b/tests/task/task_runner/test_standard_task_runner.py
index 2ed266a543..f561f3dc33 100644
--- a/tests/task/task_runner/test_standard_task_runner.py
+++ b/tests/task/task_runner/test_standard_task_runner.py
@@ -24,8 +24,8 @@ from unittest import mock
 import psutil
 import pytest
 
-from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG
 from airflow.jobs.local_task_job import LocalTaskJob
+from airflow.logging_config import DEFAULT_LOGGING_CONFIG
 from airflow.models.dagbag import DagBag
 from airflow.models.taskinstance import TaskInstance
 from airflow.task.task_runner.standard_task_runner import StandardTaskRunner
diff --git a/tests/utils/log/test_log_reader.py b/tests/utils/log/test_log_reader.py
index 9a76ada725..1e19607e5b 100644
--- a/tests/utils/log/test_log_reader.py
+++ b/tests/utils/log/test_log_reader.py
@@ -27,7 +27,7 @@ import pendulum
 import pytest
 
 from airflow import settings
-from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG
+from airflow.logging_config import DEFAULT_LOGGING_CONFIG
 from airflow.models import DagRun
 from airflow.models.tasklog import LogTemplate
 from airflow.operators.python import PythonOperator
diff --git a/tests/utils/test_log_handlers.py b/tests/utils/test_log_handlers.py
index 28b9c7cf1a..979c66b09a 100644
--- a/tests/utils/test_log_handlers.py
+++ b/tests/utils/test_log_handlers.py
@@ -21,7 +21,7 @@ import logging.config
 import os
 import re
 
-from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG
+from airflow.logging_config import DEFAULT_LOGGING_CONFIG
 from airflow.models import DAG, DagRun, TaskInstance
 from airflow.operators.python import PythonOperator
 from airflow.utils.log.file_task_handler import FileTaskHandler
diff --git a/tests/utils/test_task_handler_with_custom_formatter.py b/tests/utils/test_task_handler_with_custom_formatter.py
index df7e174138..d4b8c626eb 100644
--- a/tests/utils/test_task_handler_with_custom_formatter.py
+++ b/tests/utils/test_task_handler_with_custom_formatter.py
@@ -19,7 +19,7 @@ import logging
 
 import pytest
 
-from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG
+from airflow.logging_config import DEFAULT_LOGGING_CONFIG
 from airflow.models import DAG, TaskInstance
 from airflow.operators.empty import EmptyOperator
 from airflow.utils.log.logging_mixin import set_context
diff --git a/tests/www/views/test_views_log.py b/tests/www/views/test_views_log.py
index 82b30f9d21..57c1b55be2 100644
--- a/tests/www/views/test_views_log.py
+++ b/tests/www/views/test_views_log.py
@@ -26,7 +26,7 @@ import urllib.parse
 import pytest
 
 from airflow import settings
-from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG
+from airflow.logging_config import DEFAULT_LOGGING_CONFIG
 from airflow.models import DagBag, DagRun
 from airflow.models.tasklog import LogTemplate
 from airflow.utils import timezone