You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by fo...@apache.org on 2018/05/04 07:02:44 UTC
[48/50] incubator-airflow git commit: [AIRFLOW-1812] Update logging
example
[AIRFLOW-1812] Update logging example
The logging has changed, therefore we should also
update the
updating.md guide
Closes #2784 from Fokko/AIRFLOW-1812-update-
logging-example
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/97ab9e76
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/97ab9e76
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/97ab9e76
Branch: refs/heads/v1-10-test
Commit: 97ab9e762c1f325317fd8003e2846e931137a1e6
Parents: 3a28ceb
Author: Fokko Driesprong <fo...@godatadriven.com>
Authored: Thu May 3 22:58:44 2018 -0700
Committer: r39132 <si...@yahoo.com>
Committed: Thu May 3 22:58:44 2018 -0700
----------------------------------------------------------------------
UPDATING.md | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 112 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/97ab9e76/UPDATING.md
----------------------------------------------------------------------
diff --git a/UPDATING.md b/UPDATING.md
index 8006876..defd95b 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -61,9 +61,9 @@ Dataflow job labeling is now supported in Dataflow{Java,Python}Operator with a d
to 2.2.0 or greater.
### Redshift to S3 Operator
-With Airflow 1.9 or lower, Unload operation always included header row. In order to include header row,
+With Airflow 1.9 or lower, Unload operation always included header row. In order to include header row,
we need to turn off parallel unload. It is preferred to perform unload operation using all nodes so that it is
-faster for larger tables. So, parameter called `include_header` is added and default is set to False.
+faster for larger tables. So, parameter called `include_header` is added and default is set to False.
Header row will be added only if this parameter is set True and also in that case parallel will be automatically turned off (`PARALLEL OFF`)
### Google cloud connection string
@@ -119,7 +119,116 @@ logging_config_class = my.path.default_local_settings.LOGGING_CONFIG
The logging configuration file needs to be on the `PYTHONPATH`, for example `$AIRFLOW_HOME/config`. This directory is loaded by default. Any directory may be added to the `PYTHONPATH`, this might be handy when the config is in another directory or a volume is mounted in case of Docker.
-The config can be taken from `airflow/config_templates/airflow_local_settings.py` as a starting point. Copy the contents to `${AIRFLOW_HOME}/config/airflow_local_settings.py`, and alter the config as is preferred.
+The config can be taken from `airflow/config_templates/airflow_local_settings.py` as a starting point. Copy the contents to `${AIRFLOW_HOME}/config/airflow_local_settings.py`, and alter the config as is preferred.
+
+```
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from airflow import configuration as conf
+
+# TODO: Logging format and level should be configured
+# in this file instead of from airflow.cfg. Currently
+# there are other log format and level configurations in
+# settings.py and cli.py. Please see AIRFLOW-1455.
+
+LOG_LEVEL = conf.get('core', 'LOGGING_LEVEL').upper()
+LOG_FORMAT = conf.get('core', 'log_format')
+
+BASE_LOG_FOLDER = conf.get('core', 'BASE_LOG_FOLDER')
+PROCESSOR_LOG_FOLDER = conf.get('scheduler', 'child_process_log_directory')
+
+FILENAME_TEMPLATE = '{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log'
+PROCESSOR_FILENAME_TEMPLATE = '{{ filename }}.log'
+
+DEFAULT_LOGGING_CONFIG = {
+ 'version': 1,
+ 'disable_existing_loggers': False,
+ 'formatters': {
+ 'airflow.task': {
+ 'format': LOG_FORMAT,
+ },
+ 'airflow.processor': {
+ 'format': LOG_FORMAT,
+ },
+ },
+ 'handlers': {
+ 'console': {
+ 'class': 'logging.StreamHandler',
+ 'formatter': 'airflow.task',
+ 'stream': 'ext://sys.stdout'
+ },
+ 'file.task': {
+ 'class': 'airflow.utils.log.file_task_handler.FileTaskHandler',
+ 'formatter': 'airflow.task',
+ 'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER),
+ 'filename_template': FILENAME_TEMPLATE,
+ },
+ 'file.processor': {
+ 'class': 'airflow.utils.log.file_processor_handler.FileProcessorHandler',
+ 'formatter': 'airflow.processor',
+ 'base_log_folder': os.path.expanduser(PROCESSOR_LOG_FOLDER),
+ 'filename_template': PROCESSOR_FILENAME_TEMPLATE,
+ }
+ # When using s3 or gcs, provide a customized LOGGING_CONFIG
+ # in airflow_local_settings within your PYTHONPATH, see UPDATING.md
+ # for details
+ # 's3.task': {
+ # 'class': 'airflow.utils.log.s3_task_handler.S3TaskHandler',
+ # 'formatter': 'airflow.task',
+ # 'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER),
+ # 's3_log_folder': S3_LOG_FOLDER,
+ # 'filename_template': FILENAME_TEMPLATE,
+ # },
+ # 'gcs.task': {
+ # 'class': 'airflow.utils.log.gcs_task_handler.GCSTaskHandler',
+ # 'formatter': 'airflow.task',
+ # 'base_log_folder': os.path.expanduser(BASE_LOG_FOLDER),
+ # 'gcs_log_folder': GCS_LOG_FOLDER,
+ # 'filename_template': FILENAME_TEMPLATE,
+ # },
+ },
+ 'loggers': {
+ '': {
+ 'handlers': ['console'],
+ 'level': LOG_LEVEL
+ },
+ 'airflow': {
+ 'handlers': ['console'],
+ 'level': LOG_LEVEL,
+ 'propagate': False,
+ },
+ 'airflow.processor': {
+ 'handlers': ['file.processor'],
+ 'level': LOG_LEVEL,
+ 'propagate': True,
+ },
+ 'airflow.task': {
+ 'handlers': ['file.task'],
+ 'level': LOG_LEVEL,
+ 'propagate': False,
+ },
+ 'airflow.task_runner': {
+ 'handlers': ['file.task'],
+ 'level': LOG_LEVEL,
+ 'propagate': True,
+ },
+ }
+}
+```
To customize the logging (for example, use logging rotate), define one or more of the logging handles that [Python has to offer](https://docs.python.org/3/library/logging.handlers.html). For more details about the Python logging, please refer to the [official logging documentation](https://docs.python.org/3/library/logging.html).