You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2022/08/15 18:44:44 UTC
[airflow] 08/45: Add %z for %(asctime)s to fix timezone for logs on UI (#24811)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 9b4eec2a0ea9b85a457e463cf2f251820e24ba0d
Author: Ko HanJong <ha...@navercorp.com>
AuthorDate: Thu Jul 14 00:27:27 2022 +0900
Add %z for %(asctime)s to fix timezone for logs on UI (#24811)
(cherry picked from commit 851e5cad165a043654116a8a2717c9d3643d8251)
---
airflow/config_templates/airflow_local_settings.py | 11 ++++-
airflow/config_templates/config.yml | 6 +++
airflow/config_templates/default_airflow.cfg | 1 +
airflow/utils/log/timezone_aware.py | 49 ++++++++++++++++++++++
airflow/www/static/js/ti_log.js | 4 +-
newsfragments/24811.significant.rst | 22 ++++++++++
6 files changed, 90 insertions(+), 3 deletions(-)
diff --git a/airflow/config_templates/airflow_local_settings.py b/airflow/config_templates/airflow_local_settings.py
index 6684fd18e5..ea8a19e80c 100644
--- a/airflow/config_templates/airflow_local_settings.py
+++ b/airflow/config_templates/airflow_local_settings.py
@@ -38,6 +38,10 @@ FAB_LOG_LEVEL: str = conf.get_mandatory_value('logging', 'FAB_LOGGING_LEVEL').up
LOG_FORMAT: str = conf.get_mandatory_value('logging', 'LOG_FORMAT')
+LOG_FORMATTER_CLASS: str = conf.get_mandatory_value(
+ 'logging', 'LOG_FORMATTER_CLASS', fallback='airflow.utils.log.timezone_aware.TimezoneAware'
+)
+
COLORED_LOG_FORMAT: str = conf.get_mandatory_value('logging', 'COLORED_LOG_FORMAT')
COLORED_LOG: bool = conf.getboolean('logging', 'COLORED_CONSOLE_LOG')
@@ -60,10 +64,13 @@ DEFAULT_LOGGING_CONFIG: Dict[str, Any] = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
- 'airflow': {'format': LOG_FORMAT},
+ 'airflow': {
+ 'format': LOG_FORMAT,
+ 'class': LOG_FORMATTER_CLASS,
+ },
'airflow_coloured': {
'format': COLORED_LOG_FORMAT if COLORED_LOG else LOG_FORMAT,
- 'class': COLORED_FORMATTER_CLASS if COLORED_LOG else 'logging.Formatter',
+ 'class': COLORED_FORMATTER_CLASS if COLORED_LOG else LOG_FORMATTER_CLASS,
},
},
'filters': {
diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml
index f7409373c2..4ea090bcd9 100644
--- a/airflow/config_templates/config.yml
+++ b/airflow/config_templates/config.yml
@@ -625,6 +625,12 @@
type: string
example: ~
default: "%%(asctime)s %%(levelname)s - %%(message)s"
+ - name: log_formatter_class
+ description: ~
+ version_added: 2.3.4
+ type: string
+ example: ~
+ default: "airflow.utils.log.timezone_aware.TimezoneAware"
- name: task_log_prefix_template
description: |
Specify prefix pattern like mentioned below with stream handler TaskHandlerWithCustomFormatter
diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg
index 6591f82dc0..86f9cf93fc 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -345,6 +345,7 @@ colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatte
# Format of Log line
log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s
simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s
+log_formatter_class = airflow.utils.log.timezone_aware.TimezoneAware
# Specify prefix pattern like mentioned below with stream handler TaskHandlerWithCustomFormatter
# Example: task_log_prefix_template = {{ti.dag_id}}-{{ti.task_id}}-{{execution_date}}-{{try_number}}
diff --git a/airflow/utils/log/timezone_aware.py b/airflow/utils/log/timezone_aware.py
new file mode 100644
index 0000000000..d01205233a
--- /dev/null
+++ b/airflow/utils/log/timezone_aware.py
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import logging
+
+import pendulum
+
+
+class TimezoneAware(logging.Formatter):
+ """
+ Override `default_time_format`, `default_msec_format` and `formatTime` to specify utc offset.
+ utc offset is the matter, without it, time conversion could be wrong.
+ With this Formatter, `%(asctime)s` will be formatted containing utc offset. (ISO 8601)
+ (e.g. 2022-06-12T13:00:00.123+0000)
+ """
+
+ default_time_format = '%Y-%m-%dT%H:%M:%S'
+ default_msec_format = '%s.%03d'
+ default_tz_format = '%z'
+
+ def formatTime(self, record, datefmt=None):
+ """
+ Returns the creation time of the specified LogRecord in ISO 8601 date and time format
+ in the local time zone.
+ """
+ dt = pendulum.from_timestamp(record.created, tz=pendulum.local_timezone())
+ if datefmt:
+ s = dt.strftime(datefmt)
+ else:
+ s = dt.strftime(self.default_time_format)
+
+ if self.default_msec_format:
+ s = self.default_msec_format % (s, record.msecs)
+ if self.default_tz_format:
+ s += dt.strftime(self.default_tz_format)
+ return s
diff --git a/airflow/www/static/js/ti_log.js b/airflow/www/static/js/ti_log.js
index 1bf6b501a6..ae72837345 100644
--- a/airflow/www/static/js/ti_log.js
+++ b/airflow/www/static/js/ti_log.js
@@ -103,6 +103,7 @@ function autoTailingLog(tryNumber, metadata = null, autoTailing = false) {
// Detect urls and log timestamps
const urlRegex = /http(s)?:\/\/[\w.-]+(\.?:[\w.-]+)*([/?#][\w\-._~:/?#[\]@!$&'()*+,;=.%]+)?/g;
const dateRegex = /\d{4}[./-]\d{2}[./-]\d{2} \d{2}:\d{2}:\d{2},\d{3}/g;
+ const iso8601Regex = /\d{4}[./-]\d{2}[./-]\d{2}T\d{2}:\d{2}:\d{2}.\d{3}[+-]\d{4}/g;
res.message.forEach((item) => {
const logBlockElementId = `try-${tryNumber}-${item[0]}`;
@@ -120,7 +121,8 @@ function autoTailingLog(tryNumber, metadata = null, autoTailing = false) {
const escapedMessage = escapeHtml(item[1]);
const linkifiedMessage = escapedMessage
.replace(urlRegex, (url) => `<a href="${url}" target="_blank">${url}</a>`)
- .replaceAll(dateRegex, (date) => `<time datetime="${date}+00:00" data-with-tz="true">${formatDateTime(`${date}+00:00`)}</time>`);
+ .replaceAll(dateRegex, (date) => `<time datetime="${date}+00:00" data-with-tz="true">${formatDateTime(`${date}+00:00`)}</time>`)
+ .replaceAll(iso8601Regex, (date) => `<time datetime="${date}" data-with-tz="true">${formatDateTime(`${date}`)}</time>`);
logBlock.innerHTML += `${linkifiedMessage}\n`;
});
diff --git a/newsfragments/24811.significant.rst b/newsfragments/24811.significant.rst
new file mode 100644
index 0000000000..cb7208843c
--- /dev/null
+++ b/newsfragments/24811.significant.rst
@@ -0,0 +1,22 @@
+Added new config ``[logging]log_formatter_class`` to fix timezone display for logs on UI
+
+If you are using a custom Formatter subclass in your ``[logging]logging_config_class``, please inherit from ``airflow.utils.log.timezone_aware.TimezoneAware`` instead of ``logging.Formatter``.
+For example, in your ``custom_config.py``:
+
+.. code-block:: python
+ from airflow.utils.log.timezone_aware import TimezoneAware
+
+ # before
+ class YourCustomFormatter(logging.Formatter):
+ ...
+
+
+ # after
+ class YourCustomFormatter(TimezoneAware):
+ ...
+
+
+ AIRFLOW_FORMATTER = LOGGING_CONFIG["formatters"]["airflow"]
+ AIRFLOW_FORMATTER["class"] = "somewhere.your.custom_config.YourCustomFormatter"
+ # or use TimezoneAware class directly. If you don't have custom Formatter.
+ AIRFLOW_FORMATTER["class"] = "airflow.utils.log.timezone_aware.TimezoneAware"