You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by sa...@apache.org on 2017/06/23 01:38:09 UTC
incubator-airflow git commit: [AIRFLOW-1335] Use MemoryHandler for
buffered logging
Repository: incubator-airflow
Updated Branches:
refs/heads/master e0491bcb2 -> 0d23d30d6
[AIRFLOW-1335] Use MemoryHandler for buffered logging
Closes #2386 from saguziel/aguziel-buffer-logger-
apache
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/0d23d30d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/0d23d30d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/0d23d30d
Branch: refs/heads/master
Commit: 0d23d30d6812fe1eb3cfb52d2992131cbf028062
Parents: e0491bc
Author: Alex Guziel <al...@airbnb.com>
Authored: Thu Jun 22 18:38:05 2017 -0700
Committer: Alex Guziel <al...@airbnb.com>
Committed: Thu Jun 22 18:38:05 2017 -0700
----------------------------------------------------------------------
airflow/bin/cli.py | 5 ++++-
airflow/config_templates/default_airflow.cfg | 7 +++++++
airflow/config_templates/default_test.cfg | 2 ++
3 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0d23d30d/airflow/bin/cli.py
----------------------------------------------------------------------
diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py
index 4b3a0ed..d4ddcb5 100755
--- a/airflow/bin/cli.py
+++ b/airflow/bin/cli.py
@@ -90,7 +90,10 @@ def sigquit_handler(sig, frame):
def setup_logging(filename):
root = logging.getLogger()
- handler = logging.FileHandler(filename)
+ handler_buffer_size = conf.getInt('core', 'logging_buffer_size')
+ handler_flush_level = conf.getInt('core', 'logging_flush_level')
+ # Buffered wrapper
+ handler = logging.MemoryHandler(handler_buffer_size, handler_flush_level, target=logging.FileHandler(filename))
formatter = logging.Formatter(settings.SIMPLE_LOG_FORMAT)
handler.setFormatter(formatter)
root.addHandler(handler)
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0d23d30d/airflow/config_templates/default_airflow.cfg
----------------------------------------------------------------------
diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg
index c6c1da2..13a4b20 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -49,6 +49,13 @@ s3_log_folder =
# Logging level
logging_level = INFO
+# Size of logging buffer
+logging_buffer_size = 4096
+# Logging level minimum to flush buffer before full
+# See python doc on core module logging, DEBUG=10,INFO=20,WARN=30,ERROR=40
+# default to no buffering
+logging_flush_level = 0
+
# The executor class that airflow should use. Choices include
# SequentialExecutor, LocalExecutor, CeleryExecutor, DaskExecutor
executor = SequentialExecutor
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0d23d30d/airflow/config_templates/default_test.cfg
----------------------------------------------------------------------
diff --git a/airflow/config_templates/default_test.cfg b/airflow/config_templates/default_test.cfg
index 2fb5bb0..1c0d34c 100644
--- a/airflow/config_templates/default_test.cfg
+++ b/airflow/config_templates/default_test.cfg
@@ -29,6 +29,8 @@ dags_folder = {TEST_DAGS_FOLDER}
plugins_folder = {TEST_PLUGINS_FOLDER}
base_log_folder = {AIRFLOW_HOME}/logs
logging_level = INFO
+logging_flush_level = 0
+logging_buffer_size = 4096
executor = SequentialExecutor
sql_alchemy_conn = sqlite:///{AIRFLOW_HOME}/unittests.db
load_examples = True