You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bo...@apache.org on 2017/07/23 18:30:02 UTC
incubator-airflow git commit: [AIRFLOW-1399] Fix cli reading logfile
in memory
Repository: incubator-airflow
Updated Branches:
refs/heads/master aa64f370b -> 2de4b7cfb
[AIRFLOW-1399] Fix cli reading logfile in memory
Don't read logfile in memory if remote base isn't
specified
Closes #2433 from
NielsZeilemaker/no_remote_base_no_read
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/2de4b7cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/2de4b7cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/2de4b7cf
Branch: refs/heads/master
Commit: 2de4b7cfb12f5a36eeaf5e78d3ee0fb12d67f3b2
Parents: aa64f37
Author: Niels Zeilemaker <ni...@godatadriven.com>
Authored: Sun Jul 23 20:29:41 2017 +0200
Committer: Bolke de Bruin <bo...@xs4all.nl>
Committed: Sun Jul 23 20:29:47 2017 +0200
----------------------------------------------------------------------
airflow/bin/cli.py | 11 +++++++++--
tests/core.py | 26 ++++++++++++++++++++++++++
2 files changed, 35 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/2de4b7cf/airflow/bin/cli.py
----------------------------------------------------------------------
diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py
index a8543d3..de8e9f3 100755
--- a/airflow/bin/cli.py
+++ b/airflow/bin/cli.py
@@ -461,6 +461,10 @@ def run(args, dag=None):
logging.root.handlers = []
# store logs remotely
+ _store_logs_remotely(log_base, filename)
+
+
+def _store_logs_remotely(log_base, filename):
remote_base = conf.get('core', 'REMOTE_BASE_LOG_FOLDER')
# deprecated as of March 2016
@@ -472,7 +476,10 @@ def run(args, dag=None):
DeprecationWarning)
remote_base = conf.get('core', 'S3_LOG_FOLDER')
- if os.path.exists(filename):
+ if remote_base == 'None':
+ remote_base = None
+
+ if remote_base and os.path.exists(filename):
# read log and remove old logs to get just the latest additions
with open(filename, 'r') as logfile:
@@ -487,7 +494,7 @@ def run(args, dag=None):
elif remote_base.startswith('gs:/'):
logging_utils.GCSLog().write(log, remote_log_location)
# Other
- elif remote_base and remote_base != 'None':
+ else:
logging.error(
'Unsupported remote log location: {}'.format(remote_base))
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/2de4b7cf/tests/core.py
----------------------------------------------------------------------
diff --git a/tests/core.py b/tests/core.py
index 923e0c3..4e81534 100644
--- a/tests/core.py
+++ b/tests/core.py
@@ -1326,6 +1326,32 @@ class CliTests(unittest.TestCase):
'run', 'example_bash_operator', 'runme_0', '-l',
DEFAULT_DATE.isoformat()]))
+ def test_cli_store_logs_remotely_no_remote_base(self):
+ with mock.patch('__main__.open', mock.mock_open(read_data='42'), create=True) as open_mock:
+ with mock.patch('os.path.exists') as path_mock:
+ path_mock.return_value = True
+
+ cli._store_logs_remotely("42", "existing_file")
+
+ # remote base not specified, hence no call to open
+ self.assertEqual(open_mock.call_count, 0)
+
+ def test_cli_store_logs_remotely_with_remote_base(self):
+ orig_base_log_folder = configuration.get('core', 'REMOTE_BASE_LOG_FOLDER')
+ configuration.set("core", "REMOTE_BASE_LOG_FOLDER", "42")
+
+ try:
+ with mock.patch('airflow.bin.cli.open', mock.mock_open(read_data='42'), create=True) as open_mock:
+ with mock.patch('os.path.exists') as path_mock:
+ path_mock.return_value = True
+
+ cli._store_logs_remotely("42", "existing_file")
+
+ # remote base specified, hence one call to open
+ self.assertEqual(open_mock.call_count, 1)
+ finally:
+ configuration.set("core", "REMOTE_BASE_LOG_FOLDER", orig_base_log_folder)
+
def test_task_state(self):
cli.task_state(self.parser.parse_args([
'task_state', 'example_bash_operator', 'runme_0',