You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bo...@apache.org on 2017/07/12 09:27:46 UTC
incubator-airflow git commit:
[AIRFLOW-1326][[AIRFLOW-1326][AIRFLOW-1184] Don't split argument array --
it's already an array.[
Repository: incubator-airflow
Updated Branches:
refs/heads/master 71594751c -> f5ffb9deb
[AIRFLOW-1326][[AIRFLOW-1326][AIRFLOW-1184] Don't split argument array -- it's already an array.[
Closes #2382 from ashb/spark-submit-operator-
preserve-spaces
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/f5ffb9de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/f5ffb9de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/f5ffb9de
Branch: refs/heads/master
Commit: f5ffb9deb51e896cdda8992c8b28102f7d650662
Parents: 7159475
Author: Ash Berlin-Taylor <as...@firemirror.com>
Authored: Wed Jul 12 11:27:39 2017 +0200
Committer: Bolke de Bruin <bo...@xs4all.nl>
Committed: Wed Jul 12 11:27:39 2017 +0200
----------------------------------------------------------------------
airflow/contrib/hooks/spark_submit_hook.py | 8 ++---
tests/contrib/hooks/test_spark_submit_hook.py | 6 ++--
.../operators/test_spark_submit_operator.py | 31 +++++++++++---------
3 files changed, 23 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/f5ffb9de/airflow/contrib/hooks/spark_submit_hook.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/hooks/spark_submit_hook.py b/airflow/contrib/hooks/spark_submit_hook.py
index f3d6e34..88d547b 100644
--- a/airflow/contrib/hooks/spark_submit_hook.py
+++ b/airflow/contrib/hooks/spark_submit_hook.py
@@ -194,12 +194,8 @@ class SparkSubmitHook(BaseHook):
# Append any application arguments
if self._application_args:
- for arg in self._application_args:
- if len(arg.split()) > 1:
- for splitted_option in arg.split():
- connection_cmd += [splitted_option]
- else:
- connection_cmd += [arg]
+ connection_cmd += self._application_args
+
logging.debug("Spark-Submit cmd: {}".format(connection_cmd))
return connection_cmd
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/f5ffb9de/tests/contrib/hooks/test_spark_submit_hook.py
----------------------------------------------------------------------
diff --git a/tests/contrib/hooks/test_spark_submit_hook.py b/tests/contrib/hooks/test_spark_submit_hook.py
index ef0857b..98e959b 100644
--- a/tests/contrib/hooks/test_spark_submit_hook.py
+++ b/tests/contrib/hooks/test_spark_submit_hook.py
@@ -45,8 +45,9 @@ class TestSparkSubmitHook(unittest.TestCase):
'driver_memory': '3g',
'java_class': 'com.foo.bar.AppMain',
'application_args': [
- '-f foo',
- '--bar bar',
+ '-f', 'foo',
+ '--bar', 'bar',
+ '--with-spaces', 'args should keep embdedded spaces',
'baz'
]
}
@@ -129,6 +130,7 @@ class TestSparkSubmitHook(unittest.TestCase):
'test_application.py',
'-f', 'foo',
'--bar', 'bar',
+ '--with-spaces', 'args should keep embdedded spaces',
'baz'
]
self.assertEquals(expected_build_cmd, cmd)
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/f5ffb9de/tests/contrib/operators/test_spark_submit_operator.py
----------------------------------------------------------------------
diff --git a/tests/contrib/operators/test_spark_submit_operator.py b/tests/contrib/operators/test_spark_submit_operator.py
index 09c5a93..1d41941 100644
--- a/tests/contrib/operators/test_spark_submit_operator.py
+++ b/tests/contrib/operators/test_spark_submit_operator.py
@@ -46,10 +46,11 @@ class TestSparkSubmitOperator(unittest.TestCase):
'driver_memory': '3g',
'java_class': 'com.foo.bar.AppMain',
'application_args': [
- '-f foo',
- '--bar bar',
- '--start {{ macros.ds_add(ds, -1)}}',
- '--end {{ ds }}'
+ '-f', 'foo',
+ '--bar', 'bar',
+ '--start', '{{ macros.ds_add(ds, -1)}}',
+ '--end', '{{ ds }}',
+ '--with-spaces', 'args should keep embdedded spaces',
]
}
@@ -95,10 +96,11 @@ class TestSparkSubmitOperator(unittest.TestCase):
'driver_memory': '3g',
'java_class': 'com.foo.bar.AppMain',
'application_args': [
- '-f foo',
- '--bar bar',
- '--start {{ macros.ds_add(ds, -1)}}',
- '--end {{ ds }}'
+ '-f', 'foo',
+ '--bar', 'bar',
+ '--start', '{{ macros.ds_add(ds, -1)}}',
+ '--end', '{{ ds }}',
+ '--with-spaces', 'args should keep embdedded spaces',
]
}
@@ -130,14 +132,15 @@ class TestSparkSubmitOperator(unittest.TestCase):
ti.render_templates()
# Then
- expected_application_args = [u'-f foo',
- u'--bar bar',
- u'--start %s' % (DEFAULT_DATE - datetime.timedelta(days=1)).strftime("%Y-%m-%d"),
- u'--end %s' % DEFAULT_DATE.strftime("%Y-%m-%d")]
+ expected_application_args = [u'-f', 'foo',
+ u'--bar', 'bar',
+ u'--start', (DEFAULT_DATE - datetime.timedelta(days=1)).strftime("%Y-%m-%d"),
+ u'--end', DEFAULT_DATE.strftime("%Y-%m-%d"),
+ u'--with-spaces', u'args should keep embdedded spaces',
+ ]
expected_name = "spark_submit_job"
- self.assertListEqual(sorted(expected_application_args), sorted(getattr(operator, '_application_args')))
+ self.assertListEqual(expected_application_args, getattr(operator, '_application_args'))
self.assertEqual(expected_name, getattr(operator, '_name'))
-
if __name__ == '__main__':
unittest.main()