You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ms...@apache.org on 2017/10/23 09:57:46 UTC
incubator-airflow git commit: [AIRFLOW-1657] Handle failing qubole
operator
Repository: incubator-airflow
Updated Branches:
refs/heads/master 4386cd414 -> 263861ed2
[AIRFLOW-1657] Handle failing qubole operator
Closes #2643 from rupesh92/AIRFLOW-1657
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/263861ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/263861ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/263861ed
Branch: refs/heads/master
Commit: 263861ed2ccefc21665697af4903bdb021431dea
Parents: 4386cd4
Author: Rupesh Bansal <ru...@qubole.com>
Authored: Mon Oct 23 14:48:30 2017 +0530
Committer: Sumit Maheshwari <su...@gmail.com>
Committed: Mon Oct 23 14:48:30 2017 +0530
----------------------------------------------------------------------
airflow/contrib/hooks/qubole_hook.py | 5 +----
tests/contrib/operators/test_qubole_operator.py | 15 ++++++++++++++-
2 files changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/263861ed/airflow/contrib/hooks/qubole_hook.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/hooks/qubole_hook.py b/airflow/contrib/hooks/qubole_hook.py
index 833c1c7..f3bcc20 100755
--- a/airflow/contrib/hooks/qubole_hook.py
+++ b/airflow/contrib/hooks/qubole_hook.py
@@ -202,9 +202,6 @@ class QuboleHook(BaseHook, LoggingMixin):
args.append("--tags={0}".format(','.join(filter(None,tags))))
if inplace_args is not None:
- if cmd_type == 'hadoopcmd':
- args += inplace_args.split(' ', 1)
- else:
- args += inplace_args.split(' ')
+ args += inplace_args.split(' ')
return args
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/263861ed/tests/contrib/operators/test_qubole_operator.py
----------------------------------------------------------------------
diff --git a/tests/contrib/operators/test_qubole_operator.py b/tests/contrib/operators/test_qubole_operator.py
index 0e6e13d..9b5bf3a 100644
--- a/tests/contrib/operators/test_qubole_operator.py
+++ b/tests/contrib/operators/test_qubole_operator.py
@@ -89,6 +89,19 @@ class QuboleOperatorTest(unittest.TestCase):
self.assertEqual(task.get_hook().create_cmd_args({'run_id':'dummy'})[2],
"key2=value2")
-
+ task = QuboleOperator(task_id=TASK_ID, command_type='hadoopcmd',
+ sub_command="s3distcp --src s3n://airflow/source_hadoopcmd " +
+ "--dest s3n://airflow/destination_hadoopcmd", dag=dag)
+
+ self.assertEqual(task.get_hook().create_cmd_args({'run_id': 'dummy'})[1],
+ "s3distcp")
+ self.assertEqual(task.get_hook().create_cmd_args({'run_id': 'dummy'})[2],
+ "--src")
+ self.assertEqual(task.get_hook().create_cmd_args({'run_id': 'dummy'})[3],
+ "s3n://airflow/source_hadoopcmd")
+ self.assertEqual(task.get_hook().create_cmd_args({'run_id': 'dummy'})[4],
+ "--dest")
+ self.assertEqual(task.get_hook().create_cmd_args({'run_id': 'dummy'})[5],
+ "s3n://airflow/destination_hadoopcmd")