You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bo...@apache.org on 2016/10/05 10:58:28 UTC
incubator-airflow git commit: [AIRFLOW-378] Add string casting to
params of spark-sql operator
Repository: incubator-airflow
Updated Branches:
refs/heads/master 2aaa629a7 -> 573fb991f
[AIRFLOW-378] Add string casting to params of spark-sql operator
For parameters num_executors and executor_cores
add casts to strings to prevent issues when these
parameters are passed as integers (as comments specify).
Also fix minor typo that breaks the use of num-executors param.
Closes #1694 from
danielvdende/spark_sql_operator_bugfixes
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/573fb991
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/573fb991
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/573fb991
Branch: refs/heads/master
Commit: 573fb991f4c7cecd1244be6bd687c1360f790a44
Parents: 2aaa629
Author: Daniel van der Ende <da...@gmail.com>
Authored: Wed Oct 5 12:56:40 2016 +0200
Committer: Bolke de Bruin <bo...@xs4all.nl>
Committed: Wed Oct 5 12:56:52 2016 +0200
----------------------------------------------------------------------
airflow/contrib/hooks/spark_sql_hook.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/573fb991/airflow/contrib/hooks/spark_sql_hook.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/hooks/spark_sql_hook.py b/airflow/contrib/hooks/spark_sql_hook.py
index ff5270b..d263cc0 100644
--- a/airflow/contrib/hooks/spark_sql_hook.py
+++ b/airflow/contrib/hooks/spark_sql_hook.py
@@ -90,13 +90,13 @@ class SparkSqlHook(BaseHook):
for conf_el in self._conf.split(","):
connection_cmd += ["--conf", conf_el]
if self._executor_cores:
- connection_cmd += ["--executor-cores", self._executor_cores]
+ connection_cmd += ["--executor-cores", str(self._executor_cores)]
if self._executor_memory:
connection_cmd += ["--executor-memory", self._executor_memory]
if self._keytab:
connection_cmd += ["--keytab", self._keytab]
if self._num_executors:
- connection_cmd += ["--num_executors", self._num_executors]
+ connection_cmd += ["--num-executors", str(self._num_executors)]
if self._sql:
if self._sql.endswith('.sql'):
connection_cmd += ["-f", self._sql]