You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by ch...@apache.org on 2016/04/28 16:41:28 UTC
incubator-toree git commit: Enabling auto convert,
Spark SQL imports for Spark 1.5, 1.6
Repository: incubator-toree
Updated Branches:
refs/heads/master b149da3cf -> 8828b7851
Enabling auto convert, Spark SQL imports for Spark 1.5, 1.6
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/8828b785
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/8828b785
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/8828b785
Branch: refs/heads/master
Commit: 8828b7851c0bf9ca09fe05b7fae0a7203a48228f
Parents: b149da3
Author: Eric Chang <ec...@fitbit.com>
Authored: Wed Apr 27 15:47:09 2016 -0700
Committer: Eric Chang <ec...@fitbit.com>
Committed: Wed Apr 27 16:56:48 2016 -0700
----------------------------------------------------------------------
.../src/main/resources/PySpark/pyspark_runner.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/8828b785/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
----------------------------------------------------------------------
diff --git a/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py b/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
index 52cf927..042de60 100644
--- a/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
+++ b/pyspark-interpreter/src/main/resources/PySpark/pyspark_runner.py
@@ -15,7 +15,7 @@
# limitations under the License.
#
-import sys, getopt, traceback
+import sys, getopt, traceback, re
from py4j.java_gateway import java_import, JavaGateway, GatewayClient
from py4j.protocol import Py4JJavaError
@@ -36,7 +36,7 @@ from pyspark.sql import SQLContext, HiveContext, SchemaRDD, Row
client = GatewayClient(port=int(sys.argv[1]))
sparkVersion = sys.argv[2]
-if sparkVersion.startswith("1.4"):
+if re.match("^1\.[456]\..*$", sparkVersion):
gateway = JavaGateway(client, auto_convert = True)
else:
gateway = JavaGateway(client)
@@ -61,7 +61,7 @@ if sparkVersion.startswith("1.2"):
elif sparkVersion.startswith("1.3"):
java_import(gateway.jvm, "org.apache.spark.sql.*")
java_import(gateway.jvm, "org.apache.spark.sql.hive.*")
-elif sparkVersion.startswith("1.4"):
+elif re.match("^1\.[456]\..*$", sparkVersion):
java_import(gateway.jvm, "org.apache.spark.sql.*")
java_import(gateway.jvm, "org.apache.spark.sql.hive.*")