You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2015/04/01 18:01:20 UTC
incubator-zeppelin git commit: [ZEPPELIN-17] PySpark Interpreter
should allow starting with a specific version of Python
Repository: incubator-zeppelin
Updated Branches:
refs/heads/master c7645abee -> c9db780a1
[ZEPPELIN-17] PySpark Interpreter should allow starting with a specific version of Python
Add PYSPARK_PYTHON.
We could also add PYSPARK_DRIVER_PYTHON (Zeppelin Interpreter is the driver) but it doesn't seem to be documented, and more importantly running different Python between driver and worker can cause errors.
http://spark.apache.org/docs/1.3.0/configuration.html
http://spark.apache.org/docs/1.3.0/configuration.html#environment-variables
Author: Felix Cheung <fe...@hotmail.com>
Closes #19 from felixcheung/master and squashes the following commits:
e89ba08 [Felix Cheung] PySpark Interpreter should allow starting with a specific version of Python, as PySpark does.
65ba046 [Felix Cheung] Merge commit 'a007a9b5f235ebd9c608a005c5243503291d94d5'
7a30a14 [Felix Cheung] minor doc update for running on YARN
Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/c9db780a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/c9db780a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/c9db780a
Branch: refs/heads/master
Commit: c9db780a1744aa3900c6d08f2727187731839cd3
Parents: c7645ab
Author: Felix Cheung <fe...@hotmail.com>
Authored: Tue Mar 31 13:24:16 2015 -0700
Committer: Lee moon soo <mo...@apache.org>
Committed: Thu Apr 2 01:01:11 2015 +0900
----------------------------------------------------------------------
.../java/com/nflabs/zeppelin/spark/PySparkInterpreter.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/c9db780a/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java b/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java
index 071fcea..36cdcae 100644
--- a/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java
+++ b/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java
@@ -64,7 +64,10 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
new InterpreterPropertyBuilder()
.add("spark.home",
SparkInterpreter.getSystemDefault("SPARK_HOME", "spark.home", ""),
- "Spark home path. Should be provided for pyspark").build());
+ "Spark home path. Should be provided for pyspark")
+ .add("zeppelin.pyspark.python",
+ SparkInterpreter.getSystemDefault("PYSPARK_PYTHON", null, "python"),
+ "Python command to run pyspark with").build());
}
public PySparkInterpreter(Properties property) {
@@ -115,7 +118,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
gatewayServer.start();
// Run python shell
- CommandLine cmd = CommandLine.parse("python");
+ CommandLine cmd = CommandLine.parse(getProperty("zeppelin.pyspark.python"));
cmd.addArgument(scriptPath, false);
cmd.addArgument(Integer.toString(port), false);
executor = new DefaultExecutor();