You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2016/11/25 19:22:08 UTC
zeppelin git commit: Fix Yarn configuration of Spark2.0
Repository: zeppelin
Updated Branches:
refs/heads/master 95895d1fc -> e29608ff3
Fix Yarn configuration of Spark2.0
### What is this PR for?
`yarn-client` is deprecated since Spark 2.0, so SparkInterpreter also support `yarn` as `application master`.
### What type of PR is it?
Bug Fix | Improvement
### How should this be tested?
Run spark paragraph with Spark2.0 + yarn.
### Questions:
* Does the licenses files need update? no
* Is there breaking changes for older versions? no
* Does this needs documentation? no
Author: astroshim <hs...@nflabs.com>
Closes #1656 from astroshim/fix/yarn-spark2.0 and squashes the following commits:
b993628 [astroshim] if yarn mode
4374ce0 [astroshim] fix spark configuration
b225dd4 [astroshim] add master yarn
93f7f9c [astroshim] update spark conf
a4f3fa3 [astroshim] fix spark2.0 connection error with EMR
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/e29608ff
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/e29608ff
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/e29608ff
Branch: refs/heads/master
Commit: e29608ff399ae6238db7086c214da0cc5c45a0fd
Parents: 95895d1
Author: astroshim <hs...@nflabs.com>
Authored: Thu Nov 24 00:40:47 2016 +0900
Committer: Lee moon soo <mo...@apache.org>
Committed: Fri Nov 25 11:22:03 2016 -0800
----------------------------------------------------------------------
.../org/apache/zeppelin/spark/SparkInterpreter.java | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/e29608ff/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
index 1cd6c71..e8c76bc 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
@@ -296,6 +296,10 @@ public class SparkInterpreter extends Interpreter {
return (DepInterpreter) p;
}
+ private boolean isYarnMode() {
+ return getProperty("master").startsWith("yarn");
+ }
+
/**
* Spark 2.x
* Create SparkSession
@@ -319,6 +323,10 @@ public class SparkInterpreter extends Interpreter {
conf.set("spark.scheduler.mode", "FAIR");
conf.setMaster(getProperty("master"));
+ if (isYarnMode()) {
+ conf.set("master", "yarn");
+ conf.set("spark.submit.deployMode", "client");
+ }
Properties intpProperty = getProperty();
@@ -510,7 +518,7 @@ public class SparkInterpreter extends Interpreter {
// Distributes needed libraries to workers
// when spark version is greater than or equal to 1.5.0
- if (getProperty("master").equals("yarn-client")) {
+ if (isYarnMode()) {
conf.set("spark.yarn.isPython", "true");
}
}
@@ -559,7 +567,7 @@ public class SparkInterpreter extends Interpreter {
@Override
public void open() {
// set properties and do login before creating any spark stuff for secured cluster
- if (getProperty("master").equals("yarn-client")) {
+ if (isYarnMode()) {
System.setProperty("SPARK_YARN_MODE", "true");
}
if (getProperty().containsKey("spark.yarn.keytab") &&