zeppelin git commit: ZEPPELIN-3531. Don't look for py4j in NewSparkInterpreter

[zj...@apache.org]

Repository: zeppelin
Updated Branches:
  refs/heads/master 6fc3057d5 -> 767a50b41


ZEPPELIN-3531. Don't look for py4j in NewSparkInterpreter

### What is this PR for?
Just remove setupConfForPySpark in NewSparkInterpreter as it is not necessary and will cause NPE when the node launch spark interpreter doesn't have spark installed in yarn cluster mode.

### What type of PR is it?
[Bug Fix]

### Todos
* [ ] - Task

### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-3531

### How should this be tested?
* CI pass & Manually tested in a 3 node cluster

### Screenshots (if appropriate)

### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No

Author: Jeff Zhang <zj...@apache.org>

Closes #3008 from zjffdu/ZEPPELIN-3531 and squashes the following commits:

e8b2969e9 [Jeff Zhang] ZEPPELIN-3531. Don't look for py4j in NewSparkInterpreter


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/767a50b4
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/767a50b4
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/767a50b4

Branch: refs/heads/master
Commit: 767a50b4164ef323ef18160485d226c013c7c00d
Parents: 6fc3057
Author: Jeff Zhang <zj...@apache.org>
Authored: Tue Jun 5 21:38:17 2018 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Thu Jun 7 16:19:14 2018 +0800

----------------------------------------------------------------------
 .../zeppelin/spark/NewSparkInterpreter.java     | 43 --------------------
 1 file changed, 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/767a50b4/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java
----------------------------------------------------------------------
diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java
index b77e1a7..fcfee27 100644
--- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java
+++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/NewSparkInterpreter.java
@@ -81,7 +81,6 @@ public class NewSparkInterpreter extends AbstractSparkInterpreter {
     try {
       String scalaVersion = extractScalaVersion();
       LOGGER.info("Using Scala Version: " + scalaVersion);
-      setupConfForPySpark();
       SparkConf conf = new SparkConf();
       for (Map.Entry<Object, Object> entry : getProperties().entrySet()) {
         if (!StringUtils.isBlank(entry.getValue().toString())) {
@@ -126,48 +125,6 @@ public class NewSparkInterpreter extends AbstractSparkInterpreter {
     }
   }
 
-  private void setupConfForPySpark() {
-    String sparkHome = getProperty("SPARK_HOME");
-    File pysparkFolder = null;
-    if (sparkHome == null) {
-      String zeppelinHome =
-          new DefaultInterpreterProperty("ZEPPELIN_HOME", "zeppelin.home", "../../")
-              .getValue().toString();
-      pysparkFolder = new File(zeppelinHome,
-          "interpreter" + File.separator + "spark" + File.separator + "pyspark");
-    } else {
-      pysparkFolder = new File(sparkHome, "python" + File.separator + "lib");
-    }
-
-    ArrayList<String> pysparkPackages = new ArrayList<>();
-    for (File file : pysparkFolder.listFiles()) {
-      if (file.getName().equals("pyspark.zip")) {
-        pysparkPackages.add(file.getAbsolutePath());
-      }
-      if (file.getName().startsWith("py4j-")) {
-        pysparkPackages.add(file.getAbsolutePath());
-      }
-    }
-
-    if (pysparkPackages.size() != 2) {
-      throw new RuntimeException("Not correct number of pyspark packages: " +
-          StringUtils.join(pysparkPackages, ","));
-    }
-    // Distribute two libraries(pyspark.zip and py4j-*.zip) to workers
-    System.setProperty("spark.files", mergeProperty(System.getProperty("spark.files", ""),
-        StringUtils.join(pysparkPackages, ",")));
-    System.setProperty("spark.submit.pyFiles", mergeProperty(
-        System.getProperty("spark.submit.pyFiles", ""), StringUtils.join(pysparkPackages, ",")));
-
-  }
-
-  private String mergeProperty(String originalValue, String appendedValue) {
-    if (StringUtils.isBlank(originalValue)) {
-      return appendedValue;
-    }
-    return originalValue + "," + appendedValue;
-  }
-
   @Override
   public void close() {
     LOGGER.info("Close SparkInterpreter");