You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by fe...@apache.org on 2017/01/16 17:28:44 UTC
zeppelin git commit: ZEPPELIN-1918. Fix build with Spark 2.1.0
Repository: zeppelin
Updated Branches:
refs/heads/master ae1cb0527 -> 9bc4dce98
ZEPPELIN-1918. Fix build with Spark 2.1.0
### What is this PR for?
It's my misunderstanding of `SPARK_VER` in travis. It is only used for downloading spark distribution. We need to specify spark profile explicitly for building with specific version of spark. This PR add new profile for spark 2.1 and fix the build issue with spark 2.1.0 because `SecurityManager` changes its constructor signature in spark 2.1
### What type of PR is it?
[Bug Fix ]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-1918
### How should this be tested?
Build with spark 2.1.0 and tested it manually as below screenshot.
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #1880 from zjffdu/ZEPPELIN-1918 and squashes the following commits:
8f87459 [Jeff Zhang] update release script
34772e0 [Jeff Zhang] update build doc for spark 2.1
5efdb11 [Jeff Zhang] fix unit test and address comments
def502f [Jeff Zhang] ZEPPELIN-1918. Fix build with Spark 2.1.0
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/9bc4dce9
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/9bc4dce9
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/9bc4dce9
Branch: refs/heads/master
Commit: 9bc4dce9884ec4011f677560f708286968549374
Parents: ae1cb05
Author: Jeff Zhang <zj...@apache.org>
Authored: Wed Jan 11 10:56:53 2017 +0800
Committer: Felix Cheung <fe...@apache.org>
Committed: Mon Jan 16 09:28:41 2017 -0800
----------------------------------------------------------------------
.travis.yml | 8 ++--
dev/create_release.sh | 4 +-
dev/publish_release.sh | 2 +-
docs/install/build.md | 5 +++
spark-dependencies/pom.xml | 14 ++++++-
spark/pom.xml | 14 ++++++-
.../apache/zeppelin/spark/SparkInterpreter.java | 41 ++++++++++++++++++--
7 files changed, 73 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9bc4dce9/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 48e8aa7..5b3371d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -40,13 +40,13 @@ matrix:
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
- # Test all modules with spark 2.0.2 and scala 2.11
+ # Test all modules with spark 2.1.0 and scala 2.11
- jdk: "oraclejdk7"
- env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
+ env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.1 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
- # Test all modules with spark 2.1.0 and scala 2.11
+ # Test all modules with spark 2.0.2 and scala 2.11
- jdk: "oraclejdk7"
- env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
+ env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" TEST_PROJECTS=""
# Test all modules with scala 2.10
- jdk: "oraclejdk7"
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9bc4dce9/dev/create_release.sh
----------------------------------------------------------------------
diff --git a/dev/create_release.sh b/dev/create_release.sh
index 9b0b231..873bb10 100755
--- a/dev/create_release.sh
+++ b/dev/create_release.sh
@@ -124,8 +124,8 @@ function make_binary_release() {
build_docker_base
git_clone
make_source_package
-make_binary_release all "-Pspark-2.0 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pscala-${SCALA_VERSION}"
-make_binary_release netinst "-Pspark-2.0 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am"
+make_binary_release all "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pscala-${SCALA_VERSION}"
+make_binary_release netinst "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am"
# remove non release files and dirs
rm -rf "${WORKING_DIR}/zeppelin"
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9bc4dce9/dev/publish_release.sh
----------------------------------------------------------------------
diff --git a/dev/publish_release.sh b/dev/publish_release.sh
index 7b99f39..a2acc98 100755
--- a/dev/publish_release.sh
+++ b/dev/publish_release.sh
@@ -44,7 +44,7 @@ NC='\033[0m' # No Color
RELEASE_VERSION="$1"
GIT_TAG="$2"
-PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.0 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pr"
+PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.1 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pr"
PROJECT_OPTIONS="-pl !zeppelin-distribution"
NEXUS_STAGING="https://repository.apache.org/service/local/staging"
NEXUS_PROFILE="153446d1ac37c4"
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9bc4dce9/docs/install/build.md
----------------------------------------------------------------------
diff --git a/docs/install/build.md b/docs/install/build.md
index f3012a5..e452559 100644
--- a/docs/install/build.md
+++ b/docs/install/build.md
@@ -97,6 +97,7 @@ Set spark major version
Available profiles are
```
+-Pspark-2.1
-Pspark-2.0
-Pspark-1.6
-Pspark-1.5
@@ -185,6 +186,10 @@ Bulid examples under zeppelin-examples directory
Here are some examples with several options:
```bash
+# build with spark-2.1, scala-2.11
+./dev/change_scala_version.sh 2.11
+mvn clean package -Pspark-2.1 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pscala-2.11 -DskipTests
+
# build with spark-2.0, scala-2.11
./dev/change_scala_version.sh 2.11
mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pscala-2.11 -DskipTests
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9bc4dce9/spark-dependencies/pom.xml
----------------------------------------------------------------------
diff --git a/spark-dependencies/pom.xml b/spark-dependencies/pom.xml
index 198f2bf..887964d 100644
--- a/spark-dependencies/pom.xml
+++ b/spark-dependencies/pom.xml
@@ -523,13 +523,23 @@
<profile>
<id>spark-2.0</id>
+ <properties>
+ <spark.version>2.0.2</spark.version>
+ <protobuf.version>2.5.0</protobuf.version>
+ <py4j.version>0.10.3</py4j.version>
+ <scala.version>2.11.8</scala.version>
+ </properties>
+ </profile>
+
+ <profile>
+ <id>spark-2.1</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
- <spark.version>2.0.2</spark.version>
+ <spark.version>2.1.0</spark.version>
<protobuf.version>2.5.0</protobuf.version>
- <py4j.version>0.10.3</py4j.version>
+ <py4j.version>0.10.4</py4j.version>
<scala.version>2.11.8</scala.version>
</properties>
</profile>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9bc4dce9/spark/pom.xml
----------------------------------------------------------------------
diff --git a/spark/pom.xml b/spark/pom.xml
index 54690ea..4989af9 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -515,13 +515,23 @@
<profile>
<id>spark-2.0</id>
+ <properties>
+ <spark.version>2.0.2</spark.version>
+ <protobuf.version>2.5.0</protobuf.version>
+ <py4j.version>0.10.3</py4j.version>
+ <scala.version>2.11.8</scala.version>
+ </properties>
+ </profile>
+
+ <profile>
+ <id>spark-2.1</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
- <spark.version>2.0.2</spark.version>
+ <spark.version>2.1.0</spark.version>
<protobuf.version>2.5.0</protobuf.version>
- <py4j.version>0.10.3</py4j.version>
+ <py4j.version>0.10.4</py4j.version>
<scala.version>2.11.8</scala.version>
</properties>
</profile>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9bc4dce9/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
index 7882303..0584a30 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
@@ -488,8 +488,9 @@ public class SparkInterpreter extends Interpreter {
}
//Only one of py4j-0.9-src.zip and py4j-0.8.2.1-src.zip should exist
+ //TODO(zjffdu), this is not maintainable when new version is added.
String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.9-src.zip", "py4j-0.8.2.1-src.zip",
- "py4j-0.10.1-src.zip", "py4j-0.10.3-src.zip"};
+ "py4j-0.10.1-src.zip", "py4j-0.10.3-src.zip", "py4j-0.10.4-src.zip"};
ArrayList<String> pythonLibUris = new ArrayList<>();
for (String lib : pythonLibs) {
File libFile = new File(pysparkPath, lib);
@@ -1452,8 +1453,10 @@ public class SparkInterpreter extends Interpreter {
.getConstructor(new Class[]{
SparkConf.class, File.class, SecurityManager.class, int.class, String.class});
- return constructor.newInstance(new Object[] {
- conf, outputDir, new SecurityManager(conf), 0, "HTTP Server"});
+ Object securityManager = createSecurityManager(conf);
+ return constructor.newInstance(new Object[]{
+ conf, outputDir, securityManager, 0, "HTTP Server"});
+
} catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException |
InstantiationException | InvocationTargetException e) {
// fallback to old constructor
@@ -1464,7 +1467,7 @@ public class SparkInterpreter extends Interpreter {
.getConstructor(new Class[]{
File.class, SecurityManager.class, int.class, String.class});
return constructor.newInstance(new Object[] {
- outputDir, new SecurityManager(conf), 0, "HTTP Server"});
+ outputDir, createSecurityManager(conf), 0, "HTTP Server"});
} catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException |
InstantiationException | InvocationTargetException e1) {
logger.error(e1.getMessage(), e1);
@@ -1472,4 +1475,34 @@ public class SparkInterpreter extends Interpreter {
}
}
}
+
+ /**
+ * Constructor signature of SecurityManager changes in spark 2.1.0, so we use this method to
+ * create SecurityManager properly for different versions of spark
+ *
+ * @param conf
+ * @return
+ * @throws ClassNotFoundException
+ * @throws NoSuchMethodException
+ * @throws IllegalAccessException
+ * @throws InvocationTargetException
+ * @throws InstantiationException
+ */
+ private Object createSecurityManager(SparkConf conf) throws ClassNotFoundException,
+ NoSuchMethodException, IllegalAccessException, InvocationTargetException,
+ InstantiationException {
+ Object securityManager = null;
+ try {
+ Constructor<?> smConstructor = getClass().getClassLoader()
+ .loadClass("org.apache.spark.SecurityManager")
+ .getConstructor(new Class[]{ SparkConf.class, scala.Option.class });
+ securityManager = smConstructor.newInstance(conf, null);
+ } catch (NoSuchMethodException e) {
+ Constructor<?> smConstructor = getClass().getClassLoader()
+ .loadClass("org.apache.spark.SecurityManager")
+ .getConstructor(new Class[]{ SparkConf.class });
+ securityManager = smConstructor.newInstance(conf);
+ }
+ return securityManager;
+ }
}