You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2015/05/23 07:39:19 UTC
incubator-zeppelin git commit: Spark 1.4 support
Repository: incubator-zeppelin
Updated Branches:
refs/heads/master a911b0ec4 -> 928d09987
Spark 1.4 support
This PR enables Zeppelin work with upcoming Spark 1.4
to test, install spark 1.4 maven artifacts into your local .m2 repository by doing
```
sbt/sbt publish-local
```
in spark-1.4 source code, and then build Zeppelin
```
mvn clean package -Pspark-1.4 -Dspark.version=1.4.0 -DskipTests
```
I have a simple screenshot
Author: Lee moon soo <mo...@apache.org>
Closes #76 from Leemoonsoo/spark-1.4 and squashes the following commits:
77cb7b9 [Lee moon soo] Spark 1.4 support
Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/928d0998
Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/928d0998
Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/928d0998
Branch: refs/heads/master
Commit: 928d09987b0b59676d82bbfe336d1688ef01d9bc
Parents: a911b0e
Author: Lee moon soo <mo...@apache.org>
Authored: Wed May 20 13:34:27 2015 +0900
Committer: Lee moon soo <mo...@apache.org>
Committed: Sat May 23 14:39:11 2015 +0900
----------------------------------------------------------------------
README.md | 4 ++
pom.xml | 60 +++++++++++++++++++-
spark/pom.xml | 1 -
.../zeppelin/spark/PySparkInterpreter.java | 5 +-
.../apache/zeppelin/spark/SparkInterpreter.java | 10 ++++
.../zeppelin/spark/SparkSqlInterpreter.java | 2 +
6 files changed, 77 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/928d0998/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 46e188c..44f50ee 100644
--- a/README.md
+++ b/README.md
@@ -54,6 +54,10 @@ Spark 1.3.x
```
mvn clean package -Pspark-1.3 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests
```
+Spark 1.4.x
+```
+mvn clean package -Pspark-1.4 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests
+```
CDH 5.X
```
mvn clean package -Pspark-1.2 -Dhadoop.version=2.5.0-cdh5.3.0 -Phadoop-2.4 -DskipTests
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/928d0998/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b4f78df..e43ccab 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,6 +126,7 @@
<jets3t.version>0.7.1</jets3t.version>
<commons.math3.version>3.2</commons.math3.version>
<commons.httpclient.version>4.3.6</commons.httpclient.version>
+ <fasterxml.jackson.version>2.4.4</fasterxml.jackson.version>
<codehaus.jackson.version>1.8.8</codehaus.jackson.version>
<snappy.version>1.0.5</snappy.version>
<io.netty.version>4.0.17.Final</io.netty.version>
@@ -812,6 +813,26 @@
</dependency>
<dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${fasterxml.jackson.version}</version>
+ </dependency>
+
+ <!-- Guava is excluded because of SPARK-6149. The Guava version referenced in this module is
+ 15.0, which causes runtime incompatibility issues. -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.module</groupId>
+ <artifactId>jackson-module-scala_2.10</artifactId>
+ <version>${fasterxml.jackson.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
<groupId>com.thoughtworks.paranamer</groupId>
<artifactId>paranamer</artifactId>
<version>2.6</version>
@@ -1317,8 +1338,6 @@
<profile>
<id>spark-1.3</id>
- <dependencies>
- </dependencies>
<properties>
<akka.version>2.3.4-spark</akka.version>
<spark.version>1.3.0</spark.version>
@@ -1343,9 +1362,46 @@
<snappy.version>1.1.1.6</snappy.version>
<mesos.version>0.21.0</mesos.version>
</properties>
+
+ <dependencies>
+ </dependencies>
+
</profile>
<profile>
+ <id>spark-1.4</id>
+ <properties>
+ <akka.version>2.3.4-spark</akka.version>
+ <spark.version>1.4.0</spark.version>
+ <mesos.version>0.21.1</mesos.version>
+ <hbase.version>0.98.7-hadoop2</hbase.version>
+ <hbase.artifact>hbase</hbase.artifact>
+ <hive.group>org.spark-project.hive</hive.group>
+ <hive.version>0.13.1a</hive.version>
+ <derby.version>10.10.1.1</derby.version>
+ <orbit.version>3.0.0.v201112011016</orbit.version>
+ <parquet.version>1.6.0rc3</parquet.version>
+ <chill.version>0.5.0</chill.version>
+ <ivy.version>2.4.0</ivy.version>
+ <oro.version>2.0.8</oro.version>
+ <avro.mapred.classifier>hadoop2</avro.mapred.classifier>
+ <codahale.metrics.version>3.1.0</codahale.metrics.version>
+ <commons.httpclient.version>4.3.2</commons.httpclient.version>
+ <commons.math3.version>3.4.1</commons.math3.version>
+ <io.netty.version>4.0.23.Final</io.netty.version>
+ <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
+ <fasterxml.jackson.version>2.4.4</fasterxml.jackson.version>
+ <snappy.version>1.1.1.7</snappy.version>
+ <mesos.version>0.21.0</mesos.version>
+ </properties>
+
+ <dependencies>
+ </dependencies>
+
+ </profile>
+
+
+ <profile>
<id>hadoop-0.23</id>
<!-- SPARK-1121: Adds an explicit dependency on Avro to work around a
Hadoop 0.23.X issue -->
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/928d0998/spark/pom.xml
----------------------------------------------------------------------
diff --git a/spark/pom.xml b/spark/pom.xml
index 72f216d..b4c05e5 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -76,7 +76,6 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <version>2.3.0</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/928d0998/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
index 9fce093..95eefd8 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
@@ -48,9 +48,9 @@ import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.WrappedInterpreter;
-import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -298,7 +298,8 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
SparkInterpreter sparkInterpreter = getSparkInterpreter();
if (!sparkInterpreter.getSparkContext().version().startsWith("1.2") &&
- !sparkInterpreter.getSparkContext().version().startsWith("1.3")) {
+ !sparkInterpreter.getSparkContext().version().startsWith("1.3") &&
+ !sparkInterpreter.getSparkContext().version().startsWith("1.4")) {
return new InterpreterResult(Code.ERROR, "pyspark "
+ sparkInterpreter.getSparkContext().version() + " is not supported");
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/928d0998/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
index c875e85..731068e 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
@@ -415,6 +415,10 @@ public class SparkInterpreter extends Interpreter {
Method loadFiles = this.interpreter.getClass().getMethod(
"org$apache$spark$repl$SparkILoop$$loadFiles", Settings.class);
loadFiles.invoke(this.interpreter, settings);
+ } else if (sc.version().startsWith("1.4")) {
+ Method loadFiles = this.interpreter.getClass().getMethod(
+ "org$apache$spark$repl$SparkILoop$$loadFiles", Settings.class);
+ loadFiles.invoke(this.interpreter, settings);
}
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
@@ -447,6 +451,10 @@ public class SparkInterpreter extends Interpreter {
intp.interpret("import sqlContext.implicits._");
intp.interpret("import sqlContext.sql");
intp.interpret("import org.apache.spark.sql.functions._");
+ } else if (sc.version().startsWith("1.4")) {
+ intp.interpret("import sqlContext.implicits._");
+ intp.interpret("import sqlContext.sql");
+ intp.interpret("import org.apache.spark.sql.functions._");
}
// add jar
@@ -621,6 +629,8 @@ public class SparkInterpreter extends Interpreter {
progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage());
} else if (sc.version().startsWith("1.3")) {
progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage());
+ } else if (sc.version().startsWith("1.4")) {
+ progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage());
} else {
continue;
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/928d0998/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java
index 618579d..3138fbc 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java
@@ -171,6 +171,8 @@ public class SparkSqlInterpreter extends Interpreter {
progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage());
} else if (sc.version().startsWith("1.3")) {
progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage());
+ } else if (sc.version().startsWith("1.4")) {
+ progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage());
} else {
logger.warn("Spark {} getting progress information not supported" + sc.version());
continue;