You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2019/03/13 01:42:50 UTC
[zeppelin] branch master updated: [ZEPPELIN-4030] spark-scala-2.10
won't be built under some profile
This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new d4a2f64 [ZEPPELIN-4030] spark-scala-2.10 won't be built under some profile
d4a2f64 is described below
commit d4a2f6403aa06950b0786362f2aba88cb4df7ad0
Author: Jeff Zhang <zj...@apache.org>
AuthorDate: Wed Mar 6 15:24:36 2019 +0800
[ZEPPELIN-4030] spark-scala-2.10 won't be built under some profile
### What is this PR for?
Before this PR, module spark-scala-2.10 is only built under spark profile which support both scala 2.10 and 2.11. But it should not be depend on which version of spark to build with. The spark profile should only affect the embedded spark distribution of spark interpreter, it should support multiple spark version no matter which profile it is built against.
### What type of PR is it?
[Bug Fix]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://jira.apache.org/jira/browse/ZEPPELIN-4030
### How should this be tested?
* CI pass
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #3325 from zjffdu/ZEPPELIN-4030 and squashes the following commits:
aec33d24c [Jeff Zhang] [ZEPPELIN-4030] spark-scala-2.10 won't be built under some profile
---
spark/interpreter/pom.xml | 69 ++------------
spark/pom.xml | 23 +----
spark/scala-2.10/pom.xml | 1 +
spark/scala-2.11/pom.xml | 1 +
spark/spark-dependencies/pom.xml | 190 +--------------------------------------
spark/spark-scala-parent/pom.xml | 100 +++++++++++++++++----
spark/spark-shims/pom.xml | 7 --
spark/spark1-shims/pom.xml | 7 --
spark/spark2-shims/pom.xml | 7 --
9 files changed, 101 insertions(+), 304 deletions(-)
diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml
index 32f625d..d7d8418 100644
--- a/spark/interpreter/pom.xml
+++ b/spark/interpreter/pom.xml
@@ -52,8 +52,6 @@
<!-- settings -->
<pyspark.test.exclude>**/PySparkInterpreterMatplotlibTest.java</pyspark.test.exclude>
<pyspark.test.include>**/*Test.*</pyspark.test.include>
-
-
</properties>
<dependencies>
@@ -65,6 +63,12 @@
<dependency>
<groupId>org.apache.zeppelin</groupId>
+ <artifactId>spark-scala-2.10</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.zeppelin</groupId>
<artifactId>spark-scala-2.11</artifactId>
<version>${project.version}</version>
</dependency>
@@ -139,6 +143,7 @@
</exclusion>
</exclusions>
</dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
@@ -151,18 +156,8 @@
<artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- </exclusions>
</dependency>
-
+
<!-- Aether :: maven dependency resolution -->
<dependency>
<groupId>org.apache.maven</groupId>
@@ -602,52 +597,4 @@
</plugins>
</build>
-
- <profiles>
-
- <profile>
- <id>spark-2.2</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.zeppelin</groupId>
- <artifactId>spark-scala-2.10</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </profile>
-
- <profile>
- <id>spark-2.1</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.zeppelin</groupId>
- <artifactId>spark-scala-2.10</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </profile>
-
- <profile>
- <id>spark-2.0</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.zeppelin</groupId>
- <artifactId>spark-scala-2.10</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </profile>
-
- <profile>
- <id>spark-1.6</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.zeppelin</groupId>
- <artifactId>spark-scala-2.10</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </profile>
-
- </profiles>
</project>
diff --git a/spark/pom.xml b/spark/pom.xml
index b16e53f..0d88246 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -59,6 +59,7 @@
<modules>
<module>interpreter</module>
<module>spark-scala-parent</module>
+ <module>scala-2.10</module>
<module>scala-2.11</module>
<module>spark-dependencies</module>
<module>spark-shims</module>
@@ -162,8 +163,6 @@
</executions>
<configuration>
<scalaVersion>${scala.compile.version}</scalaVersion>
- <!--<recompileMode>incremental</recompileMode>-->
- <!--<useZincServer>true</useZincServer>-->
<args>
<arg>-unchecked</arg>
<arg>-deprecation</arg>
@@ -188,7 +187,6 @@
</plugins>
</build>
-
<profiles>
<profile>
@@ -198,6 +196,9 @@
<protobuf.version>2.5.0</protobuf.version>
<py4j.version>0.10.7</py4j.version>
</properties>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
</profile>
<profile>
@@ -215,12 +216,6 @@
<spark.version>2.2.1</spark.version>
<py4j.version>0.10.4</py4j.version>
</properties>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
- <modules>
- <module>scala-2.10</module>
- </modules>
</profile>
<profile>
@@ -229,9 +224,6 @@
<spark.version>2.1.2</spark.version>
<py4j.version>0.10.4</py4j.version>
</properties>
- <modules>
- <module>scala-2.10</module>
- </modules>
</profile>
<profile>
@@ -240,9 +232,6 @@
<spark.version>2.0.2</spark.version>
<py4j.version>0.10.3</py4j.version>
</properties>
- <modules>
- <module>scala-2.10</module>
- </modules>
</profile>
<profile>
@@ -251,10 +240,6 @@
<spark.version>1.6.3</spark.version>
<py4j.version>0.9</py4j.version>
</properties>
- <modules>
- <module>scala-2.10</module>
- </modules>
</profile>
-
</profiles>
</project>
diff --git a/spark/scala-2.10/pom.xml b/spark/scala-2.10/pom.xml
index 3d34f4f..a200443 100644
--- a/spark/scala-2.10/pom.xml
+++ b/spark/scala-2.10/pom.xml
@@ -33,6 +33,7 @@
<name>Zeppelin: Spark Interpreter Scala_2.10</name>
<properties>
+ <spark.version>2.2.0</spark.version>
<scala.version>2.10.5</scala.version>
<scala.binary.version>2.10</scala.binary.version>
<scala.compile.version>${scala.version}</scala.compile.version>
diff --git a/spark/scala-2.11/pom.xml b/spark/scala-2.11/pom.xml
index fc55afd..04632d5 100644
--- a/spark/scala-2.11/pom.xml
+++ b/spark/scala-2.11/pom.xml
@@ -33,6 +33,7 @@
<name>Zeppelin: Spark Interpreter Scala_2.11</name>
<properties>
+ <spark.version>2.4.0</spark.version>
<scala.version>2.11.8</scala.version>
<scala.binary.version>2.11</scala.binary.version>
<scala.compile.version>${scala.version}</scala.compile.version>
diff --git a/spark/spark-dependencies/pom.xml b/spark/spark-dependencies/pom.xml
index 6181393..be77eb9 100644
--- a/spark/spark-dependencies/pom.xml
+++ b/spark/spark-dependencies/pom.xml
@@ -53,196 +53,8 @@
<akka.group>org.spark-project.akka</akka.group>
<akka.version>2.3.4-spark</akka.version>
-
- <!--plugin versions-->
</properties>
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- <version>${avro.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-ipc</artifactId>
- <version>${avro.version}</version>
- <exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.velocity</groupId>
- <artifactId>velocity</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-mapred</artifactId>
- <version>${avro.version}</version>
- <classifier>${avro.mapred.classifier}</classifier>
- <exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.velocity</groupId>
- <artifactId>velocity</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <!-- See SPARK-1556 for info on this dependency: -->
- <dependency>
- <groupId>net.java.dev.jets3t</groupId>
- <artifactId>jets3t</artifactId>
- <version>${jets3t.version}</version>
- <scope>runtime</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-api</artifactId>
- <version>${yarn.version}</version>
- <exclusions>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- <version>${yarn.version}</version>
- <exclusions>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-web-proxy</artifactId>
- <version>${yarn.version}</version>
- <exclusions>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- <version>${yarn.version}</version>
- <exclusions>
- <exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
<dependencies>
<!-- Spark -->
<dependency>
@@ -505,4 +317,6 @@
</plugin>
</plugins>
</build>
+
+
</project>
diff --git a/spark/spark-scala-parent/pom.xml b/spark/spark-scala-parent/pom.xml
index 8aeb403..c53688c 100644
--- a/spark/spark-scala-parent/pom.xml
+++ b/spark/spark-scala-parent/pom.xml
@@ -21,10 +21,10 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
- <artifactId>spark-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
+ <artifactId>zeppelin-interpreter-parent</artifactId>
<version>0.9.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
+ <relativePath>../../zeppelin-interpreter-parent/pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -34,7 +34,20 @@
<packaging>pom</packaging>
<name>Zeppelin: Spark Scala Parent</name>
+ <properties>
+ <spark.version>2.4.0</spark.version>
+ <scala.binary.version>2.11</scala.binary.version>
+ <scala.version>2.11.8</scala.version>
+ <scala.compile.version>${scala.binary.version}</scala.compile.version>
+ </properties>
+
<dependencies>
+ <dependency>
+ <groupId>org.apache.zeppelin</groupId>
+ <artifactId>zeppelin-interpreter</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.zeppelin</groupId>
@@ -84,10 +97,38 @@
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </dependency>
</dependencies>
<build>
<plugins>
+
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-interpreter-setting</id>
+ <phase>none</phase>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
@@ -148,23 +189,52 @@
</plugin>
<plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
-
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <version>3.2.2</version>
<executions>
<execution>
- <id>copy-interpreter-setting</id>
- <phase>none</phase>
- <configuration>
- <skip>true</skip>
- </configuration>
+ <id>eclipse-add-source</id>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>scala-compile-first</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>scala-test-compile-first</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
</execution>
</executions>
+ <configuration>
+ <scalaVersion>${scala.compile.version}</scalaVersion>
+ <args>
+ <arg>-unchecked</arg>
+ <arg>-deprecation</arg>
+ <arg>-feature</arg>
+ </args>
+ <jvmArgs>
+ <jvmArg>-Xms1024m</jvmArg>
+ <jvmArg>-Xmx1024m</jvmArg>
+ <jvmArg>-XX:PermSize=${PermGen}</jvmArg>
+ <jvmArg>-XX:MaxPermSize=${MaxPermGen}</jvmArg>
+ </jvmArgs>
+ <javacArgs>
+ <javacArg>-source</javacArg>
+ <javacArg>${java.version}</javacArg>
+ <javacArg>-target</javacArg>
+ <javacArg>${java.version}</javacArg>
+ <javacArg>-Xlint:all,-serial,-path,-options</javacArg>
+ </javacArgs>
+ </configuration>
</plugin>
</plugins>
diff --git a/spark/spark-shims/pom.xml b/spark/spark-shims/pom.xml
index 1397ebe..7523c1b 100644
--- a/spark/spark-shims/pom.xml
+++ b/spark/spark-shims/pom.xml
@@ -57,13 +57,6 @@
<build>
<plugins>
<plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
-
- <plugin>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
diff --git a/spark/spark1-shims/pom.xml b/spark/spark1-shims/pom.xml
index c28acef..051a66d 100644
--- a/spark/spark1-shims/pom.xml
+++ b/spark/spark1-shims/pom.xml
@@ -72,13 +72,6 @@
<build>
<plugins>
<plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
-
- <plugin>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
diff --git a/spark/spark2-shims/pom.xml b/spark/spark2-shims/pom.xml
index 00abc75..8c8d145 100644
--- a/spark/spark2-shims/pom.xml
+++ b/spark/spark2-shims/pom.xml
@@ -71,13 +71,6 @@
<build>
<plugins>
<plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
-
- <plugin>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>