You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2017/07/18 16:54:05 UTC
spark git commit: [SPARK-15526][ML][FOLLOWUP] Make JPMML provided
scope to avoid including unshaded JARs, and repromote to compile in MLlib
Repository: spark
Updated Branches:
refs/heads/master e26dac5fe -> d3f4a2119
[SPARK-15526][ML][FOLLOWUP] Make JPMML provided scope to avoid including unshaded JARs, and repromote to compile in MLlib
Following the comment at https://issues.apache.org/jira/browse/SPARK-15526?focusedCommentId=16086106&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-16086106 -- this change actually needed a little more work to be complete.
This also marks JPMML as `provided` to make sure its JARs aren't included in the `jars` output, but then scopes to `compile` in `mllib`. This is how Guava is handled.
Checked result in `assembly/target/scala-2.11/jars` to verify there are no JPMML jars. Maven and SBT builds still work.
Author: Sean Owen <so...@cloudera.com>
Closes #18637 from srowen/SPARK-15526.2.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d3f4a211
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d3f4a211
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d3f4a211
Branch: refs/heads/master
Commit: d3f4a2119634134320ebeb1ddbe0c49db080c040
Parents: e26dac5
Author: Sean Owen <so...@cloudera.com>
Authored: Tue Jul 18 09:53:49 2017 -0700
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Tue Jul 18 09:53:51 2017 -0700
----------------------------------------------------------------------
core/pom.xml | 1 +
dev/deps/spark-deps-hadoop-2.6 | 2 --
dev/deps/spark-deps-hadoop-2.7 | 2 --
.../spark/launcher/AbstractCommandBuilder.java | 1 +
mllib/pom.xml | 38 ++++++++++++++++----
pom.xml | 21 +++++++++--
6 files changed, 52 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d3f4a211/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 91ee941..bc6b1c4 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -429,6 +429,7 @@
<!-- When using SPARK_PREPEND_CLASSES Spark classes compiled locally don't use
shaded deps. So here we store jars in their original form which are added
when the classpath is computed. -->
+ <!-- See similar execution in mllib/pom.xml -->
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
http://git-wip-us.apache.org/repos/asf/spark/blob/d3f4a211/dev/deps/spark-deps-hadoop-2.6
----------------------------------------------------------------------
diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6
index 1a6515b..76e3764 100644
--- a/dev/deps/spark-deps-hadoop-2.6
+++ b/dev/deps/spark-deps-hadoop-2.6
@@ -158,8 +158,6 @@ parquet-format-2.3.1.jar
parquet-hadoop-1.8.2.jar
parquet-hadoop-bundle-1.6.0.jar
parquet-jackson-1.8.2.jar
-pmml-model-1.2.15.jar
-pmml-schema-1.2.15.jar
protobuf-java-2.5.0.jar
py4j-0.10.6.jar
pyrolite-4.13.jar
http://git-wip-us.apache.org/repos/asf/spark/blob/d3f4a211/dev/deps/spark-deps-hadoop-2.7
----------------------------------------------------------------------
diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7
index 09e5a42..f417347 100644
--- a/dev/deps/spark-deps-hadoop-2.7
+++ b/dev/deps/spark-deps-hadoop-2.7
@@ -159,8 +159,6 @@ parquet-format-2.3.1.jar
parquet-hadoop-1.8.2.jar
parquet-hadoop-bundle-1.6.0.jar
parquet-jackson-1.8.2.jar
-pmml-model-1.2.15.jar
-pmml-schema-1.2.15.jar
protobuf-java-2.5.0.jar
py4j-0.10.6.jar
pyrolite-4.13.jar
http://git-wip-us.apache.org/repos/asf/spark/blob/d3f4a211/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
----------------------------------------------------------------------
diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
index 481ff20..8004247 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
@@ -185,6 +185,7 @@ abstract class AbstractCommandBuilder {
// Add this path to include jars that are shaded in the final deliverable created during
// the maven build. These jars are copied to this directory during the build.
addToClassPath(cp, String.format("%s/core/target/jars/*", sparkHome));
+ addToClassPath(cp, String.format("%s/mllib/target/jars/*", sparkHome));
}
// Add Spark jars to the classpath. For the testing case, we rely on the test code to set and
http://git-wip-us.apache.org/repos/asf/spark/blob/d3f4a211/mllib/pom.xml
----------------------------------------------------------------------
diff --git a/mllib/pom.xml b/mllib/pom.xml
index 572670d..c72a16a 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -101,13 +101,7 @@
<dependency>
<groupId>org.jpmml</groupId>
<artifactId>pmml-model</artifactId>
- <version>1.2.15</version>
- <exclusions>
- <exclusion>
- <groupId>org.jpmml</groupId>
- <artifactId>pmml-agent</artifactId>
- </exclusion>
- </exclusions>
+ <scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
@@ -139,8 +133,38 @@
</dependencies>
</profile>
</profiles>
+
<build>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <!-- When using SPARK_PREPEND_CLASSES Spark classes compiled locally don't use
+ shaded deps. So here we store jars in their original form which are added
+ when the classpath is computed. -->
+ <!-- See similar execution in core/pom.xml -->
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}</outputDirectory>
+ <overWriteReleases>false</overWriteReleases>
+ <overWriteSnapshots>false</overWriteSnapshots>
+ <overWriteIfNewer>true</overWriteIfNewer>
+ <useSubDirectoryPerType>true</useSubDirectoryPerType>
+ <includeGroupIds>org.jpmml</includeGroupIds>
+ <silent>true</silent>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
</build>
+
</project>
http://git-wip-us.apache.org/repos/asf/spark/blob/d3f4a211/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 11cd028..1b81263 100644
--- a/pom.xml
+++ b/pom.xml
@@ -382,6 +382,18 @@
<version>14.0.1</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.jpmml</groupId>
+ <artifactId>pmml-model</artifactId>
+ <version>1.2.15</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.jpmml</groupId>
+ <artifactId>pmml-agent</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<!-- End of shaded deps -->
<dependency>
<groupId>org.apache.commons</groupId>
@@ -2511,8 +2523,8 @@
<profiles>
<!--
- This profile is enabled automatically by the sbt built. It changes the scope for the guava
- dependency, since we don't shade it in the artifacts generated by the sbt build.
+ This profile is enabled automatically by the sbt build. It changes the scope for shaded
+ dependencies, since we don't shade it in the artifacts generated by the sbt build.
-->
<profile>
<id>sbt</id>
@@ -2522,6 +2534,11 @@
<artifactId>guava</artifactId>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.jpmml</groupId>
+ <artifactId>pmml-model</artifactId>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
</profile>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org