You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by re...@apache.org on 2017/11/04 05:31:15 UTC
systemml git commit: [Minor] update creation of lite jar
Repository: systemml
Updated Branches:
refs/heads/master 820b53046 -> ae819a253
[Minor] update creation of lite jar
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/ae819a25
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/ae819a25
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/ae819a25
Branch: refs/heads/master
Commit: ae819a253097da796e689764e4926d46228e055e
Parents: 820b530
Author: Berthold Reinwald <re...@us.ibm.com>
Authored: Fri Nov 3 22:24:04 2017 -0700
Committer: Berthold Reinwald <re...@us.ibm.com>
Committed: Fri Nov 3 22:30:59 2017 -0700
----------------------------------------------------------------------
pom.xml | 308 ++++++++++---------
src/assembly/lite.xml | 22 +-
.../org/apache/sysml/utils/lite/BuildLite.java | 22 +-
3 files changed, 198 insertions(+), 154 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/ae819a25/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ff90986..6201c77 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,23 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<!-- * Licensed to the Apache Software Foundation (ASF) under one * or more
+ contributor license agreements. See the NOTICE file * distributed with this
+ work for additional information * regarding copyright ownership. The ASF
+ licenses this file * to you under the Apache License, Version 2.0 (the *
+ "License"); you may not use this file except in compliance * with the License.
+ You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0
+ * * Unless required by applicable law or agreed to in writing, * software
+ distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT
+ WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the
+ License for the * specific language governing permissions and limitations
+ * under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache</groupId>
@@ -78,45 +72,46 @@
</properties>
<repositories>
- <repository>
- <id>central</id>
- <url>https://repo1.maven.org/maven2</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- </repository>
- </repositories>
+ <repository>
+ <id>central</id>
+ <url>https://repo1.maven.org/maven2</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ </repository>
+ </repositories>
<build>
- <!-- Adds scripts to main jar, in-memory jar, sources jar, and standalone jar -->
- <resources>
- <resource>
- <directory>scripts</directory>
- <excludes>
- <exclude>algorithms/obsolete/*</exclude>
- <exclude>datagen/obsolete/*</exclude>
- <exclude>perftest/*</exclude>
- <exclude>staging/**/*</exclude>
- <exclude>nn/test/compare_backends/*</exclude>
- <exclude>nn/test/compare_backends/*</exclude>
- <exclude>nn/examples/caffe2dml/**/*</exclude>
- <!-- <exclude>*.sh</exclude> --> <!-- applies to sparkDML.sh -->
- </excludes>
- <targetPath>scripts</targetPath>
- </resource>
- <resource>
- <directory>src/main/cpp/kernels</directory>
- <excludes>
- <exclude>*.cu</exclude>
- </excludes>
- <targetPath>kernels</targetPath>
- </resource>
- <resource>
- <directory>src/main/cpp/lib</directory>
- <targetPath>lib</targetPath>
- </resource>
- </resources>
+ <!-- Adds scripts to main jar, in-memory jar, sources jar, and standalone
+ jar -->
+ <resources>
+ <resource>
+ <directory>scripts</directory>
+ <excludes>
+ <exclude>algorithms/obsolete/*</exclude>
+ <exclude>datagen/obsolete/*</exclude>
+ <exclude>perftest/*</exclude>
+ <exclude>staging/**/*</exclude>
+ <exclude>nn/test/compare_backends/*</exclude>
+ <exclude>nn/test/compare_backends/*</exclude>
+ <exclude>nn/examples/caffe2dml/**/*</exclude>
+ <!-- <exclude>*.sh</exclude> --> <!-- applies to sparkDML.sh -->
+ </excludes>
+ <targetPath>scripts</targetPath>
+ </resource>
+ <resource>
+ <directory>src/main/cpp/kernels</directory>
+ <excludes>
+ <exclude>*.cu</exclude>
+ </excludes>
+ <targetPath>kernels</targetPath>
+ </resource>
+ <resource>
+ <directory>src/main/cpp/lib</directory>
+ <targetPath>lib</targetPath>
+ </resource>
+ </resources>
<plugins>
<plugin>
@@ -180,8 +175,8 @@
</executions>
<configuration>
- <!-- Include signature files so that recent versions of Java will run
- the resulting jar without complaining about "Invalid signature file digest
+ <!-- Include signature files so that recent versions of Java will run
+ the resulting jar without complaining about "Invalid signature file digest
for Manifest main attributes". -->
<filters>
<filter>
@@ -209,8 +204,8 @@
</goals>
<configuration>
<sources>
- <!-- Test suite classes kept in a separate folder for compatibility
- with the Eclipse JUnit test runner. -->
+ <!-- Test suite classes kept in a separate folder for compatibility
+ with the Eclipse JUnit test runner. -->
<source>${basedir}/src/test_suites/java</source>
</sources>
</configuration>
@@ -226,55 +221,57 @@
</configuration>
</plugin>
<plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <version>3.2.2</version>
- <executions>
- <execution>
- <id>eclipse-add-source</id>
- <goals>
- <goal>add-source</goal>
- </goals>
- </execution>
- <execution>
- <id>scala-compile-first</id>
- <phase>process-resources</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- <execution>
- <id>scala-test-compile-first</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <version>3.2.2</version>
+ <executions>
+ <execution>
+ <id>eclipse-add-source</id>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>scala-compile-first</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>scala-test-compile-first</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ </execution>
+ </executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
- <execution>
- <id>default-jar</id>
- <goals><goal>jar</goal></goals>
- <phase>package</phase>
- <configuration>
- <archive>
- <index>true</index>
- <manifestEntries>
- <Build-Timestamp>${maven.build.timestamp}</Build-Timestamp>
- <Main-Class>org.apache.sysml.api.DMLScript</Main-Class>
- </manifestEntries>
- </archive>
- <excludes>
- <exclude>**/caffe/*</exclude>
- <exclude>**/org/apache/sysml/api/dl/*</exclude>
- </excludes>
- </configuration>
- </execution>
- </executions>
+ <execution>
+ <id>default-jar</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <archive>
+ <index>true</index>
+ <manifestEntries>
+ <Build-Timestamp>${maven.build.timestamp}</Build-Timestamp>
+ <Main-Class>org.apache.sysml.api.DMLScript</Main-Class>
+ </manifestEntries>
+ </archive>
+ <excludes>
+ <exclude>**/caffe/*</exclude>
+ <exclude>**/org/apache/sysml/api/dl/*</exclude>
+ </excludes>
+ </configuration>
+ </execution>
+ </executions>
</plugin>
<plugin>
@@ -343,50 +340,51 @@
</plugin>
<plugin>
- <groupId>com.github.os72</groupId>
- <artifactId>protoc-jar-maven-plugin</artifactId>
- <version>3.4.0.1</version>
- <executions>
- <execution>
- <id>caffe-sources</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <protocVersion>2.5.0</protocVersion> <!-- 2.4.1, 2.5.0, 2.6.1, 3.4.0 -->
- <inputDirectories>
- <include>src/main/proto/caffe</include>
- </inputDirectories>
- <outputDirectories>
- <include>src/main/java</include>
- </outputDirectories>
- </configuration>
- </execution>
- </executions>
+ <groupId>com.github.os72</groupId>
+ <artifactId>protoc-jar-maven-plugin</artifactId>
+ <version>3.4.0.1</version>
+ <executions>
+ <execution>
+ <id>caffe-sources</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <protocVersion>2.5.0</protocVersion> <!-- 2.4.1, 2.5.0, 2.6.1, 3.4.0 -->
+ <inputDirectories>
+ <include>src/main/proto/caffe</include>
+ </inputDirectories>
+ <outputDirectories>
+ <include>src/main/java</include>
+ </outputDirectories>
+ </configuration>
+ </execution>
+ </executions>
</plugin>
<!-- Currently, all tests are integration tests. -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
- <version>2.18</version><!--$NO-MVN-MAN-VER$-->
+ <version>2.18</version><!--$NO-MVN-MAN-VER$ -->
<configuration>
<!-- STDERR/STDOUT to individual .txt files instead of console -->
<redirectTestOutputToFile>true</redirectTestOutputToFile>
- <!-- Run (num cores) test harness classes in parallel, each in a separate
+ <!-- Run (num cores) test harness classes in parallel, each in a separate
process -->
<forkCount>1C</forkCount>
<reuseForks>true</reuseForks>
- <!-- Third argument prevents Java from popping up lots of windows on
+ <!-- Third argument prevents Java from popping up lots of windows on
MacOS -->
- <argLine>-Dfile.encoding=UTF-8 -Xmx2g -Xms2g -Xmn200m -Djava.awt.headless=true</argLine>
+ <argLine>-Dfile.encoding=UTF-8 -Xmx2g -Xms2g -Xmn200m
+ -Djava.awt.headless=true</argLine>
<includes>
- <!-- All tests are integration tests as far as Maven is concerned. -->
+ <!-- All tests are integration tests as far as Maven is concerned. -->
</includes>
<excludes>
@@ -405,7 +403,7 @@
<!-- *** HACK ALERT *** HACK ALERT *** HACK ALERT *** -->
<!-- Failsafe 2.18 has a bug in computing # cores, so use 2.17 -->
- <version>2.17</version><!--$NO-MVN-MAN-VER$-->
+ <version>2.17</version><!--$NO-MVN-MAN-VER$ -->
<!-- *** END HACK *** -->
<executions>
@@ -429,7 +427,7 @@
<forkCount>0.5C</forkCount>
<reuseForks>false</reuseForks>
- <!-- Last argument prevents Java from popping up lots of windows on
+ <!-- Last argument prevents Java from popping up lots of windows on
MacOS -->
<argLine>-Dfile.encoding=UTF-8 -Xmx2g -Xms2g -Xmn200m
${integrationTestExtraJVMArgs}
@@ -479,8 +477,8 @@
</configuration>
</execution>
<execution>
- <!-- NOTE: We don't clean up systemml.egg-info since this makes it difficult
- to uninstall a dev version after a maven clean -->
+ <!-- NOTE: We don't clean up systemml.egg-info since this makes it
+ difficult to uninstall a dev version after a maven clean -->
<id>clean-python-files</id>
<phase>clean</phase>
<goals>
@@ -582,7 +580,9 @@
<phase>package</phase>
<configuration>
<target name="copy and rename JAR">
- <copy file="${project.build.directory}/${project.artifactId}-${project.version}.jar" tofile="${project.build.directory}/SystemML.jar" />
+ <copy
+ file="${project.build.directory}/${project.artifactId}-${project.version}.jar"
+ tofile="${project.build.directory}/SystemML.jar" />
</target>
</configuration>
<goals>
@@ -702,7 +702,7 @@
</profile>
<profile>
- <!-- Profile for Windows builds. Not currently needed, but might be needed
+ <!-- Profile for Windows builds. Not currently needed, but might be needed
in the future. -->
<id>platform-windows</id>
<activation>
@@ -711,7 +711,7 @@
</os>
</activation>
<properties>
- <!-- Extra JVM args for Windows go here.-->
+ <!-- Extra JVM args for Windows go here. -->
<integrationTestExtraJVMArgs />
</properties>
</profile>
@@ -852,7 +852,7 @@
<exclude>**/*.keep</exclude>
<exclude>**/target/**</exclude>
<exclude>**/README.md</exclude>
- <exclude>**/*.svg</exclude>
+ <exclude>**/*.svg</exclude>
<!-- Jupyter Notebooks -->
<exclude>**/*.ipynb</exclude>
<!-- Generated antlr files -->
@@ -898,17 +898,15 @@
<profile>
<!-- Can be used to ignore doclint javadoc issues -->
<id>ignore-doclint</id>
- <!-- <activation>
- <jdk>[1.8,)</jdk>
- </activation> -->
+ <!-- <activation> <jdk>[1.8,)</jdk> </activation> -->
<properties>
<javadoc.opts>-Xdoclint:none</javadoc.opts>
</properties>
</profile>
<profile>
- <!-- Profile to create binary distributions.
- Execute with `mvn clean package -P distribution` -->
+ <!-- Profile to create binary distributions. Execute with `mvn clean package
+ -P distribution` -->
<id>distribution</id>
<build>
<plugins>
@@ -1017,13 +1015,10 @@
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.10.3</version>
<configuration>
- <!-- Need to include the following packages, so exclude others:
- org.apache.sysml.api
- org.apache.sysml.runtime.instructions.spark.utils (for RDDConverterUtils, etc)
- org.apache.sysml.runtime.matrix (for MatrixCharacteristics, etc)
- org.apache.sysml.runtime.matrix.data (for MatrixIndexes, MatrixBlock, etc)
- org.apache.sysml.udf
- -->
+ <!-- Need to include the following packages, so exclude others: org.apache.sysml.api
+ org.apache.sysml.runtime.instructions.spark.utils (for RDDConverterUtils,
+ etc) org.apache.sysml.runtime.matrix (for MatrixCharacteristics, etc) org.apache.sysml.runtime.matrix.data
+ (for MatrixIndexes, MatrixBlock, etc) org.apache.sysml.udf -->
<excludePackageNames>caffe:org.apache.sysml.conf:org.apache.sysml.debug:org.apache.sysml.hops:org.apache.sysml.lops:org.apache.sysml.parser:org.apache.sysml.runtime.controlprogram:org.apache.sysml.runtime.functionobjects:org.apache.sysml.runtime.instructions.cp:org.apache.sysml.runtime.instructions.cpfile:org.apache.sysml.runtime.instructions.mr:org.apache.sysml.runtime.instructions.spark.data:org.apache.sysml.runtime.instructions.spark.functions:org.apache.sysml.runtime.io:org.apache.sysml.runtime.matrix.data.hadoopfix:org.apache.sysml.runtime.matrix.mapred:org.apache.sysml.runtime.matrix.operators:org.apache.sysml.runtime.matrix.sort:org.apache.sysml.runtime.transform:org.apache.sysml.runtime.util:org.apache.sysml.utils:org.apache.sysml.yarn</excludePackageNames>
<additionalparam>${javadoc.opts}</additionalparam>
</configuration>
@@ -1162,6 +1157,17 @@
<descriptors>
<descriptor>src/assembly/lite.xml</descriptor>
</descriptors>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.sysml.api.DMLScript</mainClass>
+ </manifest>
+ <manifestEntries>
+ <Build-Time>${maven.build.timestamp}</Build-Time>
+ <Group-Id>${project.groupId}</Group-Id>
+ <Artifact-Id>${project.artifactId}-lite</Artifact-Id>
+ <Version>${project.version}</Version>
+ </manifestEntries>
+ </archive>
</configuration>
</execution>
</executions>
@@ -1171,8 +1177,8 @@
</profile>
<profile>
- <!-- Profile to create standalone jar.
- Execute with `mvn clean package -P standalone-jar` -->
+ <!-- Profile to create standalone jar. Execute with `mvn clean package
+ -P standalone-jar` -->
<id>standalone-jar</id>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/systemml/blob/ae819a25/src/assembly/lite.xml
----------------------------------------------------------------------
diff --git a/src/assembly/lite.xml b/src/assembly/lite.xml
index 908c72e..0212ace 100644
--- a/src/assembly/lite.xml
+++ b/src/assembly/lite.xml
@@ -53,7 +53,7 @@
<dependencySets>
<dependencySet>
<includes>
- <include>*:antlr4-runtime</include>
+ <include>*:antlr4</include>
</includes>
<unpackOptions>
<includes>
@@ -210,6 +210,8 @@
</includes>
<unpackOptions>
<includes>
+ <include>org/apache/commons/cli/OptionValidator.class</include>
+ <include>org/apache/commons/cli/Util.class</include>
<include>org/apache/commons/cli/AlreadySelectedException.class</include>
<include>org/apache/commons/cli/CommandLine.class</include>
<include>org/apache/commons/cli/CommandLineParser.class</include>
@@ -620,7 +622,12 @@
<unpackOptions>
<includes>
<include>META-INF/services/org.apache.hadoop.fs.FileSystem</include>
+ <include>common-version-info.properties</include>
<include>org/apache/hadoop/log/metrics/EventCounter.class</include>
+ <include>org/apache/hadoop/security/Groups$CachedGroups.class</include>
+ <include>org/apache/hadoop/security/token/Token.class</include>
+ <include>org/apache/hadoop/security/token/TokenIdentifier.class</include>
+ <include>org/apache/hadoop/util/GenericOptionsParser.class</include>
<include>org/apache/hadoop/util/ShutdownHookManager$2.class</include>
<include>org/apache/hadoop/HadoopIllegalArgumentException.class</include>
<include>org/apache/hadoop/conf/Configurable.class</include>
@@ -1027,6 +1034,19 @@
<dependencySet>
<includes>
+ <include>*:hadoop-mapreduce-client-jobclient</include>
+ </includes>
+ <unpackOptions>
+ <includes>
+ <include>org/apache/hadoop/mapred/YarnClientProtocolProvider.class</include>
+ </includes>
+ </unpackOptions>
+ <scope>compile</scope>
+ <unpack>true</unpack>
+ </dependencySet>
+
+ <dependencySet>
+ <includes>
<include>*:jcl-over-slf4j</include>
</includes>
<unpackOptions>
http://git-wip-us.apache.org/repos/asf/systemml/blob/ae819a25/src/main/java/org/apache/sysml/utils/lite/BuildLite.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/lite/BuildLite.java b/src/main/java/org/apache/sysml/utils/lite/BuildLite.java
index 9048ce6..ad76085 100644
--- a/src/main/java/org/apache/sysml/utils/lite/BuildLite.java
+++ b/src/main/java/org/apache/sysml/utils/lite/BuildLite.java
@@ -92,6 +92,14 @@ public class BuildLite {
additionalResources.add("org/apache/hadoop/util/ShutdownHookManager$2.class");
additionalResources.add("org/apache/hadoop/log/metrics/EventCounter.class");
+ additionalResources.add("org/apache/hadoop/util/GenericOptionsParser.class");
+ additionalResources.add("org/apache/hadoop/security/token/Token.class");
+ additionalResources.add("org/apache/hadoop/security/token/TokenIdentifier.class");
+ additionalResources.add("org/apache/hadoop/security/Groups$CachedGroups.class");
+ additionalResources.add("org/apache/commons/cli/OptionValidator.class");
+ additionalResources.add("org/apache/commons/cli/Util.class");
+ additionalResources.add("common-version-info.properties");
+
}
/**
@@ -103,8 +111,18 @@ public class BuildLite {
SortedSet<String> hadoopCommonResources = new TreeSet<>();
hadoopCommonResources.add("META-INF/services/org.apache.hadoop.fs.FileSystem");
hadoopCommonResources.add("org/apache/hadoop/util/ShutdownHookManager$2.class");
- hadoopCommonResources.add("org/apache/hadoop/log/metrics/EventCounter.class");
- additionalJarToFileMappingsForDependencySets.put("hadoop-common", hadoopCommonResources);
+ hadoopCommonResources.add("org/apache/hadoop/log/metrics/EventCounter.class");
+ hadoopCommonResources.add("org/apache/hadoop/util/GenericOptionsParser.class");
+ hadoopCommonResources.add("org/apache/hadoop/security/token/Token.class");
+ hadoopCommonResources.add("org/apache/hadoop/security/token/TokenIdentifier.class");
+ hadoopCommonResources.add("org/apache/hadoop/security/Groups$CachedGroups.class");
+ hadoopCommonResources.add("common-version-info.properties");
+ additionalJarToFileMappingsForDependencySets.put("hadoop-common", hadoopCommonResources);
+
+ SortedSet<String> commonsCliResources = new TreeSet<String>();
+ commonsCliResources.add("org/apache/commons/cli/OptionValidator.class");
+ commonsCliResources.add("org/apache/commons/cli/Util.class");
+ additionalJarToFileMappingsForDependencySets.put("commons-cli", commonsCliResources);
}
/**