You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by ni...@apache.org on 2017/04/30 19:30:29 UTC
incubator-systemml git commit: [SYSTEMML-692] Separated Caffe2dml
into a separate jar
Repository: incubator-systemml
Updated Branches:
refs/heads/master 39a37ae40 -> 738ddfc17
[SYSTEMML-692] Separated Caffe2dml into a separate jar
1. Caffe2DML should be integrated in main SystemML jar. Pros:
One jar for all. Cons: +1 MB.
2. Caffe2DML should be released as a separate jar. Pros: -1MB. Cons: One more artifact to release.
3 Caffe2DML should be a separate project. I don't like this option as Caffe2DML is an API of SystemML, not a standalone tool.
This commit uses option 2. We can later revisit this if other committers
have strong opinion.
Closes #466.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/738ddfc1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/738ddfc1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/738ddfc1
Branch: refs/heads/master
Commit: 738ddfc17ca56d7c366bbea83f584e9f10cd8d44
Parents: 39a37ae
Author: Niketan Pansare <np...@us.ibm.com>
Authored: Sun Apr 30 12:27:26 2017 -0700
Committer: Niketan Pansare <np...@us.ibm.com>
Committed: Sun Apr 30 12:27:26 2017 -0700
----------------------------------------------------------------------
pom.xml | 45 ++++++++++++++--
src/main/python/pre_setup.py | 3 ++
src/main/python/systemml/classloader.py | 57 ++++++++++++--------
.../apache/sysml/api/dl/Caffe2DMLLoader.scala | 37 +++++++++++++
4 files changed, 117 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/738ddfc1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9a33853..af788e1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -254,15 +254,51 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
+ <executions>
+ <execution>
+ <goals><goal>jar</goal></goals>
+ <phase>package</phase>
+ <configuration>
+ <archive>
<index>true</index>
<manifestEntries>
<Build-Timestamp>${maven.build.timestamp}</Build-Timestamp>
<Main-Class>org.apache.sysml.api.DMLScript</Main-Class>
</manifestEntries>
- </archive>
- </configuration>
+ </archive>
+ <classifier>lite</classifier>
+ <excludes>
+ <exclude>**/caffe/*</exclude>
+ <exclude>**/org/tensorflow/*</exclude>
+ <exclude>**/org/tensorflow/framework/*</exclude>
+ <exclude>**/org/tensorflow/util/*</exclude>
+ <exclude>**/org/apache/sysml/api/dl/*</exclude>
+ </excludes>
+ </configuration>
+ </execution>
+ <execution>
+ <id>extra</id>
+ <goals><goal>jar</goal></goals>
+ <phase>package</phase>
+ <configuration>
+ <archive>
+ <index>true</index>
+ <manifestEntries>
+ <Build-Timestamp>${maven.build.timestamp}</Build-Timestamp>
+ <Main-Class>org.apache.sysml.api.DMLScript</Main-Class>
+ </manifestEntries>
+ </archive>
+ <classifier>extra</classifier>
+ <includes>
+ <include>**/META-INF/*</include>
+ <include>**/caffe/*</include>
+ <include>**/org/tensorflow/framework/*</include>
+ <include>**/org/tensorflow/util/*</include>
+ <include>**/org/apache/sysml/api/dl/*</include>
+ </includes>
+ </configuration>
+ </execution>
+ </executions>
</plugin>
<plugin>
@@ -582,6 +618,7 @@
<phase>package</phase>
<configuration>
<target name="copy and rename JAR">
+ <move file="${project.build.directory}/${project.artifactId}-${project.version}-lite.jar" tofile="${project.build.directory}/${project.artifactId}-${project.version}.jar" />
<copy file="${project.build.directory}/${project.artifactId}-${project.version}.jar" tofile="${project.build.directory}/SystemML.jar" />
</target>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/738ddfc1/src/main/python/pre_setup.py
----------------------------------------------------------------------
diff --git a/src/main/python/pre_setup.py b/src/main/python/pre_setup.py
index 5443a6d..cb421a9 100644
--- a/src/main/python/pre_setup.py
+++ b/src/main/python/pre_setup.py
@@ -33,3 +33,6 @@ for file in os.listdir(os.path.join(root_dir, 'target')):
if fnmatch.fnmatch(file, 'systemml-*-incubating-SNAPSHOT.jar') or fnmatch.fnmatch(file, 'systemml-*-incubating.jar'):
shutil.copyfile(os.path.join(root_dir, 'target', file),
os.path.join(java_dir_full_path, file))
+ if fnmatch.fnmatch(file, 'systemml-*-incubating-SNAPSHOT-extra.jar') or fnmatch.fnmatch(file, 'systemml-*-incubating-extra.jar'):
+ shutil.copyfile(os.path.join(root_dir, 'target', file),
+ os.path.join(java_dir_full_path, file))
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/738ddfc1/src/main/python/systemml/classloader.py
----------------------------------------------------------------------
diff --git a/src/main/python/systemml/classloader.py b/src/main/python/systemml/classloader.py
index 79c0aad..45fe4de 100644
--- a/src/main/python/systemml/classloader.py
+++ b/src/main/python/systemml/classloader.py
@@ -38,6 +38,29 @@ def _createJavaObject(sc, obj_type):
else:
raise ValueError('Incorrect usage: supported values: mlcontext or dummy')
+def _getJarFileName(sc, suffix):
+ import imp, fnmatch
+ jar_file_name = '_ignore.jar'
+ java_dir = os.path.join(imp.find_module("systemml")[1], "systemml-java")
+ for file in os.listdir(java_dir):
+ if fnmatch.fnmatch(file, 'systemml-*-incubating-SNAPSHOT' + suffix + '.jar') or fnmatch.fnmatch(file, 'systemml-*-incubating' + suffix + '.jar'):
+ jar_file_name = os.path.join(java_dir, file)
+ return jar_file_name
+
+def _getLoaderInstance(sc, jar_file_name, className, hint):
+ err_msg = 'Unable to load systemml-*.jar into current pyspark session.'
+ if os.path.isfile(jar_file_name):
+ sc._jsc.addJar(jar_file_name)
+ jar_file_url = sc._jvm.java.io.File(jar_file_name).toURI().toURL()
+ url_class = sc._jvm.java.net.URL
+ jar_file_url_arr = sc._gateway.new_array(url_class, 1)
+ jar_file_url_arr[0] = jar_file_url
+ url_class_loader = sc._jvm.java.net.URLClassLoader(jar_file_url_arr, sc._jsc.getClass().getClassLoader())
+ c1 = sc._jvm.java.lang.Class.forName(className, True, url_class_loader)
+ return c1.newInstance()
+ else:
+ raise ImportError(err_msg + ' Hint: Download the jar from http://systemml.apache.org/download and ' + hint )
+
def createJavaObject(sc, obj_type):
"""
Performs appropriate check if SystemML.jar is available and returns the handle to MLContext object on JVM
@@ -51,27 +74,19 @@ def createJavaObject(sc, obj_type):
try:
return _createJavaObject(sc, obj_type)
except (py4j.protocol.Py4JError, TypeError):
- import imp, fnmatch
- jar_file_name = '_ignore.jar'
- java_dir = os.path.join(imp.find_module("systemml")[1], "systemml-java")
- for file in os.listdir(java_dir):
- if fnmatch.fnmatch(file, 'systemml-*-incubating-SNAPSHOT.jar') or fnmatch.fnmatch(file, 'systemml-*-incubating.jar'):
- jar_file_name = os.path.join(java_dir, file)
- err_msg = 'Unable to load SystemML.jar into current pyspark session.'
+ ret = None
+ err_msg = 'Unable to load systemml-*.jar into current pyspark session.'
hint = 'Provide the following argument to pyspark: --driver-class-path '
- if os.path.isfile(jar_file_name):
- sc._jsc.addJar(jar_file_name)
- jar_file_url = sc._jvm.java.io.File(jar_file_name).toURI().toURL()
- url_class = sc._jvm.java.net.URL
- jar_file_url_arr = sc._gateway.new_array(url_class, 1)
- jar_file_url_arr[0] = jar_file_url
- url_class_loader = sc._jvm.java.net.URLClassLoader(jar_file_url_arr, sc._jsc.getClass().getClassLoader())
- c1 = sc._jvm.java.lang.Class.forName('org.apache.sysml.utils.SystemMLLoaderUtils', True, url_class_loader)
- x = c1.newInstance()
- x.loadSystemML(jar_file_name)
- else:
- raise ImportError(err_msg + ' Hint: Download the jar from http://systemml.apache.org/download and ' + hint + 'SystemML.jar')
+ # First load SystemML
+ jar_file_name = _getJarFileName(sc, '')
+ x = _getLoaderInstance(sc, jar_file_name, 'org.apache.sysml.utils.SystemMLLoaderUtils', hint + 'SystemML.jar')
+ x.loadSystemML(jar_file_name)
try:
- return _createJavaObject(sc, obj_type)
+ ret = _createJavaObject(sc, obj_type)
except (py4j.protocol.Py4JError, TypeError):
- raise ImportError(err_msg + ' Hint: ' + hint + jar_file_name)
\ No newline at end of file
+ raise ImportError(err_msg + ' Hint: ' + hint + jar_file_name)
+ # Now load caffe2dml
+ jar_file_name = _getJarFileName(sc, '-extra')
+ x = _getLoaderInstance(sc, jar_file_name, 'org.apache.sysml.api.dl.Caffe2DMLLoader', hint + 'systemml-*-extra.jar')
+ x.loadCaffe2DML(jar_file_name)
+ return ret
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/738ddfc1/src/main/scala/org/apache/sysml/api/dl/Caffe2DMLLoader.scala
----------------------------------------------------------------------
diff --git a/src/main/scala/org/apache/sysml/api/dl/Caffe2DMLLoader.scala b/src/main/scala/org/apache/sysml/api/dl/Caffe2DMLLoader.scala
new file mode 100644
index 0000000..30d86fd
--- /dev/null
+++ b/src/main/scala/org/apache/sysml/api/dl/Caffe2DMLLoader.scala
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.api.dl
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.io.File;
+
+class Caffe2DMLLoader {
+ def loadCaffe2DML(filePath:String):Unit = {
+ val url = new File(filePath).toURI().toURL();
+ val classLoader = ClassLoader.getSystemClassLoader().asInstanceOf[URLClassLoader];
+ val method = classOf[URLClassLoader].getDeclaredMethod("addURL", classOf[URL]);
+ method.setAccessible(true);
+ method.invoke(classLoader, url);
+ }
+}
\ No newline at end of file