You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by ni...@apache.org on 2017/05/02 23:49:54 UTC
incubator-systemml git commit: [MINOR] Updated documentation and
improved log messages
Repository: incubator-systemml
Updated Branches:
refs/heads/master 8324b69f1 -> 47c2dd10f
[MINOR] Updated documentation and improved log messages
- Also, BLAS is disabled by default. We can enable it after more rigorous
testing.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/47c2dd10
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/47c2dd10
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/47c2dd10
Branch: refs/heads/master
Commit: 47c2dd10f43e3f348c4372a8a7b34fa04705930a
Parents: 8324b69
Author: Niketan Pansare <np...@us.ibm.com>
Authored: Tue May 2 15:48:49 2017 -0800
Committer: Niketan Pansare <np...@us.ibm.com>
Committed: Tue May 2 16:48:49 2017 -0700
----------------------------------------------------------------------
conf/SystemML-config.xml.template | 2 +-
docs/beginners-guide-caffe2dml.md | 34 ++++++++++++++++++
docs/index.md | 1 +
docs/native-backend.md | 35 +++++++++++-------
.../sysml/api/mlcontext/ScriptExecutor.java | 12 +++----
.../java/org/apache/sysml/conf/DMLConfig.java | 2 +-
.../org/apache/sysml/utils/NativeHelper.java | 38 +++++++++++++++-----
.../org/apache/sysml/api/dl/Caffe2DML.scala | 9 ++---
.../org/apache/sysml/api/dl/CaffeNetwork.scala | 6 ++--
.../scala/org/apache/sysml/api/dl/Utils.scala | 12 -------
10 files changed, 102 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/conf/SystemML-config.xml.template
----------------------------------------------------------------------
diff --git a/conf/SystemML-config.xml.template b/conf/SystemML-config.xml.template
index 0ccf7da..8a6ac54 100644
--- a/conf/SystemML-config.xml.template
+++ b/conf/SystemML-config.xml.template
@@ -67,7 +67,7 @@
<codegen.literals>1</codegen.literals>
<!-- enables native blas for matrix multiplication and convolution, experimental feature -->
- <native.blas>true</native.blas>
+ <native.blas>false</native.blas>
<!-- prints extra statistics information for GPU -->
<systemml.stats.extraGPU>false</systemml.stats.extraGPU>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/docs/beginners-guide-caffe2dml.md
----------------------------------------------------------------------
diff --git a/docs/beginners-guide-caffe2dml.md b/docs/beginners-guide-caffe2dml.md
index cfcc0cb..dea53fd 100644
--- a/docs/beginners-guide-caffe2dml.md
+++ b/docs/beginners-guide-caffe2dml.md
@@ -31,6 +31,40 @@ limitations under the License.
Caffe2DML is an experimental API that converts an Caffe specification to DML.
+## Example: Train Lenet
+
+1. Install `mlextend` package to get MNIST data: `pip install mlxtend`.
+2. (Optional but recommended) Follow the steps mentioned in [the user guide]([the user guide of native backend](http://apache.github.io/incubator-systemml/native-backend)) and install Intel MKL.
+3. Install [SystemML](http://apache.github.io/incubator-systemml/beginners-guide-python#install-systemml).
+4. Invoke PySpark shell: `pyspark --conf spark.executorEnv.LD_LIBRARY_PATH=/path/to/blas-n-other-dependencies`.
+
+```bash
+# Download the MNIST dataset
+from mlxtend.data import mnist_data
+import numpy as np
+from sklearn.utils import shuffle
+X, y = mnist_data()
+X, y = shuffle(X, y)
+
+# Split the data into training and test
+n_samples = len(X)
+X_train = X[:int(.9 * n_samples)]
+y_train = y[:int(.9 * n_samples)]
+X_test = X[int(.9 * n_samples):]
+y_test = y[int(.9 * n_samples):]
+
+# Download the Lenet network
+import urllib
+urllib.urlretrieve('https://raw.githubusercontent.com/niketanpansare/model_zoo/master/caffe/vision/lenet/mnist/lenet.proto', 'lenet.proto')
+urllib.urlretrieve('https://raw.githubusercontent.com/niketanpansare/model_zoo/master/caffe/vision/lenet/mnist/lenet_solver.proto', 'lenet_solver.proto')
+
+# Train Lenet On MNIST using scikit-learn like API
+from systemml.mllearn import Caffe2DML
+lenet = Caffe2DML(sqlCtx, solver='lenet_solver.proto', input_shape=(1, 28, 28)).set(debug=True).setStatistics(True)
+lenet.fit(X_train, y_train)
+y_predicted = lenet.predict(X_test)
+```
+
## Frequently asked questions
- How to set batch size ?
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/docs/index.md
----------------------------------------------------------------------
diff --git a/docs/index.md b/docs/index.md
index c84e7b7..080dfd2 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -57,6 +57,7 @@ machine in R-like and Python-like declarative languages.
in Standalone Mode.
* [JMLC](jmlc) - Java Machine Learning Connector.
* See [Java Machine Learning Connector (JMLC)](jmlc) for more information.
+* *Experimental* [Caffe2DML API](http://apache.github.io/incubator-systemml/beginners-guide-caffe2dml) for Deep Learning.
## Language Guides
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/docs/native-backend.md
----------------------------------------------------------------------
diff --git a/docs/native-backend.md b/docs/native-backend.md
index 86a1340..c64c2fe 100644
--- a/docs/native-backend.md
+++ b/docs/native-backend.md
@@ -1,3 +1,8 @@
+---
+layout: global
+title: Using SystemML with Native BLAS support
+description: Using SystemML with Native BLAS support
+---
<!--
{% comment %}
Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,6 +22,11 @@ limitations under the License.
{% endcomment %}
-->
+* This will become a table of contents (this text will be scraped).
+{:toc}
+
+<br/>
+
# User Guide
By default, SystemML implements all its matrix operations in Java.
@@ -25,16 +35,16 @@ This simplifies deployment especially in a distributed environment.
In some cases (such as deep learning), the user might want to use native BLAS
rather than SystemML's internal Java library for performing single-node
operations such matrix multiplication, convolution, etc.
+
+To allow SystemML to use native BLAS rather than internal Java library,
+please set the configuration property `native.blas` to `true`.
+
By default, SystemML will first attempt to use Intel MKL (if installed)
and then OpenBLAS (if installed).
If both Intel MKL and OpenBLAS are not available, SystemML
falls back to its internal Java library.
-To force SystemML to use internal Java library rather than native BLAS,
-please set the configuration property `native.blas` to `false`.
-
-The current version of SystemML only supports BLAS on Linux machines.
-
+The current version of SystemML only supports BLAS on **Linux** machines.
## Step 1: Install BLAS
@@ -95,19 +105,20 @@ sudo ln -s /lib64/libgomp.so.1 /lib64/libgomp.so
## Step 3: Provide the location of the native libraries
-1. Add the location of the native libraries (i.e. BLAS and other dependencies)
+1. Pass the location of the native libraries using command-line options:
+
+- [Spark](http://spark.apache.org/docs/latest/configuration.html): `--conf spark.executorEnv.LD_LIBRARY_PATH=/path/to/blas-n-other-dependencies`
+- Java: `-Djava.library.path=/path/to/blas-n-other-dependencies`
+
+2. Alternatively, you can add the location of the native libraries (i.e. BLAS and other dependencies)
to the environment variable `LD_LIBRARY_PATH` (on Linux).
-If you want to use SystemML with Spark, please add the following line to `spark-env.sh`
+If you want to use SystemML with Spark, please add the following line to `spark-env.sh`
+(or to the bash profile).
```bash
export LD_LIBRARY_PATH=/path/to/blas-n-other-dependencies
- # Or export SPARK_LIBRARY_PATH=/path/to/blas-n-other-dependencies
```
-2. Alternatively, you can pass the location of the native libraries using command-line options:
-
-- Java: `-Djava.library.path=/path/to/blas-n-other-dependencies`
-- [Spark](http://spark.apache.org/docs/latest/configuration.html): `--driver-library-path`
## Common issues on Linux
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
index 56beef3..c70fbcf 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
@@ -256,12 +256,12 @@ public class ScriptExecutor {
*/
protected void setGlobalFlags() {
oldStatistics = DMLScript.STATISTICS;
- DMLScript.STATISTICS = statistics;
- oldForceGPU = DMLScript.FORCE_ACCELERATOR;
- DMLScript.FORCE_ACCELERATOR = forceGPU;
- oldGPU = DMLScript.USE_ACCELERATOR;
- DMLScript.USE_ACCELERATOR = gpu;
- DMLScript.STATISTICS_COUNT = statisticsMaxHeavyHitters;
+ DMLScript.STATISTICS = statistics;
+ oldForceGPU = DMLScript.FORCE_ACCELERATOR;
+ DMLScript.FORCE_ACCELERATOR = forceGPU;
+ oldGPU = DMLScript.USE_ACCELERATOR;
+ DMLScript.USE_ACCELERATOR = gpu;
+ DMLScript.STATISTICS_COUNT = statisticsMaxHeavyHitters;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/src/main/java/org/apache/sysml/conf/DMLConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/conf/DMLConfig.java b/src/main/java/org/apache/sysml/conf/DMLConfig.java
index 84bea42..5cc496e 100644
--- a/src/main/java/org/apache/sysml/conf/DMLConfig.java
+++ b/src/main/java/org/apache/sysml/conf/DMLConfig.java
@@ -116,7 +116,7 @@ public class DMLConfig
_defaultVals.put(CODEGEN, "false" );
_defaultVals.put(CODEGEN_PLANCACHE, "true" );
_defaultVals.put(CODEGEN_LITERALS, "1" );
- _defaultVals.put(NATIVE_BLAS, "true" );
+ _defaultVals.put(NATIVE_BLAS, "false" );
_defaultVals.put(EXTRA_GPU_STATS, "false" );
_defaultVals.put(EXTRA_DNN_STATS, "false" );
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/src/main/java/org/apache/sysml/utils/NativeHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/NativeHelper.java b/src/main/java/org/apache/sysml/utils/NativeHelper.java
index 2b997ed..303b7a8 100644
--- a/src/main/java/org/apache/sysml/utils/NativeHelper.java
+++ b/src/main/java/org/apache/sysml/utils/NativeHelper.java
@@ -25,6 +25,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import java.util.HashMap;
+import java.util.Vector;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.File;
@@ -94,24 +95,45 @@ public class NativeHelper {
if(userSpecifiedBLAS.equalsIgnoreCase("")) {
blasType = isMKLAvailable() ? "mkl" : isOpenBLASAvailable() ? "openblas" : null;
if(blasType == null)
- LOG.warn("Unable to load either MKL or OpenBLAS");
+ LOG.info("Unable to load either MKL or OpenBLAS. Please set ");
}
else if(userSpecifiedBLAS.equalsIgnoreCase("mkl")) {
blasType = isMKLAvailable() ? "mkl" : null;
if(blasType == null)
- LOG.warn("Unable to load MKL");
+ LOG.info("Unable to load MKL");
}
else if(userSpecifiedBLAS.equalsIgnoreCase("openblas")) {
blasType = isOpenBLASAvailable() ? "openblas" : null;
if(blasType == null)
- LOG.warn("Unable to load OpenBLAS");
+ LOG.info("Unable to load OpenBLAS");
}
else {
- LOG.warn("Unsupported BLAS:" + userSpecifiedBLAS);
+ LOG.info("Unsupported BLAS:" + userSpecifiedBLAS);
}
// =============================================================================
if(blasType != null && loadLibraryHelper("libsystemml_" + blasType + "-Linux-x86_64.so")) {
- LOG.info("Using native blas: " + blasType);
+ String blasPathAndHint = "";
+ // ------------------------------------------------------------
+ // This logic gets the list of native libraries that are loaded
+ try {
+ java.lang.reflect.Field loadedLibraryNamesField = ClassLoader.class.getDeclaredField("loadedLibraryNames");
+ loadedLibraryNamesField.setAccessible(true);
+ @SuppressWarnings("unchecked")
+ Vector<String> libraries = (Vector<String>) loadedLibraryNamesField.get(ClassLoader.getSystemClassLoader());
+ LOG.debug("List of native libraries loaded:" + libraries);
+ for(String library : libraries) {
+ if(library.endsWith("libmkl_rt.so"))
+ blasPathAndHint = " from the path " + library;
+ else if(library.endsWith("libopenblas.so")) {
+ blasPathAndHint = " from the path " + library + ". Hint: Please make sure that the libopenblas.so is built with GNU OpenMP threading (ldd " + library + " | grep libgomp).";
+ }
+ }
+ } catch (NoSuchFieldException | SecurityException | IllegalArgumentException | IllegalAccessException e) {
+ LOG.debug("Error while finding list of native libraries:" + e.getMessage());
+ }
+ // ------------------------------------------------------------
+
+ LOG.info("Using native blas: " + blasType + blasPathAndHint);
isSystemMLLoaded = true;
}
}
@@ -155,7 +177,7 @@ public class NativeHelper {
// ------------------------------------------------------------
// Set environment variable MKL_THREADING_LAYER to GNU on Linux for performance
if(!loadLibraryHelper("libpreload_systemml-Linux-x86_64.so")) {
- LOG.warn("Unable to load preload_systemml (required for loading MKL-enabled SystemML library)");
+ LOG.debug("Unable to load preload_systemml (required for loading MKL-enabled SystemML library)");
return false;
}
// The most reliable way in my investigation to ensure that MKL runs smoothly with OpenMP (used by conv2d*)
@@ -181,9 +203,9 @@ public class NativeHelper {
}
catch (UnsatisfiedLinkError e) {
if(optionalMsg != null)
- LOG.warn("Unable to load " + blas + "(" + optionalMsg + "):" + e.getMessage());
+ LOG.debug("Unable to load " + blas + "(" + optionalMsg + "):" + e.getMessage());
else
- LOG.warn("Unable to load " + blas + ":" + e.getMessage());
+ LOG.debug("Unable to load " + blas + ":" + e.getMessage());
return false;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala
----------------------------------------------------------------------
diff --git a/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala b/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala
index 7ab9160..22de112 100644
--- a/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala
+++ b/src/main/scala/org/apache/sysml/api/dl/Caffe2DML.scala
@@ -53,13 +53,10 @@ import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyze
object Caffe2DML {
- val LOG = LogFactory.getLog(classOf[Caffe2DML].getName())
- def fileSep():String = { if(File.separator.equals("\\")) "\\\\" else File.separator }
- def setNNLibraryPath(path:String):Unit = { prefix = path + fileSep + "nn"}
+ val LOG = LogFactory.getLog(classOf[Caffe2DML].getName())
// ------------------------------------------------------------------------
- var prefix = Utils.getPrefix()
- def layerDir = prefix + fileSep + "layers" + fileSep
- def optimDir = prefix + fileSep + "optim" + fileSep
+ def layerDir = "nn/layers/"
+ def optimDir = "nn/optim/"
// Naming conventions:
val X = "X"; val y = "y"; val batchSize = "BATCH_SIZE"; val numImages = "num_images"; val numValidationImages = "num_validation"
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/src/main/scala/org/apache/sysml/api/dl/CaffeNetwork.scala
----------------------------------------------------------------------
diff --git a/src/main/scala/org/apache/sysml/api/dl/CaffeNetwork.scala b/src/main/scala/org/apache/sysml/api/dl/CaffeNetwork.scala
index e585e30..73490b4 100644
--- a/src/main/scala/org/apache/sysml/api/dl/CaffeNetwork.scala
+++ b/src/main/scala/org/apache/sysml/api/dl/CaffeNetwork.scala
@@ -136,14 +136,14 @@ class CaffeNetwork(netFilePath:String, val currentPhase:Phase,
// The bottom layers are the layers available in the getBottomList (from Caffe .proto files)
private val _bottomLayers:Map[String, Set[String]] = convertTupleListToMap(
_caffeLayerParams.flatMap(l => expandBottomList(l.getName, l.getBottomList)))
- CaffeNetwork.LOG.info("Bottom layers:" + _bottomLayers)
+ CaffeNetwork.LOG.debug("Bottom layers:" + _bottomLayers)
// Find the top layers by reversing the bottom list
private val _topLayers:Map[String, Set[String]] = convertTupleListToMap(flipKeyValues(_bottomLayers.toList))
- CaffeNetwork.LOG.info("Top layers:" + _topLayers)
+ CaffeNetwork.LOG.debug("Top layers:" + _topLayers)
private val _layers: Map[String, CaffeLayer] = _caffeLayerParams.map(l => l.getName -> convertLayerParameterToCaffeLayer(l)).toMap
- CaffeNetwork.LOG.info("Layers:" + _layers)
+ CaffeNetwork.LOG.debug("Layers:" + _layers)
private val _layerIDs: Map[String, Int] = _layers.entrySet().map(x => x.getKey -> x.getValue.id).toMap
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/47c2dd10/src/main/scala/org/apache/sysml/api/dl/Utils.scala
----------------------------------------------------------------------
diff --git a/src/main/scala/org/apache/sysml/api/dl/Utils.scala b/src/main/scala/org/apache/sysml/api/dl/Utils.scala
index b9d6d33..5181c9b 100644
--- a/src/main/scala/org/apache/sysml/api/dl/Utils.scala
+++ b/src/main/scala/org/apache/sysml/api/dl/Utils.scala
@@ -76,18 +76,6 @@ object Utils {
}
}
-
- def getPrefix():String = {
- val f = new File("nn")
- if(f.exists() && f.isDirectory()) {
- Caffe2DML.LOG.info("Since nn directory exists in current folder, using it.")
- return "nn"
- }
- else {
- // TODO: Extract from the jar
- throw new RuntimeException("In current version, we require that you download the nn folder into current directory from https://github.com/apache/incubator-systemml/tree/master/scripts/staging/SystemML-NN")
- }
- }
// --------------------------------------------------------------
// Caffe utility functions