You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by ni...@apache.org on 2017/11/15 20:48:36 UTC
systemml git commit: [SYSTEMML-1630] [MINOR] Integrated setting of
blas directory with SystemML configuration property
Repository: systemml
Updated Branches:
refs/heads/master 2f87565d8 -> 1336d32a0
[SYSTEMML-1630] [MINOR] Integrated setting of blas directory with
SystemML configuration property
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/1336d32a
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/1336d32a
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/1336d32a
Branch: refs/heads/master
Commit: 1336d32a08d2458476be6bc38fa42367e27c3918
Parents: 2f87565
Author: Niketan Pansare <np...@us.ibm.com>
Authored: Wed Nov 15 12:47:40 2017 -0800
Committer: Niketan Pansare <np...@us.ibm.com>
Committed: Wed Nov 15 12:47:40 2017 -0800
----------------------------------------------------------------------
conf/SystemML-config.xml.template | 3 +
.../apache/sysml/api/ScriptExecutorUtils.java | 7 +
.../java/org/apache/sysml/conf/DMLConfig.java | 4 +-
.../org/apache/sysml/utils/NativeHelper.java | 152 ++++++++++---------
src/main/python/systemml/mlcontext.py | 21 +--
5 files changed, 95 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/1336d32a/conf/SystemML-config.xml.template
----------------------------------------------------------------------
diff --git a/conf/SystemML-config.xml.template b/conf/SystemML-config.xml.template
index 8a4a5d6..e5f0137 100644
--- a/conf/SystemML-config.xml.template
+++ b/conf/SystemML-config.xml.template
@@ -75,6 +75,9 @@
<!-- enables native blas for matrix multiplication and convolution, experimental feature (options: auto, mkl, openblas, none) -->
<sysml.native.blas>none</sysml.native.blas>
+ <!-- custom directory where BLAS libraries are available, experimental feature (options: absolute directory path or none). If set to none, we use standard LD_LIBRARY_PATH. -->
+ <sysml.native.blas.directory>none</sysml.native.blas.directory>
+
<!-- prints finegrained statistics information (includes extra GPU information and extra statistics information for Deep Neural Networks done in CP mode) -->
<sysml.stats.finegrained>false</sysml.stats.finegrained>
http://git-wip-us.apache.org/repos/asf/systemml/blob/1336d32a/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java b/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java
index cb39340..d69a863 100644
--- a/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java
+++ b/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java
@@ -30,6 +30,7 @@ import org.apache.sysml.runtime.controlprogram.Program;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.instructions.gpu.context.GPUContext;
import org.apache.sysml.runtime.instructions.gpu.context.GPUContextPool;
+import org.apache.sysml.utils.NativeHelper;
import org.apache.sysml.utils.Statistics;
public class ScriptExecutorUtils {
@@ -77,6 +78,12 @@ public class ScriptExecutorUtils {
DMLScript.SYNCHRONIZE_GPU = dmlconf.getBooleanValue(DMLConfig.SYNCHRONIZE_GPU);
DMLScript.EAGER_CUDA_FREE = dmlconf.getBooleanValue(DMLConfig.EAGER_CUDA_FREE);
DMLScript.STATISTICS_MAX_WRAP_LEN = dmlconf.getIntValue(DMLConfig.STATS_MAX_WRAP_LEN);
+
+ String customLibPath = dmlconf.getTextValue(DMLConfig.NATIVE_BLAS_DIR);
+ if(!customLibPath.equalsIgnoreCase("none")) {
+ NativeHelper.initializeCustomBLAS(customLibPath, dmlconf.getTextValue(DMLConfig.NATIVE_BLAS));
+ }
+
if(DMLScript.USE_ACCELERATOR) {
DMLScript.FLOATING_POINT_PRECISION = dmlconf.getTextValue(DMLConfig.FLOATING_POINT_PRECISION);
org.apache.sysml.runtime.matrix.data.LibMatrixCUDA.resetFloatingPointPrecision();
http://git-wip-us.apache.org/repos/asf/systemml/blob/1336d32a/src/main/java/org/apache/sysml/conf/DMLConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/conf/DMLConfig.java b/src/main/java/org/apache/sysml/conf/DMLConfig.java
index 42037a1..b5db1c1 100644
--- a/src/main/java/org/apache/sysml/conf/DMLConfig.java
+++ b/src/main/java/org/apache/sysml/conf/DMLConfig.java
@@ -76,6 +76,7 @@ public class DMLConfig
public static final String CP_PARALLEL_IO = "sysml.cp.parallel.io";
public static final String COMPRESSED_LINALG = "sysml.compressed.linalg"; //auto, true, false
public static final String NATIVE_BLAS = "sysml.native.blas";
+ public static final String NATIVE_BLAS_DIR = "sysml.native.blas.directory";
public static final String CODEGEN = "sysml.codegen.enabled"; //boolean
public static final String CODEGEN_COMPILER = "sysml.codegen.compiler"; //see SpoofCompiler.CompilerType
public static final String CODEGEN_OPTIMIZER = "sysml.codegen.optimizer"; //see SpoofCompiler.PlanSelector
@@ -130,6 +131,7 @@ public class DMLConfig
_defaultVals.put(CODEGEN_PLANCACHE, "true" );
_defaultVals.put(CODEGEN_LITERALS, "1" );
_defaultVals.put(NATIVE_BLAS, "none" );
+ _defaultVals.put(NATIVE_BLAS_DIR, "none" );
_defaultVals.put(EXTRA_FINEGRAINED_STATS,"false" );
_defaultVals.put(STATS_MAX_WRAP_LEN, "30" );
_defaultVals.put(GPU_MEMORY_UTILIZATION_FACTOR, "0.9" );
@@ -415,7 +417,7 @@ public class DMLConfig
LOCAL_TMP_DIR,SCRATCH_SPACE,OPTIMIZATION_LEVEL,
NUM_REDUCERS, DEFAULT_BLOCK_SIZE,
YARN_APPMASTER, YARN_APPMASTERMEM, YARN_MAPREDUCEMEM,
- CP_PARALLEL_OPS, CP_PARALLEL_IO, NATIVE_BLAS,
+ CP_PARALLEL_OPS, CP_PARALLEL_IO, NATIVE_BLAS, NATIVE_BLAS_DIR,
COMPRESSED_LINALG,
CODEGEN, CODEGEN_COMPILER, CODEGEN_OPTIMIZER, CODEGEN_PLANCACHE, CODEGEN_LITERALS,
EXTRA_FINEGRAINED_STATS, STATS_MAX_WRAP_LEN,
http://git-wip-us.apache.org/repos/asf/systemml/blob/1336d32a/src/main/java/org/apache/sysml/utils/NativeHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/NativeHelper.java b/src/main/java/org/apache/sysml/utils/NativeHelper.java
index f70ea14..c9c2e08 100644
--- a/src/main/java/org/apache/sysml/utils/NativeHelper.java
+++ b/src/main/java/org/apache/sysml/utils/NativeHelper.java
@@ -36,6 +36,7 @@ import org.apache.commons.lang.SystemUtils;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
/**
* This class helps in loading native library.
@@ -50,24 +51,33 @@ public class NativeHelper {
private static boolean setMaxNumThreads = false;
static {
// Note: we only support 64 bit Java on x86 and AMD machine
- supportedArchitectures.put("x86_64", "x86_64");
- supportedArchitectures.put("amd64", "x86_64");
+ supportedArchitectures.put("x86_64", "x86_64");
+ supportedArchitectures.put("amd64", "x86_64");
}
-
+
private static boolean attemptedLoading = false;
-
+
private static String hintOnFailures = "";
-
- public static void setBLASPath(String customLibPath, String userSpecifiedBLAS) {
- init(customLibPath, userSpecifiedBLAS);
+
+ public static void initializeCustomBLAS(String customLibPath, String userSpecifiedBLAS) throws DMLRuntimeException {
+ if(attemptedLoading && blasType != null && isSupportedBLAS(userSpecifiedBLAS) && !blasType.equalsIgnoreCase(userSpecifiedBLAS) ) {
+ throw new DMLRuntimeException("Cannot replace previously loaded blas \"" + blasType + "\" with \"" + userSpecifiedBLAS + "\".");
+ }
+ else {
+ init(customLibPath, userSpecifiedBLAS);
+ }
}
+ private static boolean isSupportedBLAS(String userSpecifiedBLAS) {
+ return userSpecifiedBLAS.equalsIgnoreCase("auto") || userSpecifiedBLAS.equalsIgnoreCase("mkl") || userSpecifiedBLAS.equalsIgnoreCase("openblas");
+ }
+
// Performing loading in a method instead of a static block will throw a detailed stack trace in case of fatal errors
private static void init(String customLibPath, String userSpecifiedBLAS) {
// Only Linux supported for BLAS
if(!SystemUtils.IS_OS_LINUX)
return;
-
+
// attemptedLoading variable ensures that we don't try to load SystemML and other dependencies
// again and again especially in the parfor (hence the double-checking with synchronized).
if(!attemptedLoading || customLibPath != null) {
@@ -76,47 +86,47 @@ public class NativeHelper {
DMLConfig dmlConfig = ConfigurationManager.getDMLConfig();
userSpecifiedBLAS = (dmlConfig == null) ? "auto" : dmlConfig.getTextValue(DMLConfig.NATIVE_BLAS).trim().toLowerCase();
}
- if(userSpecifiedBLAS.equals("auto") || userSpecifiedBLAS.equals("mkl") || userSpecifiedBLAS.equals("openblas")) {
+ if(isSupportedBLAS(userSpecifiedBLAS)) {
long start = System.nanoTime();
if(!supportedArchitectures.containsKey(SystemUtils.OS_ARCH)) {
LOG.info("Unsupported architecture for native BLAS:" + SystemUtils.OS_ARCH);
return;
}
- synchronized(NativeHelper.class) {
- if(!attemptedLoading || customLibPath != null) {
- // -----------------------------------------------------------------------------
- // =============================================================================
- // By default, we will native.blas=true and we will attempt to load MKL first.
- // If MKL is not enabled then we try to load OpenBLAS.
- // If both MKL and OpenBLAS are not available we fall back to Java BLAS.
- if(userSpecifiedBLAS.equals("auto")) {
- blasType = isMKLAvailable(customLibPath) ? "mkl" : isOpenBLASAvailable(customLibPath) ? "openblas" : null;
- if(blasType == null)
- LOG.info("Unable to load either MKL or OpenBLAS due to " + hintOnFailures);
- }
- else if(userSpecifiedBLAS.equals("mkl")) {
- blasType = isMKLAvailable(customLibPath) ? "mkl" : null;
- if(blasType == null)
- LOG.info("Unable to load MKL due to " + hintOnFailures);
- }
- else if(userSpecifiedBLAS.equals("openblas")) {
- blasType = isOpenBLASAvailable(customLibPath) ? "openblas" : null;
- if(blasType == null)
- LOG.info("Unable to load OpenBLAS due to " + hintOnFailures);
- }
- else {
- // Only thrown at development time.
- throw new RuntimeException("Unsupported BLAS:" + userSpecifiedBLAS);
- }
- // =============================================================================
- if(blasType != null && loadLibraryHelper("libsystemml_" + blasType + "-Linux-x86_64.so")) {
- String blasPathAndHint = "";
- // ------------------------------------------------------------
- // This logic gets the list of native libraries that are loaded
- if(LOG.isDebugEnabled()) {
- // Only perform the checking of library paths when DEBUG is enabled to avoid runtime overhead.
- try {
- java.lang.reflect.Field loadedLibraryNamesField = ClassLoader.class.getDeclaredField("loadedLibraryNames");
+ synchronized(NativeHelper.class) {
+ if(!attemptedLoading || customLibPath != null) {
+ // -----------------------------------------------------------------------------
+ // =============================================================================
+ // By default, we will native.blas=true and we will attempt to load MKL first.
+ // If MKL is not enabled then we try to load OpenBLAS.
+ // If both MKL and OpenBLAS are not available we fall back to Java BLAS.
+ if(userSpecifiedBLAS.equalsIgnoreCase("auto")) {
+ blasType = isMKLAvailable(customLibPath) ? "mkl" : isOpenBLASAvailable(customLibPath) ? "openblas" : null;
+ if(blasType == null)
+ LOG.info("Unable to load either MKL or OpenBLAS due to " + hintOnFailures);
+ }
+ else if(userSpecifiedBLAS.equalsIgnoreCase("mkl")) {
+ blasType = isMKLAvailable(customLibPath) ? "mkl" : null;
+ if(blasType == null)
+ LOG.info("Unable to load MKL due to " + hintOnFailures);
+ }
+ else if(userSpecifiedBLAS.equalsIgnoreCase("openblas")) {
+ blasType = isOpenBLASAvailable(customLibPath) ? "openblas" : null;
+ if(blasType == null)
+ LOG.info("Unable to load OpenBLAS due to " + hintOnFailures);
+ }
+ else {
+ // Only thrown at development time.
+ throw new RuntimeException("Unsupported BLAS:" + userSpecifiedBLAS);
+ }
+ // =============================================================================
+ if(blasType != null && loadLibraryHelper("libsystemml_" + blasType + "-Linux-x86_64.so")) {
+ String blasPathAndHint = "";
+ // ------------------------------------------------------------
+ // This logic gets the list of native libraries that are loaded
+ if(LOG.isDebugEnabled()) {
+ // Only perform the checking of library paths when DEBUG is enabled to avoid runtime overhead.
+ try {
+ java.lang.reflect.Field loadedLibraryNamesField = ClassLoader.class.getDeclaredField("loadedLibraryNames");
loadedLibraryNamesField.setAccessible(true);
@SuppressWarnings("unchecked")
Vector<String> libraries = (Vector<String>) loadedLibraryNamesField.get(ClassLoader.getSystemClassLoader());
@@ -130,25 +140,25 @@ public class NativeHelper {
} catch (NoSuchFieldException | SecurityException | IllegalArgumentException | IllegalAccessException e) {
LOG.debug("Error while finding list of native libraries:" + e.getMessage());
}
- }
- // ------------------------------------------------------------
-
+ }
+ // ------------------------------------------------------------
+
LOG.info("Using native blas: " + blasType + blasPathAndHint);
isSystemMLLoaded = true;
}
- }
- }
- double timeToLoadInMilliseconds = (System.nanoTime()-start)*1e-6;
- if(timeToLoadInMilliseconds > 1000)
- LOG.warn("Time to load native blas: " + timeToLoadInMilliseconds + " milliseconds.");
+ }
+ }
+ double timeToLoadInMilliseconds = (System.nanoTime()-start)*1e-6;
+ if(timeToLoadInMilliseconds > 1000)
+ LOG.warn("Time to load native blas: " + timeToLoadInMilliseconds + " milliseconds.");
}
else {
- LOG.debug("Using internal Java BLAS as native BLAS support the configuration 'native.blas'=" + userSpecifiedBLAS + ".");
+ LOG.debug("Using internal Java BLAS as native BLAS support the configuration 'sysml.native.blas'=" + userSpecifiedBLAS + ".");
}
attemptedLoading = true;
}
}
-
+
public static boolean isNativeLibraryLoaded() {
// We allow BLAS to be enabled or disabled or explicitly selected in one of the two ways:
// 1. DML Configuration: native.blas (boolean flag)
@@ -169,24 +179,24 @@ public class NativeHelper {
}
return isSystemMLLoaded;
}
-
+
public static int getMaxNumThreads() {
if(maxNumThreads == -1)
maxNumThreads = OptimizerUtils.getConstrainedNumThreads(-1);
return maxNumThreads;
}
-
-
+
+
private static boolean isMKLAvailable(String customLibPath) {
return loadBLAS(customLibPath, "mkl_rt", null);
}
-
+
private static boolean isOpenBLASAvailable(String customLibPath) {
if(!loadBLAS(customLibPath, "gomp", "gomp required for loading OpenBLAS-enabled SystemML library"))
return false;
return loadBLAS(customLibPath, "openblas", null);
}
-
+
private static boolean loadBLAS(String customLibPath, String blas, String optionalMsg) {
// First attempt to load from custom library path
if(customLibPath != null) {
@@ -202,11 +212,11 @@ public class NativeHelper {
System.out.println("Unable to load " + libPath + ":" + e1.getMessage());
}
}
-
+
// Then try loading using loadLibrary
try {
- System.loadLibrary(blas);
- return true;
+ System.loadLibrary(blas);
+ return true;
}
catch (UnsatisfiedLinkError e) {
if(!hintOnFailures.contains(blas))
@@ -218,7 +228,7 @@ public class NativeHelper {
return false;
}
}
-
+
private static boolean loadLibraryHelper(String path) {
InputStream in = null; OutputStream out = null;
@@ -229,9 +239,9 @@ public class NativeHelper {
File temp = File.createTempFile(path, "");
temp.deleteOnExit();
out = FileUtils.openOutputStream(temp);
- IOUtils.copy(in, out);
- in.close(); in = null;
- out.close(); out = null;
+ IOUtils.copy(in, out);
+ in.close(); in = null;
+ out.close(); out = null;
System.load(temp.getAbsolutePath());
return true;
}
@@ -251,17 +261,17 @@ public class NativeHelper {
}
return false;
}
-
+
// TODO: Add pmm, wsloss, mmchain, etc.
public static native boolean matrixMultDenseDense(double [] m1, double [] m2, double [] ret, int m1rlen, int m1clen, int m2clen, int numThreads);
private static native boolean tsmm(double [] m1, double [] ret, int m1rlen, int m1clen, boolean isLeftTranspose, int numThreads);
-
+
// ----------------------------------------------------------------------------------------------------------------
// LibMatrixDNN operations:
// N = number of images, C = number of channels, H = image height, W = image width
// K = number of filters, R = filter height, S = filter width
// TODO: case not handled: sparse filters (which will only be executed in Java). Since filters are relatively smaller, this is a low priority.
-
+
// Returns -1 if failures or returns number of nonzeros
// Called by ConvolutionCPInstruction if both input and filter are dense
public static native int conv2dDense(double [] input, double [] filter, double [] ret, int N, int C, int H, int W,
@@ -275,7 +285,7 @@ public class NativeHelper {
// Else, called by LibMatrixDNN's thread if filter is dense. dout[n] is converted to dense if sparse.
public static native int conv2dBackwardDataDense(double [] filter, double [] dout, double [] ret, int N, int C, int H, int W,
int K, int R, int S, int stride_h, int stride_w, int pad_h, int pad_w, int P, int Q, int numThreads);
-
+
// Currently only supported with numThreads = 1 and sparse input
// Called by LibMatrixDNN's thread if input is sparse. dout[n] is converted to dense if sparse.
public static native boolean conv2dBackwardFilterSparseDense(int apos, int alen, int[] aix, double[] avals, double [] rotatedDoutPtr, double [] ret, int N, int C, int H, int W,
@@ -284,7 +294,7 @@ public class NativeHelper {
public static native boolean conv2dSparse(int apos, int alen, int[] aix, double[] avals, double [] filter, double [] ret, int N, int C, int H, int W,
int K, int R, int S, int stride_h, int stride_w, int pad_h, int pad_w, int P, int Q, int numThreads);
// ----------------------------------------------------------------------------------------------------------------
-
+
// This method helps us decide whether to use GetPrimitiveArrayCritical or GetDoubleArrayElements in JNI as each has different tradeoffs.
// In current implementation, we always use GetPrimitiveArrayCritical as it has proven to be fastest.
// We can revisit this decision later and hence I would not recommend removing this method.
http://git-wip-us.apache.org/repos/asf/systemml/blob/1336d32a/src/main/python/systemml/mlcontext.py
----------------------------------------------------------------------
diff --git a/src/main/python/systemml/mlcontext.py b/src/main/python/systemml/mlcontext.py
index 704ba98..28f2d8c 100644
--- a/src/main/python/systemml/mlcontext.py
+++ b/src/main/python/systemml/mlcontext.py
@@ -22,7 +22,7 @@
# Methods to create Script object
script_factory_methods = [ 'dml', 'pydml', 'dmlFromResource', 'pydmlFromResource', 'dmlFromFile', 'pydmlFromFile', 'dmlFromUrl', 'pydmlFromUrl' ]
# Utility methods
-util_methods = [ 'jvm_stdout', '_java2py', 'getHopDAG', 'setBLASPath' ]
+util_methods = [ 'jvm_stdout', '_java2py', 'getHopDAG' ]
__all__ = ['MLResults', 'MLContext', 'Script', 'Matrix' ] + script_factory_methods + util_methods
import os
@@ -66,25 +66,6 @@ def _get_spark_context():
-def setBLASPath(path, blas='auto'):
- """
- This method useful in the cloud environment where the user
- doesnot have sudo permission or where setting environment variables
- such as LD_LIBRARY_PATH is difficult.
-
- Parameters
- ----------
- path: String
- Custom path to the directory where the BLAS shared libraries are located.
-
- blas: String
- Can be auto, openblas or mkl
- """
- sc = _get_spark_context()
- sc._jvm.org.apache.sysml.utils.NativeHelper.setBLASPath(str(path), blas)
-
-
-
# This is useful utility class to get the output of the driver JVM from within a Jupyter notebook
# Example usage:
# with jvm_stdout():