You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by ni...@apache.org on 2017/11/15 06:16:29 UTC
systemml git commit: [SYSTEMML-1630] Allow user to specify custom
path to BLAS libraries
Repository: systemml
Updated Branches:
refs/heads/master 5df6ab6dd -> 62b5725d8
[SYSTEMML-1630] Allow user to specify custom path to BLAS libraries
- This feature is useful in the cloud environment where the user doesnot
have sudo permission or where setting environment variables such as
LD_LIBRARY_PATH is difficult.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/62b5725d
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/62b5725d
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/62b5725d
Branch: refs/heads/master
Commit: 62b5725d82c5a24f05d6178a7498a9124746bf36
Parents: 5df6ab6
Author: Niketan Pansare <np...@us.ibm.com>
Authored: Tue Nov 14 22:10:09 2017 -0800
Committer: Niketan Pansare <np...@us.ibm.com>
Committed: Tue Nov 14 22:14:23 2017 -0800
----------------------------------------------------------------------
.../org/apache/sysml/utils/NativeHelper.java | 31 +++++++++++++++++---
src/main/python/systemml/mlcontext.py | 20 ++++++++++++-
2 files changed, 46 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/62b5725d/src/main/java/org/apache/sysml/utils/NativeHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/NativeHelper.java b/src/main/java/org/apache/sysml/utils/NativeHelper.java
index 63eea16..b57cfd6 100644
--- a/src/main/java/org/apache/sysml/utils/NativeHelper.java
+++ b/src/main/java/org/apache/sysml/utils/NativeHelper.java
@@ -48,6 +48,7 @@ public class NativeHelper {
public static String blasType;
private static int maxNumThreads = -1;
private static boolean setMaxNumThreads = false;
+ private static String customLibPath = null;
static {
// Note: we only support 64 bit Java on x86 and AMD machine
supportedArchitectures.put("x86_64", "x86_64");
@@ -58,15 +59,20 @@ public class NativeHelper {
private static String hintOnFailures = "";
+ public static void setBLASPath(String path) {
+ customLibPath = path;
+ init(true);
+ }
+
// Performing loading in a method instead of a static block will throw a detailed stack trace in case of fatal errors
- private static void init() {
+ private static void init(boolean forcedInit) {
// Only Linux supported for BLAS
if(!SystemUtils.IS_OS_LINUX)
return;
// attemptedLoading variable ensures that we don't try to load SystemML and other dependencies
// again and again especially in the parfor (hence the double-checking with synchronized).
- if(!attemptedLoading) {
+ if(!attemptedLoading || forcedInit) {
DMLConfig dmlConfig = ConfigurationManager.getDMLConfig();
// -------------------------------------------------------------------------------------
// We allow BLAS to be enabled or disabled or explicitly selected in one of the two ways:
@@ -85,7 +91,7 @@ public class NativeHelper {
return;
}
synchronized(NativeHelper.class) {
- if(!attemptedLoading) {
+ if(!attemptedLoading || forcedInit) {
// -----------------------------------------------------------------------------
// =============================================================================
// By default, we will native.blas=true and we will attempt to load MKL first.
@@ -152,7 +158,7 @@ public class NativeHelper {
}
public static boolean isNativeLibraryLoaded() {
- init();
+ init(false);
if(maxNumThreads == -1)
maxNumThreads = OptimizerUtils.getConstrainedNumThreads(-1);
if(isSystemMLLoaded && !setMaxNumThreads && maxNumThreads != -1) {
@@ -183,6 +189,22 @@ public class NativeHelper {
}
private static boolean loadBLAS(String blas, String optionalMsg) {
+ // First attempt to load from custom library path
+ if(customLibPath != null) {
+ String libPath = customLibPath + File.separator + System.mapLibraryName(blas);
+ try {
+ System.load(libPath);
+ // Print to stdout as this feature is intended for cloud environment
+ System.out.println("Loaded the library:" + libPath);
+ return true;
+ }
+ catch (UnsatisfiedLinkError e1) {
+ // Print to stdout as this feature is intended for cloud environment
+ System.out.println("Unable to load " + libPath + ":" + e1.getMessage());
+ }
+ }
+
+ // Then try loading using loadLibrary
try {
System.loadLibrary(blas);
return true;
@@ -197,6 +219,7 @@ public class NativeHelper {
return false;
}
}
+
private static boolean loadLibraryHelper(String path) {
InputStream in = null; OutputStream out = null;
http://git-wip-us.apache.org/repos/asf/systemml/blob/62b5725d/src/main/python/systemml/mlcontext.py
----------------------------------------------------------------------
diff --git a/src/main/python/systemml/mlcontext.py b/src/main/python/systemml/mlcontext.py
index 54e1969..af1a57e 100644
--- a/src/main/python/systemml/mlcontext.py
+++ b/src/main/python/systemml/mlcontext.py
@@ -22,7 +22,7 @@
# Methods to create Script object
script_factory_methods = [ 'dml', 'pydml', 'dmlFromResource', 'pydmlFromResource', 'dmlFromFile', 'pydmlFromFile', 'dmlFromUrl', 'pydmlFromUrl' ]
# Utility methods
-util_methods = [ 'jvm_stdout', '_java2py', 'getHopDAG' ]
+util_methods = [ 'jvm_stdout', '_java2py', 'getHopDAG', 'setBLASPath' ]
__all__ = ['MLResults', 'MLContext', 'Script', 'Matrix' ] + script_factory_methods + util_methods
import os
@@ -64,6 +64,24 @@ def _get_spark_context():
else:
raise Exception('Expected spark context to be created.')
+
+
+def setBLASPath(path):
+ """
+ This method useful in the cloud environment where the user
+ doesnot have sudo permission or where setting environment variables
+ such as LD_LIBRARY_PATH is difficult.
+
+ Parameters
+ ----------
+ path: String
+ Custom path where the BLAS libraries where located.
+ """
+ sc = _get_spark_context()
+ sc._jvm.org.apache.sysml.utils.NativeHelper.setBLASPath(path)
+
+
+
# This is useful utility class to get the output of the driver JVM from within a Jupyter notebook
# Example usage:
# with jvm_stdout():