You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/04/01 23:18:46 UTC
systemml git commit: [SYSTEMML-1738] Fix various stream resource
leaks via auto closing
Repository: systemml
Updated Branches:
refs/heads/master 0abeb60b3 -> 607a402bc
[SYSTEMML-1738] Fix various stream resource leaks via auto closing
Closes #749.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/607a402b
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/607a402b
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/607a402b
Branch: refs/heads/master
Commit: 607a402bcf809d951e9634479d0db635e19c28ce
Parents: 0abeb60
Author: Janardhan Pulivarthi <j1...@protonmail.com>
Authored: Sun Apr 1 16:18:19 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sun Apr 1 16:18:19 2018 -0700
----------------------------------------------------------------------
.../org/apache/sysml/api/jmlc/Connection.java | 24 ++++++++++----
.../sysml/api/mlcontext/ScriptFactory.java | 10 +++---
.../org/apache/sysml/parser/ParserWrapper.java | 34 ++++++++++++--------
.../instructions/gpu/context/JCudaKernels.java | 5 +--
.../runtime/transform/meta/TfMetaUtils.java | 16 ++++++---
.../org/apache/sysml/utils/NativeHelper.java | 25 ++++++--------
6 files changed, 65 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
index 8e8bb29..8684e72 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
@@ -421,7 +421,9 @@ public class Connection implements Closeable
* @throws IOException if IOException occurs
*/
public double[][] convertToDoubleMatrix(String input, int rows, int cols) throws IOException {
- return convertToDoubleMatrix(IOUtilFunctions.toInputStream(input), rows, cols);
+ try( InputStream is = IOUtilFunctions.toInputStream(input) ) {
+ return convertToDoubleMatrix(is, rows, cols);
+ }
}
/**
@@ -467,7 +469,9 @@ public class Connection implements Closeable
* @throws IOException if IOException occurs
*/
public MatrixBlock convertToMatrix(String input, String meta) throws IOException {
- return convertToMatrix(IOUtilFunctions.toInputStream(input), meta);
+ try( InputStream is = IOUtilFunctions.toInputStream(input) ) {
+ return convertToMatrix(is, meta);
+ }
}
/**
@@ -509,7 +513,9 @@ public class Connection implements Closeable
* @throws IOException if IOException occurs
*/
public MatrixBlock convertToMatrix(String input, int rows, int cols) throws IOException {
- return convertToMatrix(IOUtilFunctions.toInputStream(input), rows, cols);
+ try( InputStream is = IOUtilFunctions.toInputStream(input) ) {
+ return convertToMatrix(is, rows, cols);
+ }
}
/**
@@ -655,7 +661,9 @@ public class Connection implements Closeable
* @throws IOException if IOException occurs
*/
public String[][] convertToStringFrame(String input, int rows, int cols) throws IOException {
- return convertToStringFrame(IOUtilFunctions.toInputStream(input), rows, cols);
+ try( InputStream is = IOUtilFunctions.toInputStream(input) ) {
+ return convertToStringFrame(is, rows, cols);
+ }
}
/**
@@ -701,7 +709,9 @@ public class Connection implements Closeable
* @throws IOException if IOException occurs
*/
public FrameBlock convertToFrame(String input, String meta) throws IOException {
- return convertToFrame(IOUtilFunctions.toInputStream(input), meta);
+ try( InputStream is = IOUtilFunctions.toInputStream(input) ) {
+ return convertToFrame(is, meta);
+ }
}
/**
@@ -743,7 +753,9 @@ public class Connection implements Closeable
* @throws IOException if IOException occurs
*/
public FrameBlock convertToFrame(String input, int rows, int cols) throws IOException {
- return convertToFrame(IOUtilFunctions.toInputStream(input), rows, cols);
+ try( InputStream is = IOUtilFunctions.toInputStream(input) ) {
+ return convertToFrame(is, rows, cols);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java
index 10cd947..567a2d7 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java
@@ -298,8 +298,11 @@ public class ScriptFactory {
if (!resourcePath.startsWith("/")) {
resourcePath = "/" + resourcePath;
}
- InputStream inputStream = ScriptFactory.class.getResourceAsStream(resourcePath);
- return scriptFromInputStream(inputStream, scriptType).setName(resourcePath);
+ try( InputStream inputStream = ScriptFactory.class.getResourceAsStream(resourcePath) ) {
+ return scriptFromInputStream(inputStream, scriptType).setName(resourcePath);
+ } catch (Exception e){
+ throw new MLContextException("Error trying to read script from resource: "+ resourcePath, e);
+ }
}
/**
@@ -425,8 +428,7 @@ public class ScriptFactory {
if ((!urlString.toLowerCase().startsWith("http:")) && (!urlString.toLowerCase().startsWith("https:"))) {
throw new MLContextException("Currently only reading from http and https URLs is supported");
}
- try {
- InputStream is = url.openStream();
+ try( InputStream is = url.openStream() ) {
return IOUtils.toString(is);
} catch (IOException e) {
throw new MLContextException("Error trying to read script string from URL: " + url, e);
http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/parser/ParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ParserWrapper.java b/src/main/java/org/apache/sysml/parser/ParserWrapper.java
index 0f43971..8dc9712 100644
--- a/src/main/java/org/apache/sysml/parser/ParserWrapper.java
+++ b/src/main/java/org/apache/sysml/parser/ParserWrapper.java
@@ -124,24 +124,30 @@ public abstract class ParserWrapper {
{
String resPath = scriptPathToResourcePath(script);
LOG.debug("Looking for the following resource from the SystemML jar file: " + resPath);
- InputStream is = ParserWrapper.class.getResourceAsStream(resPath);
- if (is == null) {
- if (resPath.startsWith("/scripts")) {
- LOG.error("Failed to read from the file system ('" + script + "') or SystemML jar file ('" + resPath + "')");
- throw ex;
- } else {
- // for accessing script packages in the scripts directory
- String scriptsResPath = "/scripts" + resPath;
- LOG.debug("Looking for the following resource from the SystemML jar file: " + scriptsResPath);
- is = ParserWrapper.class.getResourceAsStream(scriptsResPath);
- if (is == null) {
- LOG.error("Failed to read from the file system ('" + script + "') or SystemML jar file ('" + resPath + "' or '" + scriptsResPath + "')");
+ InputStream is = null;
+ try {
+ is = ParserWrapper.class.getResourceAsStream(resPath);
+ if (is == null) {
+ if (resPath.startsWith("/scripts")) {
+ LOG.error("Failed to read from the file system ('" + script + "') or SystemML jar file ('" + resPath + "')");
throw ex;
+ } else {
+ // for accessing script packages in the scripts directory
+ String scriptsResPath = "/scripts" + resPath;
+ LOG.debug("Looking for the following resource from the SystemML jar file: " + scriptsResPath);
+ is = ParserWrapper.class.getResourceAsStream(scriptsResPath);
+ if (is == null) {
+ LOG.error("Failed to read from the file system ('" + script + "') or SystemML jar file ('" + resPath + "' or '" + scriptsResPath + "')");
+ throw ex;
+ }
}
}
+ String s = IOUtils.toString(is);
+ return s;
+ }
+ finally {
+ IOUtilFunctions.closeSilently(is);
}
- String s = IOUtils.toString(is);
- return s;
}
finally {
IOUtilFunctions.closeSilently(in);
http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java
index 743c7e6..9a0b4c5 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java
@@ -125,10 +125,8 @@ public class JCudaKernels {
* @return
*/
private static Pointer initKernels(String ptxFileName) {
- InputStream in = null;
ByteArrayOutputStream out = null;
- try {
- in = JCudaKernels.class.getResourceAsStream(ptxFileName);
+ try( InputStream in = JCudaKernels.class.getResourceAsStream(ptxFileName) ) {
if (in != null) {
out = new ByteArrayOutputStream();
byte buffer[] = new byte[8192];
@@ -149,7 +147,6 @@ public class JCudaKernels {
throw new DMLRuntimeException("Could not initialize the kernels", e);
} finally {
IOUtilFunctions.closeSilently(out);
- IOUtilFunctions.closeSilently(in);
}
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java b/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java
index 9242d4a..2d89502 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java
@@ -216,7 +216,7 @@ public class TfMetaUtils
throws IOException
{
//read column names
- String colnamesStr = IOUtilFunctions.toString(Connection.class.getResourceAsStream(metapath+"/"+TfUtils.TXMTD_COLNAMES));
+ String colnamesStr = getStringFromResource(metapath+"/"+TfUtils.TXMTD_COLNAMES);
String[] colnames = IOUtilFunctions.split(colnamesStr.trim(), colDelim);
//read meta data (currently supported: recode, dummycode, bin, omit)
@@ -228,22 +228,22 @@ public class TfMetaUtils
String colName = colnames[j];
//read recode maps for recoded or dummycoded columns
String name = metapath+"/"+"Recode"+"/"+colName;
- String map = IOUtilFunctions.toString(Connection.class.getResourceAsStream(name+TfUtils.TXMTD_RCD_MAP_SUFFIX));
+ String map = getStringFromResource(name+TfUtils.TXMTD_RCD_MAP_SUFFIX);
if( map != null ) {
meta.put(colName, map);
- String ndistinct = IOUtilFunctions.toString(Connection.class.getResourceAsStream(name+TfUtils.TXMTD_RCD_DISTINCT_SUFFIX));
+ String ndistinct = getStringFromResource(name+TfUtils.TXMTD_RCD_DISTINCT_SUFFIX);
rows = Math.max(rows, Integer.parseInt(ndistinct));
}
//read binning map for binned columns
String name2 = metapath+"/"+"Bin"+"/"+colName;
- String map2 = IOUtilFunctions.toString(Connection.class.getResourceAsStream(name2+TfUtils.TXMTD_BIN_FILE_SUFFIX));
+ String map2 = getStringFromResource(name2+TfUtils.TXMTD_BIN_FILE_SUFFIX);
if( map2 != null ) {
meta.put(colName, map2);
rows = Math.max(rows, Integer.parseInt(map2.split(TfUtils.TXMTD_SEP)[4]));
}
//read impute value for mv columns
String name3 = metapath+File.separator+"Impute"+File.separator+colName;
- String map3 = IOUtilFunctions.toString(Connection.class.getResourceAsStream(name3+TfUtils.TXMTD_MV_FILE_SUFFIX));
+ String map3 = getStringFromResource(name3+TfUtils.TXMTD_MV_FILE_SUFFIX);
if( map3 != null ) {
mvmeta.put(colName, map3);
}
@@ -389,4 +389,10 @@ public class TfMetaUtils
throw new IOException(ex);
}
}
+
+ private static String getStringFromResource(String path) throws IOException {
+ try(InputStream is = Connection.class.getResourceAsStream(path) ) {
+ return IOUtilFunctions.toString(is);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/utils/NativeHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/NativeHelper.java b/src/main/java/org/apache/sysml/utils/NativeHelper.java
index 246c5bd..86d849f 100644
--- a/src/main/java/org/apache/sysml/utils/NativeHelper.java
+++ b/src/main/java/org/apache/sysml/utils/NativeHelper.java
@@ -37,6 +37,7 @@ import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.io.IOUtilFunctions;
/**
* This class helps in loading native library.
@@ -291,33 +292,25 @@ public class NativeHelper {
private static boolean loadLibraryHelper(String path) {
- InputStream in = null; OutputStream out = null;
- try {
- // This logic is added because Java doesnot allow to load library from a resource file.
- in = NativeHelper.class.getResourceAsStream("/lib/"+path);
+ OutputStream out = null;
+ try( InputStream in = NativeHelper.class.getResourceAsStream("/lib/"+path) ) {
+ // This logic is added because Java does not allow to load library from a resource file.
if(in != null) {
File temp = File.createTempFile(path, "");
temp.deleteOnExit();
out = FileUtils.openOutputStream(temp);
IOUtils.copy(in, out);
- in.close(); in = null;
- out.close(); out = null;
System.load(temp.getAbsolutePath());
return true;
}
else
LOG.warn("No lib available in the jar:" + path);
- } catch(IOException e) {
+ }
+ catch(IOException e) {
LOG.warn("Unable to load library " + path + " from resource:" + e.getMessage());
- } finally {
- if(out != null)
- try {
- out.close();
- } catch (IOException e) {}
- if(in != null)
- try {
- in.close();
- } catch (IOException e) {}
+ }
+ finally {
+ IOUtilFunctions.closeSilently(out);
}
return false;
}