You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/04/01 23:18:46 UTC

systemml git commit: [SYSTEMML-1738] Fix various stream resource leaks via auto closing

Repository: systemml
Updated Branches:
  refs/heads/master 0abeb60b3 -> 607a402bc


[SYSTEMML-1738] Fix various stream resource leaks via auto closing

Closes #749.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/607a402b
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/607a402b
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/607a402b

Branch: refs/heads/master
Commit: 607a402bcf809d951e9634479d0db635e19c28ce
Parents: 0abeb60
Author: Janardhan Pulivarthi <j1...@protonmail.com>
Authored: Sun Apr 1 16:18:19 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sun Apr 1 16:18:19 2018 -0700

----------------------------------------------------------------------
 .../org/apache/sysml/api/jmlc/Connection.java   | 24 ++++++++++----
 .../sysml/api/mlcontext/ScriptFactory.java      | 10 +++---
 .../org/apache/sysml/parser/ParserWrapper.java  | 34 ++++++++++++--------
 .../instructions/gpu/context/JCudaKernels.java  |  5 +--
 .../runtime/transform/meta/TfMetaUtils.java     | 16 ++++++---
 .../org/apache/sysml/utils/NativeHelper.java    | 25 ++++++--------
 6 files changed, 65 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
index 8e8bb29..8684e72 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
@@ -421,7 +421,9 @@ public class Connection implements Closeable
 	 * @throws IOException if IOException occurs
 	 */
 	public double[][] convertToDoubleMatrix(String input, int rows, int cols) throws IOException {
-		return convertToDoubleMatrix(IOUtilFunctions.toInputStream(input), rows, cols);
+		try( InputStream is = IOUtilFunctions.toInputStream(input) ) {
+			return convertToDoubleMatrix(is, rows, cols);
+		}
 	}
 	
 	/**
@@ -467,7 +469,9 @@ public class Connection implements Closeable
 	 * @throws IOException if IOException occurs
 	 */
 	public MatrixBlock convertToMatrix(String input, String meta) throws IOException {
-		return convertToMatrix(IOUtilFunctions.toInputStream(input), meta);
+		try( InputStream is = IOUtilFunctions.toInputStream(input) ) {
+			return convertToMatrix(is, meta);
+		}
 	}
 	
 	/**
@@ -509,7 +513,9 @@ public class Connection implements Closeable
 	 * @throws IOException if IOException occurs
 	 */
 	public MatrixBlock convertToMatrix(String input, int rows, int cols) throws IOException {
-		return convertToMatrix(IOUtilFunctions.toInputStream(input), rows, cols);
+		try( InputStream is = IOUtilFunctions.toInputStream(input) ) {
+			return convertToMatrix(is, rows, cols);
+		}
 	}
 	
 	/**
@@ -655,7 +661,9 @@ public class Connection implements Closeable
 	 * @throws IOException if IOException occurs
 	 */
 	public String[][] convertToStringFrame(String input, int rows, int cols) throws IOException {
-		return convertToStringFrame(IOUtilFunctions.toInputStream(input), rows, cols);
+		try( InputStream is = IOUtilFunctions.toInputStream(input) ) {
+			return convertToStringFrame(is, rows, cols);
+		}
 	}
 	
 	/**
@@ -701,7 +709,9 @@ public class Connection implements Closeable
 	 * @throws IOException if IOException occurs
 	 */
 	public FrameBlock convertToFrame(String input, String meta) throws IOException {
-		return convertToFrame(IOUtilFunctions.toInputStream(input), meta);
+		try( InputStream is = IOUtilFunctions.toInputStream(input) ) {
+			return convertToFrame(is, meta);
+		}
 	}
 	
 	/**
@@ -743,7 +753,9 @@ public class Connection implements Closeable
 	 * @throws IOException if IOException occurs
 	 */
 	public FrameBlock convertToFrame(String input, int rows, int cols) throws IOException {
-		return convertToFrame(IOUtilFunctions.toInputStream(input), rows, cols);
+		try( InputStream is = IOUtilFunctions.toInputStream(input) ) {
+			return convertToFrame(is, rows, cols);
+		}
 	}
 	
 	/**

http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java
index 10cd947..567a2d7 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java
@@ -298,8 +298,11 @@ public class ScriptFactory {
 		if (!resourcePath.startsWith("/")) {
 			resourcePath = "/" + resourcePath;
 		}
-		InputStream inputStream = ScriptFactory.class.getResourceAsStream(resourcePath);
-		return scriptFromInputStream(inputStream, scriptType).setName(resourcePath);
+		try( InputStream inputStream = ScriptFactory.class.getResourceAsStream(resourcePath) ) {
+			return scriptFromInputStream(inputStream, scriptType).setName(resourcePath);
+		} catch (Exception e){
+			throw new MLContextException("Error trying to read script from resource: "+ resourcePath, e);
+		}
 	}
 
 	/**
@@ -425,8 +428,7 @@ public class ScriptFactory {
 		if ((!urlString.toLowerCase().startsWith("http:")) && (!urlString.toLowerCase().startsWith("https:"))) {
 			throw new MLContextException("Currently only reading from http and https URLs is supported");
 		}
-		try {
-			InputStream is = url.openStream();
+		try( InputStream is = url.openStream() ) {
 			return IOUtils.toString(is);
 		} catch (IOException e) {
 			throw new MLContextException("Error trying to read script string from URL: " + url, e);

http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/parser/ParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ParserWrapper.java b/src/main/java/org/apache/sysml/parser/ParserWrapper.java
index 0f43971..8dc9712 100644
--- a/src/main/java/org/apache/sysml/parser/ParserWrapper.java
+++ b/src/main/java/org/apache/sysml/parser/ParserWrapper.java
@@ -124,24 +124,30 @@ public abstract class ParserWrapper {
 		{
 			String resPath = scriptPathToResourcePath(script);
 			LOG.debug("Looking for the following resource from the SystemML jar file: " + resPath);
-			InputStream is = ParserWrapper.class.getResourceAsStream(resPath);
-			if (is == null) {
-				if (resPath.startsWith("/scripts")) {
-					LOG.error("Failed to read from the file system ('" + script + "') or SystemML jar file ('" + resPath + "')");
-					throw ex;
-				} else {
-					// for accessing script packages in the scripts directory
-					String scriptsResPath = "/scripts" + resPath;
-					LOG.debug("Looking for the following resource from the SystemML jar file: " + scriptsResPath);
-					is = ParserWrapper.class.getResourceAsStream(scriptsResPath);
-					if (is == null) {
-						LOG.error("Failed to read from the file system ('" + script + "') or SystemML jar file ('" + resPath + "' or '" + scriptsResPath + "')");
+			InputStream is = null;
+			try {
+				is = ParserWrapper.class.getResourceAsStream(resPath);
+				if (is == null) {
+					if (resPath.startsWith("/scripts")) {
+						LOG.error("Failed to read from the file system ('" + script + "') or SystemML jar file ('" + resPath + "')");
 						throw ex;
+					} else {
+						// for accessing script packages in the scripts directory
+						String scriptsResPath = "/scripts" + resPath;
+						LOG.debug("Looking for the following resource from the SystemML jar file: " + scriptsResPath);
+						is = ParserWrapper.class.getResourceAsStream(scriptsResPath);
+						if (is == null) {
+							LOG.error("Failed to read from the file system ('" + script + "') or SystemML jar file ('" + resPath + "' or '" + scriptsResPath + "')");
+							throw ex;
+						}
 					}
 				}
+				String s = IOUtils.toString(is);
+				return s;
+			}
+			finally {
+				IOUtilFunctions.closeSilently(is);
 			}
-			String s = IOUtils.toString(is);
-			return s;
 		}
 		finally {
 			IOUtilFunctions.closeSilently(in);

http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java
index 743c7e6..9a0b4c5 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java
@@ -125,10 +125,8 @@ public class JCudaKernels {
 	 * @return
 	 */
 	private static Pointer initKernels(String ptxFileName) {
-		InputStream in = null;
 		ByteArrayOutputStream out = null;
-		try {
-			in = JCudaKernels.class.getResourceAsStream(ptxFileName);
+		try( InputStream in = JCudaKernels.class.getResourceAsStream(ptxFileName) ) {
 			if (in != null) {
 				out = new ByteArrayOutputStream();
 				byte buffer[] = new byte[8192];
@@ -149,7 +147,6 @@ public class JCudaKernels {
 			throw new DMLRuntimeException("Could not initialize the kernels", e);
 		} finally {
 			IOUtilFunctions.closeSilently(out);
-			IOUtilFunctions.closeSilently(in);
 		}
 	}
 }

http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java b/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java
index 9242d4a..2d89502 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java
@@ -216,7 +216,7 @@ public class TfMetaUtils
 		throws IOException 
 	{
 		//read column names
-		String colnamesStr = IOUtilFunctions.toString(Connection.class.getResourceAsStream(metapath+"/"+TfUtils.TXMTD_COLNAMES));
+		String colnamesStr = getStringFromResource(metapath+"/"+TfUtils.TXMTD_COLNAMES);
 		String[] colnames = IOUtilFunctions.split(colnamesStr.trim(), colDelim);
 		
 		//read meta data (currently supported: recode, dummycode, bin, omit)
@@ -228,22 +228,22 @@ public class TfMetaUtils
 			String colName = colnames[j];
 			//read recode maps for recoded or dummycoded columns
 			String name = metapath+"/"+"Recode"+"/"+colName;
-			String map = IOUtilFunctions.toString(Connection.class.getResourceAsStream(name+TfUtils.TXMTD_RCD_MAP_SUFFIX));
+			String map = getStringFromResource(name+TfUtils.TXMTD_RCD_MAP_SUFFIX);
 			if( map != null ) {
 				meta.put(colName, map);
-				String ndistinct = IOUtilFunctions.toString(Connection.class.getResourceAsStream(name+TfUtils.TXMTD_RCD_DISTINCT_SUFFIX));
+				String ndistinct = getStringFromResource(name+TfUtils.TXMTD_RCD_DISTINCT_SUFFIX);
 				rows = Math.max(rows, Integer.parseInt(ndistinct));
 			}
 			//read binning map for binned columns
 			String name2 = metapath+"/"+"Bin"+"/"+colName;
-			String map2 = IOUtilFunctions.toString(Connection.class.getResourceAsStream(name2+TfUtils.TXMTD_BIN_FILE_SUFFIX));
+			String map2 = getStringFromResource(name2+TfUtils.TXMTD_BIN_FILE_SUFFIX);
 			if( map2 != null ) {
 				meta.put(colName, map2);
 				rows = Math.max(rows, Integer.parseInt(map2.split(TfUtils.TXMTD_SEP)[4]));
 			}
 			//read impute value for mv columns
 			String name3 = metapath+File.separator+"Impute"+File.separator+colName;
-			String map3 = IOUtilFunctions.toString(Connection.class.getResourceAsStream(name3+TfUtils.TXMTD_MV_FILE_SUFFIX));
+			String map3 = getStringFromResource(name3+TfUtils.TXMTD_MV_FILE_SUFFIX);
 			if( map3 != null ) {
 				mvmeta.put(colName, map3);
 			}
@@ -389,4 +389,10 @@ public class TfMetaUtils
 			throw new IOException(ex);
 		}
 	}
+	
+	private static String getStringFromResource(String path) throws IOException {
+		try(InputStream is = Connection.class.getResourceAsStream(path) ) {
+			return IOUtilFunctions.toString(is);
+		}
+	}
 }

http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/utils/NativeHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/NativeHelper.java b/src/main/java/org/apache/sysml/utils/NativeHelper.java
index 246c5bd..86d849f 100644
--- a/src/main/java/org/apache/sysml/utils/NativeHelper.java
+++ b/src/main/java/org/apache/sysml/utils/NativeHelper.java
@@ -37,6 +37,7 @@ import org.apache.sysml.conf.ConfigurationManager;
 import org.apache.sysml.conf.DMLConfig;
 import org.apache.sysml.hops.OptimizerUtils;
 import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.io.IOUtilFunctions;
 
 /**
  * This class helps in loading native library.
@@ -291,33 +292,25 @@ public class NativeHelper {
 
 
 	private static boolean loadLibraryHelper(String path)  {
-		InputStream in = null; OutputStream out = null;
-		try {
-			// This logic is added because Java doesnot allow to load library from a resource file.
-			in = NativeHelper.class.getResourceAsStream("/lib/"+path);
+		OutputStream out = null;
+		try( InputStream in = NativeHelper.class.getResourceAsStream("/lib/"+path) ) {
+			// This logic is added because Java does not allow to load library from a resource file.
 			if(in != null) {
 				File temp = File.createTempFile(path, "");
 				temp.deleteOnExit();
 				out = FileUtils.openOutputStream(temp);
 				IOUtils.copy(in, out);
-				in.close(); in = null;
-				out.close(); out = null;
 				System.load(temp.getAbsolutePath());
 				return true;
 			}
 			else
 				LOG.warn("No lib available in the jar:" + path);
-		} catch(IOException e) {
+		}
+		catch(IOException e) {
 			LOG.warn("Unable to load library " + path + " from resource:" + e.getMessage());
-		} finally {
-			if(out != null)
-				try {
-					out.close();
-				} catch (IOException e) {}
-			if(in != null)
-				try {
-					in.close();
-				} catch (IOException e) {}
+		}
+		finally {
+			IOUtilFunctions.closeSilently(out);
 		}
 		return false;
 	}