You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/06/05 02:44:31 UTC

systemml git commit: [SYSTEMML-2361] Fix invalid cleanup of matrices/frames read in functions

Repository: systemml
Updated Branches:
  refs/heads/master 1bcdfaac1 -> add561b38


[SYSTEMML-2361] Fix invalid cleanup of matrices/frames read in functions

This patch fixes a severe bug that led to invalid cleanup of matrices or
frames that are read from persistent storage (e.g., HDFS) in functions.
Although there are never rmvar instructions generated for pread
variables, the function call cleans up unexpected function outputs to
safe guard against special case memory leaks. We now globally disable
the cleanup of all persistently read matrices or frames.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/add561b3
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/add561b3
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/add561b3

Branch: refs/heads/master
Commit: add561b38e55e16dca7fd0d8a520b5abca5671a3
Parents: 1bcdfaa
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon Jun 4 19:44:32 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon Jun 4 19:44:32 2018 -0700

----------------------------------------------------------------------
 src/main/java/org/apache/sysml/lops/Data.java                  | 6 +++---
 .../sysml/runtime/instructions/cp/VariableCPInstruction.java   | 4 ++++
 2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/systemml/blob/add561b3/src/main/java/org/apache/sysml/lops/Data.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Data.java b/src/main/java/org/apache/sysml/lops/Data.java
index 53f169b..95df961 100644
--- a/src/main/java/org/apache/sysml/lops/Data.java
+++ b/src/main/java/org/apache/sysml/lops/Data.java
@@ -37,11 +37,11 @@ import org.apache.sysml.parser.Expression.ValueType;
  * variables, literals. Can be for both input and output. 
  */
 
-public class Data extends Lop  
+public class Data extends Lop
 {
-	
 	public enum OperationTypes {READ,WRITE}
-	
+	public static final String PREAD_PREFIX = "p"+OperationTypes.READ.name();
+
 	FileFormatTypes formatType = FileFormatTypes.BINARY;
 	OperationTypes operation;
 	boolean literal_var = false;

http://git-wip-us.apache.org/repos/asf/systemml/blob/add561b3/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
index b46f4df..26fcb2e 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
@@ -485,6 +485,8 @@ public class VariableCPInstruction extends CPInstruction {
 				mobj.setMetaData((MetaData)metadata.clone());
 				mobj.setFileFormatProperties(_formatProperties);
 				mobj.setUpdateType(_updateType);
+				mobj.enableCleanup(!getInput1().getName()
+					.startsWith(org.apache.sysml.lops.Data.PREAD_PREFIX));
 				ec.setVariable(getInput1().getName(), mobj);
 				if(DMLScript.STATISTICS && _updateType.isInPlace())
 					Statistics.incrementTotalUIPVar();
@@ -496,6 +498,8 @@ public class VariableCPInstruction extends CPInstruction {
 				fobj.setFileFormatProperties(_formatProperties);
 				if( _schema != null )
 					fobj.setSchema(_schema); //after metadata
+				fobj.enableCleanup(!getInput1().getName()
+					.startsWith(org.apache.sysml.lops.Data.PREAD_PREFIX));
 				ec.setVariable(getInput1().getName(), fobj);
 			}
 			else if ( getInput1().getDataType() == DataType.SCALAR ){