You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/12/07 21:41:04 UTC
[3/6] systemml git commit: [MINOR] Improved primitives for memory
profiling and sparse checks
[MINOR] Improved primitives for memory profiling and sparse checks
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/f13f697e
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/f13f697e
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/f13f697e
Branch: refs/heads/master
Commit: f13f697e08a2a87cff61ca5dfd62f73f10555d3e
Parents: f84fd71
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed Dec 6 19:36:54 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Dec 7 13:36:08 2017 -0800
----------------------------------------------------------------------
.../sysml/runtime/controlprogram/LocalVariableMap.java | 9 +++++++++
.../apache/sysml/runtime/controlprogram/ProgramBlock.java | 2 +-
.../sysml/runtime/controlprogram/caching/CacheableData.java | 6 +++++-
.../org/apache/sysml/runtime/matrix/data/MatrixBlock.java | 4 ++++
4 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/f13f697e/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java
index e894495..c91eec4 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java
@@ -27,6 +27,7 @@ import java.util.Set;
import java.util.StringTokenizer;
import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
import org.apache.sysml.runtime.instructions.cp.Data;
@@ -122,6 +123,14 @@ public class LocalVariableMap implements Cloneable
return outputs;
}
+ public double getPinnedDataSize() {
+ //note: this method returns the total size of pinned data objects
+ //that are not subject to automatic eviction.
+ return localMap.values().stream()
+ .filter(d -> (d instanceof CacheableData))
+ .mapToDouble(d -> ((CacheableData<?>)d).getDataSize()).sum();
+ }
+
public String serialize() throws DMLRuntimeException {
StringBuilder sb = new StringBuilder();
int count = 0;
http://git-wip-us.apache.org/repos/asf/systemml/blob/f13f697e/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
index fd6909d..0996cd9 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
@@ -371,7 +371,7 @@ public class ProgramBlock implements ParseInfo
if( nnz1 != nnz2 )
throw new DMLRuntimeException("Matrix nnz meta data was incorrect: ("+varname+", actual="+nnz1+", expected="+nnz2+", inst="+lastInst+")");
- if( sparse1 != sparse2 )
+ if( sparse1 != sparse2 && mb.isAllocated() )
throw new DMLRuntimeException("Matrix was in wrong data representation: ("+varname+", actual="+sparse1+", expected="+sparse2 +
", nrow="+mb.getNumRows()+", ncol="+mb.getNumColumns()+", nnz="+nnz1+", inst="+lastInst+")");
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/f13f697e/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
index be7b9b9..e2b9838 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java
@@ -956,6 +956,10 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
return (_data.getInMemorySize() <= CACHING_THRESHOLD);
}
+ public long getDataSize() {
+ return (_data != null) ?_data.getInMemorySize() : 0;
+ }
+
protected ValueType[] getSchema() {
return null;
}
@@ -1262,7 +1266,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
sizePinned.set( Math.max(size,0) );
}
- protected long getPinnedSize() {
+ protected static long getPinnedSize() {
return sizePinned.get();
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/f13f697e/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index c893bc6..ac0655c 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -1312,9 +1312,13 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
int apos = sparseBlock.pos(i);
int alen = sparseBlock.size(i);
int[] aix = sparseBlock.indexes(i);
+ double[] avals = sparseBlock.values(i);
for( int k=apos+1; k<apos+alen; k++ )
if( aix[k-1] >= aix[k] )
throw new RuntimeException("Wrong sparse row ordering: "+k+" "+aix[k-1]+" "+aix[k]);
+ for( int k=apos; k<apos+alen; k++ )
+ if( avals[k] == 0 )
+ throw new RuntimeException("Wrong sparse row: zero at "+k);
}
}