You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/05/03 15:46:04 UTC
incubator-systemml git commit: [SYSTEMML-1574] Fix serialized caching
of ultra-sparse matrices
Repository: incubator-systemml
Updated Branches:
refs/heads/master aa6d38c94 -> 4e445d17e
[SYSTEMML-1574] Fix serialized caching of ultra-sparse matrices
Previously, we stored all sparse matrices that are expected to exceed
aggregate memory in serialized form in order to eliminate unnecessary
in-meomry storage overheads. With the introduction of csr we disabled
this code path as csr's in-memory representation is as large as its
serialized form. However, this is not true for ultra-sparse matrices
which still benefit from serialized storage. Accordingly this patch
tweaks the corresponding selecting of checkpoint storage levels.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/4e445d17
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/4e445d17
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/4e445d17
Branch: refs/heads/master
Commit: 4e445d17e7f06e0c0b04bc2610806fbf17dc68ca
Parents: aa6d38c
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed May 3 11:45:38 2017 -0400
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Wed May 3 11:45:38 2017 -0400
----------------------------------------------------------------------
src/main/java/org/apache/sysml/hops/Hop.java | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4e445d17/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index eb9aa5e..6c7089b 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -313,7 +313,6 @@ public abstract class Hop
}
}
- @SuppressWarnings("unused") //see CHECKPOINT_SPARSE_CSR
private void constructAndSetCheckpointLopIfRequired()
throws HopsException
{
@@ -345,11 +344,13 @@ public abstract class Hop
//investigate need for serialized storage of large sparse matrices
//(compile- instead of runtime-level for better debugging)
boolean serializedStorage = false;
- if( getDataType()==DataType.MATRIX && dimsKnown(true) && !Checkpoint.CHECKPOINT_SPARSE_CSR ) {
+ if( getDataType()==DataType.MATRIX && dimsKnown(true) ) {
double matrixPSize = OptimizerUtils.estimatePartitionedSizeExactSparsity(_dim1, _dim2, _rows_in_block, _cols_in_block, _nnz);
double dataCache = SparkExecutionContext.getDataMemoryBudget(true, true);
- serializedStorage = (MatrixBlock.evalSparseFormatInMemory(_dim1, _dim2, _nnz)
- && matrixPSize > dataCache ); //sparse in-memory does not fit in agg mem
+ serializedStorage = MatrixBlock.evalSparseFormatInMemory(_dim1, _dim2, _nnz)
+ && matrixPSize > dataCache //sparse in-memory does not fit in agg mem
+ && (OptimizerUtils.getSparsity(_dim1, _dim2, _nnz) < MatrixBlock.ULTRA_SPARSITY_TURN_POINT
+ || !Checkpoint.CHECKPOINT_SPARSE_CSR ); //ultra-sparse or sparse w/o csr
}
else if( !dimsKnown(true) ) {
setRequiresRecompile();