You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/04/21 05:11:54 UTC
[2/2] incubator-systemml git commit: [SYSTEMML-1550] Improved cp
dim/sparsity check (exploit worst-case est)
[SYSTEMML-1550] Improved cp dim/sparsity check (exploit worst-case est)
This patch improves the operator selection of all hops by taking the
worst-case estimates in account during checks for valid cp dimensions
and sparsity. With this improvement, we are able to avoid unnecessary
distributed operations for hops with large dimensions and unknown exact
but known worst-case sparsity estimates such as rexpand.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/2a1eb4c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/2a1eb4c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/2a1eb4c9
Branch: refs/heads/master
Commit: 2a1eb4c9b290c3af62f3ed2b0744cc507d1994d6
Parents: 46bd37c
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Apr 20 21:47:06 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Apr 20 22:13:49 2017 -0700
----------------------------------------------------------------------
src/main/java/org/apache/sysml/hops/Hop.java | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/2a1eb4c9/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index 15a0cc3..eb9aa5e 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -50,7 +50,6 @@ import org.apache.sysml.runtime.util.UtilFunctions;
public abstract class Hop
{
-
protected static final Log LOG = LogFactory.getLog(Hop.class.getName());
public static final long CPThreshold = 2000;
@@ -92,6 +91,7 @@ public abstract class Hop
protected double _memEstimate = OptimizerUtils.INVALID_SIZE;
protected double _processingMemEstimate = 0;
protected double _spBroadcastMemEstimate = 0;
+ protected boolean _validCPSizeEstimate = false;
// indicates if there are unknowns during compilation
// (in that case re-complication ensures robustness and efficiency)
@@ -198,10 +198,10 @@ public abstract class Hop
//Step 2: check valid output and input sizes for cp (<16GB for DENSE)
//(if the memory estimate is smaller than max_numcells we are guaranteed to have it in sparse representation)
invalid |= !( OptimizerUtils.isValidCPMatrixSize(_dim1, _dim2, OptimizerUtils.getSparsity(_dim1, _dim2, _nnz))
- || getOutputMemEstimate() < OptimizerUtils.MAX_NUMCELLS_CP_DENSE );
+ || getOutputMemEstimate() < 8*OptimizerUtils.MAX_NUMCELLS_CP_DENSE || _validCPSizeEstimate );
for( Hop in : getInput() )
invalid |= !( OptimizerUtils.isValidCPMatrixSize(in._dim1, in._dim2, OptimizerUtils.getSparsity(in._dim1, in._dim2, in._nnz))
- || in.getOutputMemEstimate() < OptimizerUtils.MAX_NUMCELLS_CP_DENSE);
+ || in.getOutputMemEstimate() < 8*OptimizerUtils.MAX_NUMCELLS_CP_DENSE || in._validCPSizeEstimate);
//force exec type mr if necessary
if( invalid ) {
@@ -612,7 +612,7 @@ public abstract class Hop
//nnz always exactly known (see dimsKnown(true))
_outputMemEstimate = computeOutputMemEstimate( _dim1, _dim2, _nnz );
}
- //1b) infer output statistics and mem estimate based on these statistics
+ //1b) infer output statistics and mem estimate based on worst-case statistics
else if( memo.hasInputStatistics(this) )
{
//infer the output stats
@@ -682,9 +682,13 @@ public abstract class Hop
////////
//Step 3) Compute final hop memory estimate
-
+
//final estimate (sum of inputs/intermediates/output)
_memEstimate = getInputOutputSize();
+
+ //update optional valid cp size estimate (based on worst-case dimensions)
+ _validCPSizeEstimate = (wstats!=null) ? OptimizerUtils.isValidCPMatrixSize(
+ wstats[0], wstats[1], OptimizerUtils.getSparsity(wstats[0], wstats[1], wstats[2])) : false;
}