You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/02/06 04:06:47 UTC
[1/2] systemml git commit: [SYSTEMML-2132] Fix intermediate memory
estimate matrix multiply hops
Repository: systemml
Updated Branches:
refs/heads/master 2d57dc576 -> 94f1b72ef
[SYSTEMML-2132] Fix intermediate memory estimate matrix multiply hops
This patch fixes a long hidden issue of unnecessarily large intermediate
memory estimate for aggregate binary (i.e., matrix multiply) operators.
This estimate is supposed to account for a potential switch to sparse
(the worst case at the sparsity turn point) but because we used this
turn point, it was already estimated as dense. With this patch, we no
longer compile unnecessary spark instructions for auto-encoder over a
100K x 10K dense input with 35GB driver.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/206e912c
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/206e912c
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/206e912c
Branch: refs/heads/master
Commit: 206e912c5bf36e27fa07e59ada1a9f3a0abe63dd
Parents: 2d57dc5
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon Feb 5 17:19:50 2018 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon Feb 5 19:10:46 2018 -0800
----------------------------------------------------------------------
src/main/java/org/apache/sysml/hops/AggBinaryOp.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/206e912c/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
index a7b6599..6e87cfb 100644
--- a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
@@ -52,6 +52,7 @@ import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.mapred.DistributedCacheInput;
import org.apache.sysml.runtime.matrix.mapred.MMCJMRReducerWithAggregator;
+import org.apache.sysml.runtime.util.UtilFunctions;
/* Aggregate binary (cell operations): Sum (aij + bij)
@@ -385,8 +386,9 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop
//account for potential final dense-sparse transformation (worst-case sparse representation)
if( dim2 >= 2 ) //vectors always dense
- ret += OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, MatrixBlock.SPARSITY_TURN_POINT);
-
+ ret += MatrixBlock.estimateSizeSparseInMemory(dim1, dim2,
+ MatrixBlock.SPARSITY_TURN_POINT - UtilFunctions.DOUBLE_EPS);
+
return ret;
}
[2/2] systemml git commit: [MINOR] Cleanup codegen candidate
exploration (preserve all candidates)
Posted by mb...@apache.org.
[MINOR] Cleanup codegen candidate exploration (preserve all candidates)
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/94f1b72e
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/94f1b72e
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/94f1b72e
Branch: refs/heads/master
Commit: 94f1b72efa7fe768ca694d894cb6d39130ed43fc
Parents: 206e912
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon Feb 5 18:21:51 2018 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon Feb 5 19:10:48 2018 -0800
----------------------------------------------------------------------
.../org/apache/sysml/hops/codegen/template/TemplateRow.java | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/94f1b72e/src/main/java/org/apache/sysml/hops/codegen/template/TemplateRow.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateRow.java b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateRow.java
index 74d7b78..effe577 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateRow.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateRow.java
@@ -33,7 +33,6 @@ import org.apache.sysml.hops.LiteralOp;
import org.apache.sysml.hops.ParameterizedBuiltinOp;
import org.apache.sysml.hops.TernaryOp;
import org.apache.sysml.hops.UnaryOp;
-import org.apache.sysml.hops.codegen.SpoofCompiler;
import org.apache.sysml.hops.codegen.cplan.CNode;
import org.apache.sysml.hops.codegen.cplan.CNodeBinary;
import org.apache.sysml.hops.codegen.cplan.CNodeBinary.BinType;
@@ -86,8 +85,7 @@ public class TemplateRow extends TemplateBase
&& hop.getInput().get(0).getDim1()>1 && hop.getInput().get(0).getDim2()>1)
|| (hop instanceof AggBinaryOp && hop.dimsKnown() && LibMatrixMult.isSkinnyRightHandSide(
hop.getInput().get(0).getDim1(), hop.getInput().get(0).getDim2(), //MM
- hop.getInput().get(1).getDim1(), hop.getInput().get(1).getDim2(),
- SpoofCompiler.PLAN_SEL_POLICY.isCostBased())
+ hop.getInput().get(1).getDim1(), hop.getInput().get(1).getDim2(), false)
&& hop.getInput().get(0).getDim1()>1 && hop.getInput().get(0).getDim2()>1
&& !HopRewriteUtils.isOuterProductLikeMM(hop))
|| (HopRewriteUtils.isTransposeOperation(hop) && hop.getParent().size()==1
@@ -158,9 +156,8 @@ public class TemplateRow extends TemplateBase
//check for fusable but not opening matrix multiply (vect_outer-mult)
Hop in1 = hop.getInput().get(0); //transpose
Hop in2 = hop.getInput().get(1);
- boolean inclSizes = SpoofCompiler.PLAN_SEL_POLICY.isCostBased();
- return LibMatrixMult.isSkinnyRightHandSide(in1.getDim2(), in1.getDim1(), hop.getDim1(), hop.getDim2(), inclSizes)
- || LibMatrixMult.isSkinnyRightHandSide(in2.getDim1(), in2.getDim2(), hop.getDim2(), hop.getDim1(), inclSizes);
+ return LibMatrixMult.isSkinnyRightHandSide(in1.getDim2(), in1.getDim1(), hop.getDim1(), hop.getDim2(), false)
+ || LibMatrixMult.isSkinnyRightHandSide(in2.getDim1(), in2.getDim2(), hop.getDim2(), hop.getDim1(), false);
}
private static boolean isPartOfValidCumAggChain(Hop hop) {