You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/02/16 08:34:33 UTC
[1/4] systemml git commit: [MINOR] Additional debug info on matrix
compression (column classify)
Repository: systemml
Updated Branches:
refs/heads/master 6a4f1e799 -> 5837953e9
[MINOR] Additional debug info on matrix compression (column classify)
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/72830f09
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/72830f09
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/72830f09
Branch: refs/heads/master
Commit: 72830f09ae0aae30fe1c9f24e2fe167f3cf848a1
Parents: 6a4f1e7
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Feb 15 17:23:23 2018 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Feb 15 17:23:58 2018 -0800
----------------------------------------------------------------------
.../sysml/runtime/compress/CompressedMatrixBlock.java | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/72830f09/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java
index 4867749..edba26e 100644
--- a/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java
@@ -269,7 +269,7 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable
CompressedSizeInfo[] sizeInfos = (k > 1) ?
computeCompressedSizeInfos(bitmapSizeEstimator, numCols, k) :
computeCompressedSizeInfos(bitmapSizeEstimator, numCols);
- long nnzUC = 0;
+ long nnzUC = 0;
for (int col = 0; col < numCols; col++) {
double uncompSize = getUncompressedSize(numRows, 1,
OptimizerUtils.getSparsity(numRows, 1, sizeInfos[col].getEstNnz()));
@@ -300,6 +300,15 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable
}
}
+ if( LOG.isTraceEnabled() ) {
+ LOG.trace("C: "+Arrays.toString(colsC.toArray(new Integer[0])));
+ LOG.trace("-- compression ratios: "+Arrays.toString(
+ colsC.stream().map(c -> compRatios.get(c)).toArray()));
+ LOG.trace("UC: "+Arrays.toString(colsUC.toArray(new Integer[0])));
+ LOG.trace("-- compression ratios: "+Arrays.toString(
+ colsUC.stream().map(c -> compRatios.get(c)).toArray()));
+ }
+
if( LOG.isDebugEnabled() ) {
_stats.timePhase1 = time.stop();
LOG.debug("Compression statistics:");
[3/4] systemml git commit: [SYSTEMML-2150] Fix codegen optimizer cost
model (incorrect input size)
Posted by mb...@apache.org.
[SYSTEMML-2150] Fix codegen optimizer cost model (incorrect input size)
This patch fixes the codegen cost-based optimizer cost model,
specifically to compute the input sizes from the output memory estimates
of inputs not their total memory estimate. SYSTEMML-1979 introduced this
issue when incorporating the handling of sparsity.
For L2SVM over Mnist80m (180GB) w/ 20 outer iterations, this patch
improved the end-to-end runtime form 750s to 593s.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/de8f2c77
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/de8f2c77
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/de8f2c77
Branch: refs/heads/master
Commit: de8f2c77c644efb4253739f62ca3f0d311ccc8ec
Parents: 62e590c
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Feb 15 22:24:01 2018 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Feb 15 22:51:48 2018 -0800
----------------------------------------------------------------------
.../sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/de8f2c77/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java b/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java
index 9b184b7..20cb955 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java
@@ -414,7 +414,7 @@ public class PlanSelectionFuseCostBasedV2 extends PlanSelection
private static double getSafeMemEst(Hop hop) {
return !hop.dimsKnown() ? getSize(hop) * 8
- : hop.getMemEstimate();
+ : hop.getOutputMemEstimate();
}
private static long getSize(Hop hop) {
[4/4] systemml git commit: [SYSTEMML-2151] Fix correctness of codegen
row powAdd for sparse
Posted by mb...@apache.org.
[SYSTEMML-2151] Fix correctness of codegen row powAdd for sparse
This patch fixes the result correctness of the codegen powAdd vector
(primitive as used in codegen row operations) for sparse inputs.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/5837953e
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/5837953e
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/5837953e
Branch: refs/heads/master
Commit: 5837953e94cc427aaf8207d9c94cea36a50499cf
Parents: de8f2c7
Author: Matthias Boehm <mb...@gmail.com>
Authored: Fri Feb 16 00:09:03 2018 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Feb 16 00:09:03 2018 -0800
----------------------------------------------------------------------
.../apache/sysml/runtime/codegen/LibSpoofPrimitives.java | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/5837953e/src/main/java/org/apache/sysml/runtime/codegen/LibSpoofPrimitives.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysml/runtime/codegen/LibSpoofPrimitives.java
index fe34507..bcb2f4d 100644
--- a/src/main/java/org/apache/sysml/runtime/codegen/LibSpoofPrimitives.java
+++ b/src/main/java/org/apache/sysml/runtime/codegen/LibSpoofPrimitives.java
@@ -655,15 +655,16 @@ public class LibSpoofPrimitives
}
public static void vectPowAdd(double[] a, double bval, double[] c, int[] aix, int ai, int ci, int alen, int len) {
- if( bval == 0 ) //handle 0^0=1
+ if( bval == 0 ) //handle 0^0=1 & a^0=1
for( int j=0; j<len; j++ )
c[ci + j] += 1;
- for( int j = ai; j < ai+alen; j++ )
- c[ci + aix[j]] += Math.pow(a[j], bval) - 1;
+ else //handle 0^b=0 & a^b
+ for( int j = ai; j < ai+alen; j++ )
+ c[ci + aix[j]] += Math.pow(a[j], bval);
}
public static void vectPowAdd(double bval, double[] a, double[] c, int[] aix, int ai, int ci, int alen, int len) {
- for( int j=0; j<len; j++ )
+ for( int j=0; j<len; j++ ) //handle 0^0=1 & b^0=1
c[ci + j] += 1;
for( int j = ai; j < ai+alen; j++ )
c[ci + aix[j]] += Math.pow(bval, a[j]) - 1;
[2/4] systemml git commit: [SYSTEMML-2149] New simplification rewrite
for replace zero w/ scalar
Posted by mb...@apache.org.
[SYSTEMML-2149] New simplification rewrite for replace zero w/ scalar
There are multiple scripts that emulate the replacement of zeros with a
scalar via X + (X==0) * s. We now rewrite this pattern to the builtin
function replace(X, 0, s), which avoids an unnecessary intermediate and
(partitioning-preserving) joins for distributed operations.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/62e590ce
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/62e590ce
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/62e590ce
Branch: refs/heads/master
Commit: 62e590ced04900364bdc294538e78de6af3f4988
Parents: 72830f0
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Feb 15 18:49:00 2018 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Feb 15 18:49:00 2018 -0800
----------------------------------------------------------------------
.../RewriteAlgebraicSimplificationStatic.java | 32 +++++++++++++++++---
1 file changed, 28 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/62e590ce/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
index ac45e77..3a3235d 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
@@ -175,6 +175,7 @@ public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
hi = fuseOrderOperationChain(hi); //e.g., order(order(X,2),1) -> order(X,(12))
hi = removeUnnecessaryReorgOperation(hop, hi, i); //e.g., t(t(X))->X; rev(rev(X))->X potentially introduced by other rewrites
hi = simplifyTransposeAggBinBinaryChains(hop, hi, i);//e.g., t(t(A)%*%t(B)+C) -> B%*%A+t(C)
+ hi = simplifyReplaceZeroOperation(hop, hi, i); //e.g., X + (X==0) * s -> replace(X, 0, s)
hi = removeUnnecessaryMinus(hop, hi, i); //e.g., -(-X)->X; potentially introduced by simplify binary or dyn rewrites
hi = simplifyGroupedAggregate(hi); //e.g., aggregate(target=X,groups=y,fn="count") -> aggregate(target=y,groups=y,fn="count")
if(OptimizerUtils.ALLOW_OPERATOR_FUSION) {
@@ -1556,14 +1557,14 @@ public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
{
if( HopRewriteUtils.isTransposeOperation(hi)
&& hi.getInput().get(0) instanceof BinaryOp //basic binary
- && ((BinaryOp)hi.getInput().get(0)).supportsMatrixScalarOperations())
+ && ((BinaryOp)hi.getInput().get(0)).supportsMatrixScalarOperations())
{
Hop left = hi.getInput().get(0).getInput().get(0);
Hop C = hi.getInput().get(0).getInput().get(1);
//check matrix mult and both inputs transposes w/ single consumer
if( left instanceof AggBinaryOp && C.getDataType().isMatrix()
- && HopRewriteUtils.isTransposeOperation(left.getInput().get(0))
+ && HopRewriteUtils.isTransposeOperation(left.getInput().get(0))
&& left.getInput().get(0).getParent().size()==1
&& HopRewriteUtils.isTransposeOperation(left.getInput().get(1))
&& left.getInput().get(1).getParent().size()==1 )
@@ -1578,13 +1579,36 @@ public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
HopRewriteUtils.replaceChildReference(parent, hi, bop, pos);
hi = bop;
- LOG.debug("Applied simplifyTransposeAggBinBinaryChains (line "+hi.getBeginLine()+").");
- }
+ LOG.debug("Applied simplifyTransposeAggBinBinaryChains (line "+hi.getBeginLine()+").");
+ }
}
return hi;
}
+ // Patterns: X + (X==0) * s -> replace(X, 0, s)
+ private static Hop simplifyReplaceZeroOperation(Hop parent, Hop hi, int pos)
+ throws HopsException
+ {
+ if( HopRewriteUtils.isBinary(hi, OpOp2.PLUS) && hi.getInput().get(0).isMatrix()
+ && HopRewriteUtils.isBinary(hi.getInput().get(1), OpOp2.MULT)
+ && hi.getInput().get(1).getInput().get(1).isScalar()
+ && HopRewriteUtils.isBinaryMatrixScalar(hi.getInput().get(1).getInput().get(0), OpOp2.EQUAL, 0)
+ && hi.getInput().get(1).getInput().get(0).getInput().contains(hi.getInput().get(0)) )
+ {
+ HashMap<String, Hop> args = new HashMap<>();
+ args.put("target", hi.getInput().get(0));
+ args.put("pattern", new LiteralOp(0));
+ args.put("replacement", hi.getInput().get(1).getInput().get(1));
+ Hop replace = HopRewriteUtils.createParameterizedBuiltinOp(
+ hi.getInput().get(0), args, ParamBuiltinOp.REPLACE);
+ HopRewriteUtils.replaceChildReference(parent, hi, replace, pos);
+ hi = replace;
+ LOG.debug("Applied simplifyReplaceZeroOperation (line "+hi.getBeginLine()+").");
+ }
+ return hi;
+ }
+
/**
* Pattners: t(t(X)) -> X, rev(rev(X)) -> X
*