You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2015/12/04 19:04:22 UTC
[1/5] incubator-systemml git commit: New simplification rewrite 'fuse
log_nz unary', incl compiler support
Repository: incubator-systemml
Updated Branches:
refs/heads/master 3cfb0fb0a -> 5f058f08d
New simplification rewrite 'fuse log_nz unary', incl compiler support
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/3f08e78f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/3f08e78f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/3f08e78f
Branch: refs/heads/master
Commit: 3f08e78f602fa17286e0e63029c5cc5d4163cdce
Parents: 3cfb0fb
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Thu Dec 3 19:04:01 2015 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Thu Dec 3 19:04:01 2015 -0800
----------------------------------------------------------------------
src/main/java/org/apache/sysml/hops/Hop.java | 1 +
.../RewriteAlgebraicSimplificationStatic.java | 46 +++++++++++++++++++-
2 files changed, 46 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3f08e78f/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index 19a5df5..08ca7a2 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -1031,6 +1031,7 @@ public abstract class Hop
SPROP, //sample proportion: P * (1 - P)
SIGMOID, //sigmoid function: 1 / (1 + exp(-X))
SELP, //select positive: X * (X>0)
+ LOG_NZ, //sparse-safe log; ppred(X,0,"!=")*log(X)
}
// Operations that require two operands
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3f08e78f/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
index bdd8d8b..68cbeb4 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
@@ -146,6 +146,7 @@ public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
hi = removeUnnecessaryMinus(hop, hi, i); //e.g., -(-X)->X; potentially introduced by simplfiy binary or dyn rewrites
hi = simplifyGroupedAggregate(hi); //e.g., aggregate(target=X,groups=y,fn="count") -> aggregate(target=y,groups=y,fn="count")
hi = fuseMinusNzBinaryOperation(hop, hi, i); //e.g., X-mean*ppred(X,0,!=) -> X -nz mean
+ hi = fuseLogNzUnaryOperation(hop, hi, i); //e.g., ppred(X,0,"!=")*log(X) -> log_nz(X)
hi = fuseLogNzBinaryOperation(hop, hi, i); //e.g., ppred(X,0,"!=")*log(X,0.5) -> log_nz(X,0.5)
hi = simplifyOuterSeqExpand(hop, hi, i); //e.g., outer(v, seq(1,m), "==") -> rexpand(v, max=m, dir=row, ignore=true, cast=false)
hi = simplifyTableSeqExpand(hop, hi, i); //e.g., table(seq(1,nrow(v)), v, nrow(v), m) -> rexpand(v, max=m, dir=row, ignore=false, cast=true)
@@ -1341,7 +1342,50 @@ public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
return hi;
}
-
+ /**
+ *
+ * @param parent
+ * @param hi
+ * @param pos
+ * @return
+ * @throws HopsException
+ */
+ private Hop fuseLogNzUnaryOperation(Hop parent, Hop hi, int pos)
+ throws HopsException
+ {
+ //pattern ppred(X,0,"!=")*log(X) -> log_nz(X)
+ //note: this is done as a hop rewrite in order to significantly reduce the
+ //memory estimate and to prevent dense intermediates if X is ultra sparse
+ if( hi instanceof BinaryOp && ((BinaryOp)hi).getOp()==OpOp2.MULT
+ && hi.getInput().get(0).getDataType()==DataType.MATRIX
+ && hi.getInput().get(1).getDataType()==DataType.MATRIX
+ && hi.getInput().get(1) instanceof UnaryOp
+ && ((UnaryOp)hi.getInput().get(1)).getOp()==OpOp1.LOG )
+ {
+ Hop pred = hi.getInput().get(0);
+ Hop X = hi.getInput().get(1).getInput().get(0);
+
+ if( pred instanceof BinaryOp && ((BinaryOp)pred).getOp()==OpOp2.NOTEQUAL
+ && pred.getInput().get(0) == X //depend on common subexpression elimination
+ && pred.getInput().get(1) instanceof LiteralOp
+ && HopRewriteUtils.getDoubleValueSafe((LiteralOp)pred.getInput().get(1))==0 )
+ {
+ Hop hnew = new UnaryOp("tmp", DataType.MATRIX, ValueType.DOUBLE, OpOp1.LOG_NZ, X);
+ HopRewriteUtils.setOutputBlocksizes(hnew, hi.getRowsInBlock(), hi.getColsInBlock());
+ hnew.refreshSizeInformation();
+
+ //relink new hop into original position
+ HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
+ HopRewriteUtils.addChildReference(parent, hnew, pos);
+ hi = hnew;
+
+ LOG.debug("Applied fuseLogNzUnaryOperation (line "+hi.getBeginLine()+").");
+ }
+ }
+
+ return hi;
+ }
+
/**
*
* @param parent
[4/5] incubator-systemml git commit: Fix spark instruction parsing
log/log_nz (updated sanity checks)
Posted by mb...@apache.org.
Fix spark instruction parsing log/log_nz (updated sanity checks)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/59d4a150
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/59d4a150
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/59d4a150
Branch: refs/heads/master
Commit: 59d4a15000e9d531dfe35294556f3f1ef08be393
Parents: 0215a5a
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Thu Dec 3 19:18:41 2015 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Thu Dec 3 19:18:41 2015 -0800
----------------------------------------------------------------------
.../apache/sysml/runtime/instructions/SPInstructionParser.java | 2 +-
.../instructions/spark/MatrixScalarBuiltinSPInstruction.java | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/59d4a150/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
index bd4d9d0..3fe1f09 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
@@ -326,7 +326,7 @@ public class SPInstructionParser extends InstructionParser {
case Builtin:
parts = InstructionUtils.getInstructionPartsWithValueType(str);
- if ( parts[0].equals("log") ) {
+ if ( parts[0].equals("log") || parts[0].equals("log_nz") ) {
if ( parts.length == 3 ) {
// B=log(A), y=log(x)
return (SPInstruction) BuiltinUnarySPInstruction.parseInstruction(str);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/59d4a150/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarBuiltinSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarBuiltinSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarBuiltinSPInstruction.java
index 2e47801..72f5f5d 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarBuiltinSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixScalarBuiltinSPInstruction.java
@@ -37,7 +37,8 @@ public class MatrixScalarBuiltinSPInstruction extends BuiltinBinarySPInstruction
{
//sanity check opcode
String opcode = getOpcode();
- if ( !(opcode.equalsIgnoreCase("max") || opcode.equalsIgnoreCase("min")) )
+ if ( !(opcode.equalsIgnoreCase("max") || opcode.equalsIgnoreCase("min"))
+ ||opcode.equalsIgnoreCase("log") || opcode.equalsIgnoreCase("log_nz") )
{
throw new DMLRuntimeException("Unknown opcode in instruction: " + opcode);
}
[5/5] incubator-systemml git commit: Improved robustness of broadcast
reuse (cleanup on demand via softref)
Posted by mb...@apache.org.
Improved robustness of broadcast reuse (cleanup on demand via softref)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/5f058f08
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/5f058f08
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/5f058f08
Branch: refs/heads/master
Commit: 5f058f08d1db0b14cd431d85eaa17746b5cf99bb
Parents: 59d4a15
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Thu Dec 3 19:29:17 2015 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Thu Dec 3 19:29:17 2015 -0800
----------------------------------------------------------------------
.../context/SparkExecutionContext.java | 13 ++++++++-----
.../instructions/spark/data/BroadcastObject.java | 17 +++++++++++++----
.../spark/data/PartitionedBroadcastMatrix.java | 3 +--
3 files changed, 22 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5f058f08/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
index e4a5a5f..a7bb99f 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
@@ -354,13 +354,15 @@ public class SparkExecutionContext extends ExecutionContext
PartitionedBroadcastMatrix bret = null;
- if( mo.getBroadcastHandle()!=null
+ //reuse existing broadcast handle
+ if( mo.getBroadcastHandle()!=null
&& mo.getBroadcastHandle().isValid() )
{
- //reuse existing broadcast handle
bret = mo.getBroadcastHandle().getBroadcast();
}
- else
+
+ //create new broadcast handle (never created, evicted)
+ if( bret == null )
{
//obtain meta data for matrix
int brlen = (int) mo.getNumRowsPerBlock();
@@ -892,8 +894,9 @@ public class SparkExecutionContext extends ExecutionContext
}
else if( lob instanceof BroadcastObject ) {
PartitionedBroadcastMatrix pbm = ((BroadcastObject)lob).getBroadcast();
- for( Broadcast<PartitionedMatrixBlock> bc : pbm.getBroadcasts() )
- cleanupBroadcastVariable(bc);
+ if( pbm != null ) //robustness for evictions
+ for( Broadcast<PartitionedMatrixBlock> bc : pbm.getBroadcasts() )
+ cleanupBroadcastVariable(bc);
}
//recursively process lineage children
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5f058f08/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java
index 2d306a7..da19319 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java
@@ -17,15 +17,18 @@
package org.apache.sysml.runtime.instructions.spark.data;
+import java.lang.ref.SoftReference;
+
import org.apache.spark.broadcast.Broadcast;
public class BroadcastObject extends LineageObject
{
- private PartitionedBroadcastMatrix _bcHandle = null;
+ //soft reference storage for graceful cleanup in case of memory pressure
+ private SoftReference<PartitionedBroadcastMatrix> _bcHandle = null;
public BroadcastObject( PartitionedBroadcastMatrix bvar, String varName )
{
- _bcHandle = bvar;
+ _bcHandle = new SoftReference<PartitionedBroadcastMatrix>(bvar);
_varName = varName;
}
@@ -35,7 +38,7 @@ public class BroadcastObject extends LineageObject
*/
public PartitionedBroadcastMatrix getBroadcast()
{
- return _bcHandle;
+ return _bcHandle.get();
}
/**
@@ -44,7 +47,13 @@ public class BroadcastObject extends LineageObject
*/
public boolean isValid()
{
- Broadcast<PartitionedMatrixBlock>[] tmp = _bcHandle.getBroadcasts();
+ //check for evicted soft reference
+ PartitionedBroadcastMatrix pbm = _bcHandle.get();
+ if( pbm == null )
+ return false;
+
+ //check for validity of individual broadcasts
+ Broadcast<PartitionedMatrixBlock>[] tmp = pbm.getBroadcasts();
for( Broadcast<PartitionedMatrixBlock> bc : tmp )
if( !bc.isValid() )
return false;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5f058f08/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcastMatrix.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcastMatrix.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcastMatrix.java
index 205ab80..1fbf605 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcastMatrix.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcastMatrix.java
@@ -36,8 +36,7 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock;
public class PartitionedBroadcastMatrix implements Serializable
{
private static final long serialVersionUID = 1225135967889810877L;
-
- private static long BROADCAST_PARTSIZE = 200L*1024*1024; //200M cells ~ 1.6GB
+ private static final long BROADCAST_PARTSIZE = 200L*1024*1024; //200M cells ~ 1.6GB
private Broadcast<PartitionedMatrixBlock>[] _pbc = null;
[3/5] incubator-systemml git commit: Improved binary max
compilation/runtime (rewrite, cond. sparse-safe)
Posted by mb...@apache.org.
Improved binary max compilation/runtime (rewrite, cond. sparse-safe)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/0215a5a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/0215a5a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/0215a5a0
Branch: refs/heads/master
Commit: 0215a5a064a2ef1c8a2c736ccb0c2f508de289e0
Parents: bb1680f
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Thu Dec 3 19:07:20 2015 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Thu Dec 3 19:07:20 2015 -0800
----------------------------------------------------------------------
.../RewriteAlgebraicSimplificationStatic.java | 16 ++++++++++++++++
.../runtime/matrix/operators/ScalarOperator.java | 5 ++++-
2 files changed, 20 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0215a5a0/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
index 68cbeb4..558c1ae 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
@@ -983,6 +983,22 @@ public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
}
}
}
+
+ //select positive (selp) operator; pattern: max(X,0) -> selp+
+ if( bop.getOp() == OpOp2.MAX && left.getDataType()==DataType.MATRIX
+ && right instanceof LiteralOp && HopRewriteUtils.getDoubleValue((LiteralOp)right)==0 )
+ {
+ UnaryOp unary = HopRewriteUtils.createUnary(left, OpOp1.SELP);
+ HopRewriteUtils.removeChildReferenceByPos(parent, bop, pos);
+ HopRewriteUtils.addChildReference(parent, unary, pos);
+
+ //cleanup if only consumer of intermediate
+ if( bop.getParent().isEmpty() )
+ HopRewriteUtils.removeAllChildReferences(bop);
+ hi = unary;
+
+ LOG.debug("Applied fuseBinarySubDAGToUnaryOperation-selp3");
+ }
}
return hi;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0215a5a0/src/main/java/org/apache/sysml/runtime/matrix/operators/ScalarOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/ScalarOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/ScalarOperator.java
index 5b86f53..1949594 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/ScalarOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/ScalarOperator.java
@@ -83,7 +83,10 @@ public class ScalarOperator extends Operator
|| (fn instanceof LessThan && _constant==0)
|| (fn instanceof NotEquals && _constant==0)
|| (fn instanceof Equals && _constant!=0)
- || (fn instanceof Minus && _constant==0))
+ || (fn instanceof Minus && _constant==0)
+ || (fn instanceof Minus && _constant==0)
+ || (fn instanceof Builtin && ((Builtin)fn).getBuiltinFunctionCode()==BuiltinFunctionCode.MAX && _constant<=0)
+ || (fn instanceof Builtin && ((Builtin)fn).getBuiltinFunctionCode()==BuiltinFunctionCode.MIN && _constant>=0))
{
sparseSafe = true;
}
[2/5] incubator-systemml git commit: Fix quaternary hop (missing
operation types for explain)
Posted by mb...@apache.org.
Fix quaternary hop (missing operation types for explain)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/bb1680fa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/bb1680fa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/bb1680fa
Branch: refs/heads/master
Commit: bb1680fa2d7a564181f9dc424d2c68bdb8236325
Parents: 3f08e78
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Thu Dec 3 19:05:13 2015 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Thu Dec 3 19:05:13 2015 -0800
----------------------------------------------------------------------
src/main/java/org/apache/sysml/hops/Hop.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/bb1680fa/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index 08ca7a2..a05b11a 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -1372,8 +1372,11 @@ public abstract class Hop
protected static final HashMap<Hop.OpOp4, String> HopsOpOp4String;
static {
HopsOpOp4String = new HashMap<Hop.OpOp4, String>();
- HopsOpOp4String.put(OpOp4.WSLOSS, "wsloss");
+ HopsOpOp4String.put(OpOp4.WSLOSS, "wsloss");
HopsOpOp4String.put(OpOp4.WSIGMOID, "wsigmoid");
+ HopsOpOp4String.put(OpOp4.WCEMM, "wcemm");
+ HopsOpOp4String.put(OpOp4.WDIVMM, "wdivmm");
+ HopsOpOp4String.put(OpOp4.WUMM, "wumm");
}
protected static final HashMap<Hop.Direction, String> HopsDirection2String;