You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2015/12/06 23:18:03 UTC
[1/2] incubator-systemml git commit: Cleanup unnecessary seq_incr
operator and instruction (default handling)
Repository: incubator-systemml
Updated Branches:
refs/heads/master 41d9d2c1d -> 8c3f97276
Cleanup unnecessary seq_incr operator and instruction (default handling)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/94641018
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/94641018
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/94641018
Branch: refs/heads/master
Commit: 946410189658219d43a4a2588185a20b3671dd20
Parents: 41d9d2c
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Dec 6 14:17:10 2015 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sun Dec 6 14:17:10 2015 -0800
----------------------------------------------------------------------
.../java/org/apache/sysml/hops/DataGenOp.java | 14 +----------
src/main/java/org/apache/sysml/hops/Hop.java | 4 +--
.../apache/sysml/hops/recompile/Recompiler.java | 8 ++----
.../org/apache/sysml/lops/BinaryScalar.java | 4 ---
.../org/apache/sysml/parser/DMLTranslator.java | 7 ++----
.../instructions/CPInstructionParser.java | 1 -
.../instructions/cp/DataGenCPInstruction.java | 3 +++
.../instructions/cp/VariableCPInstruction.java | 26 +-------------------
.../instructions/spark/RandSPInstruction.java | 3 +++
.../apache/sysml/runtime/matrix/DataGenMR.java | 3 +++
.../runtime/matrix/data/LibMatrixDatagen.java | 12 +++++++++
.../runtime/matrix/mapred/DataGenMapper.java | 3 +++
.../matrix/operators/BinaryOperator.java | 2 +-
13 files changed, 32 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/hops/DataGenOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/DataGenOp.java b/src/main/java/org/apache/sysml/hops/DataGenOp.java
index bfe42c3..7432fc2 100644
--- a/src/main/java/org/apache/sysml/hops/DataGenOp.java
+++ b/src/main/java/org/apache/sysml/hops/DataGenOp.java
@@ -258,14 +258,6 @@ public class DataGenOp extends Hop implements MultiThreadedHop
if( toVal > 0 )
return new long[]{ toVal, 1, -1 };
}
- //here, we check for the common case of seq(1,x,?), i.e., from=1, to=x, b(seqincr) operator
- if( from instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)from)==1
- && incr instanceof BinaryOp && ((BinaryOp)incr).getOp() == Hop.OpOp2.SEQINCR ) //implicit 1
- {
- long toVal = computeDimParameterInformation(to, memo);
- if( toVal > 0 )
- return new long[]{ toVal, 1, -1 };
- }
//here, we check for the common case of seq(x,1,-1), i.e. from=x, to=1 incr=-1
if( to instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)to)==1
&& incr instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)incr)==-1 )
@@ -350,12 +342,8 @@ public class DataGenOp extends Hop implements MultiThreadedHop
double incr = computeBoundsInformation(input3);
boolean incrKnown = (incr != Double.MAX_VALUE);
-
- if( !incrKnown && input3 instanceof BinaryOp //special case for incr
- && ((BinaryOp)input3).getOp() == Hop.OpOp2.SEQINCR && fromKnown && toKnown)
- {
+ if( fromKnown && toKnown && incr == 1) {
incr = ( from >= to ) ? -1 : 1;
- incrKnown = true;
}
if ( fromKnown && toKnown && incrKnown ) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index 9822fac..ca798ce 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -1040,7 +1040,7 @@ public abstract class Hop
public enum OpOp2 {
PLUS, MINUS, MULT, DIV, MODULUS, INTDIV, LESS, LESSEQUAL, GREATER, GREATEREQUAL, EQUAL, NOTEQUAL,
MIN, MAX, AND, OR, LOG, POW, PRINT, CONCAT, QUANTILE, INTERQUANTILE, IQM,
- CENTRALMOMENT, COVARIANCE, CBIND, RBIND, SEQINCR, SOLVE, MEDIAN, INVALID,
+ CENTRALMOMENT, COVARIANCE, CBIND, RBIND, SOLVE, MEDIAN, INVALID,
//fused ML-specific operators for performance
MINUS_NZ, //sparse-safe minus: X-(mean*ppred(X,0,!=))
LOG_NZ, //sparse-safe log; ppred(X,0,"!=")*log(X,0.5)
@@ -1192,7 +1192,6 @@ public abstract class Hop
HopsOpOp2LopsBS.put(OpOp2.LOG, org.apache.sysml.lops.BinaryScalar.OperationTypes.LOG);
HopsOpOp2LopsBS.put(OpOp2.POW, org.apache.sysml.lops.BinaryScalar.OperationTypes.POW);
HopsOpOp2LopsBS.put(OpOp2.PRINT, org.apache.sysml.lops.BinaryScalar.OperationTypes.PRINT);
- HopsOpOp2LopsBS.put(OpOp2.SEQINCR, org.apache.sysml.lops.BinaryScalar.OperationTypes.SEQINCR);
}
protected static final HashMap<Hop.OpOp2, org.apache.sysml.lops.Unary.OperationTypes> HopsOpOp2LopsU;
@@ -1354,7 +1353,6 @@ public abstract class Hop
HopsOpOp2String.put(OpOp2.CBIND, "cbind");
HopsOpOp2String.put(OpOp2.RBIND, "rbind");
HopsOpOp2String.put(OpOp2.SOLVE, "solve");
- HopsOpOp2String.put(OpOp2.SEQINCR, "seqincr");
}
public static String getOpOp2String( OpOp2 op ) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
index 600c5e2..48de372 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
@@ -35,7 +35,6 @@ import org.apache.wink.json4j.JSONObject;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.hops.BinaryOp;
import org.apache.sysml.hops.DataGenOp;
import org.apache.sysml.hops.DataOp;
import org.apache.sysml.hops.FunctionOp;
@@ -1692,11 +1691,8 @@ public class Recompiler
double incr = d.computeBoundsInformation(d.getInput().get(ix3), vars);
//special case increment
- Hop input3 = d.getInput().get(ix3);
- if ( input3 instanceof BinaryOp && ((BinaryOp)input3).getOp() == Hop.OpOp2.SEQINCR
- && from!=Double.MAX_VALUE && to!=Double.MAX_VALUE )
- {
- incr =( from >= to )? -1.0 : 1.0;
+ if ( from!=Double.MAX_VALUE && to!=Double.MAX_VALUE ) {
+ incr = ( from >= to && incr==1 ) ? -1.0 : 1.0;
}
if ( from!=Double.MAX_VALUE && to!=Double.MAX_VALUE && incr!=Double.MAX_VALUE ) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/lops/BinaryScalar.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/BinaryScalar.java b/src/main/java/org/apache/sysml/lops/BinaryScalar.java
index 4a2eb29..0f423e3 100644
--- a/src/main/java/org/apache/sysml/lops/BinaryScalar.java
+++ b/src/main/java/org/apache/sysml/lops/BinaryScalar.java
@@ -41,7 +41,6 @@ public class BinaryScalar extends Lop
LOG,POW,MAX,MIN,PRINT,
IQSIZE,
Over,
- SEQINCR
}
OperationTypes operation;
@@ -190,9 +189,6 @@ public class BinaryScalar extends Lop
case IQSIZE:
return "iqsize";
-
- case SEQINCR:
- return "seqincr";
default:
throw new UnsupportedOperationException("Instruction is not defined for BinaryScalar operator: " + op);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/parser/DMLTranslator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
index e9e43b8..42ada29 100644
--- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java
+++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
@@ -2606,11 +2606,8 @@ public class DMLTranslator
HashMap<String,Hop> randParams = new HashMap<String,Hop>();
randParams.put(Statement.SEQ_FROM, expr);
randParams.put(Statement.SEQ_TO, expr2);
- if ( expr3 == null ) {
- expr3 = new BinaryOp(Statement.SEQ_INCR, DataType.SCALAR, ValueType.DOUBLE, Hop.OpOp2.SEQINCR, expr, expr2);
- //throw new ParseException("Unable to search for increment field in seq()."); //randParams.put(Statement.SEQ_INCR, new LiteralOp("defaultIncr", Double.NaN));
- }
- randParams.put(Statement.SEQ_INCR, expr3);
+ randParams.put(Statement.SEQ_INCR, (expr3!=null)?expr3 : new LiteralOp(1));
+ //note incr: default -1 (for from>to) handled during runtime
currBuiltinOp = new DataGenOp(DataGenMethod.SEQ, target, randParams);
break;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
index 3d41154..d81e363 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
@@ -192,7 +192,6 @@ public class CPInstructionParser extends InstructionParser
String2CPInstructionType.put( "read" , CPINSTRUCTION_TYPE.Variable);
String2CPInstructionType.put( "write" , CPINSTRUCTION_TYPE.Variable);
String2CPInstructionType.put( "createvar" , CPINSTRUCTION_TYPE.Variable);
- String2CPInstructionType.put( "seqincr" , CPINSTRUCTION_TYPE.Variable);
// Reorg Instruction Opcodes (repositioning of existing values)
String2CPInstructionType.put( "r'" , CPINSTRUCTION_TYPE.Reorg);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/runtime/instructions/cp/DataGenCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/DataGenCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/DataGenCPInstruction.java
index 6ae3313..9ed2714 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/DataGenCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/DataGenCPInstruction.java
@@ -302,6 +302,9 @@ public class DataGenCPInstruction extends UnaryCPInstruction
}
else if ( method == DataGenMethod.SEQ )
{
+ //handle default 1 to -1 for special case of from>to
+ seq_incr = LibMatrixDatagen.updateSeqIncr(seq_from, seq_to, seq_incr);
+
if( LOG.isTraceEnabled() )
LOG.trace("Process DataGenCPInstruction seq with seqFrom="+seq_from+", seqTo="+seq_to+", seqIncr"+seq_incr);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
index 116eee1..9fa2ad0 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
@@ -86,7 +86,6 @@ public class VariableCPInstruction extends CPInstruction
Write,
Read,
SetFileName,
- SequenceIncrement,
}
private static IDSequence _uniqueVarID;
@@ -150,9 +149,6 @@ public class VariableCPInstruction extends CPInstruction
else if ( str.equalsIgnoreCase("setfilename") )
return VariableOperationCode.SetFileName;
- else if ( str.equalsIgnoreCase("seqincr") )
- return VariableOperationCode.SequenceIncrement;
-
else
throw new DMLUnsupportedOperationException("Invalid function: " + str);
}
@@ -234,7 +230,6 @@ public class VariableCPInstruction extends CPInstruction
return 1;
case Write:
case SetFileName:
- case SequenceIncrement:
return 3;
default:
return 2;
@@ -413,12 +408,6 @@ public class VariableCPInstruction extends CPInstruction
//return new VariableCPInstruction(getVariableOperationCode(opcode), in1, in2, in3, str);
break;
- case SequenceIncrement:
- in1 = new CPOperand(parts[1]);
- in2 = new CPOperand(parts[2]);
- out = new CPOperand(parts[3]);
- break;
-
}
return new VariableCPInstruction(getVariableOperationCode(opcode), in1, in2, in3, out, _arity, opcode, str);
}
@@ -583,20 +572,7 @@ public class VariableCPInstruction extends CPInstruction
throw new DMLRuntimeException("Invalid data type (" + input1.getDataType() + ") in SetFileName instruction: " + instString);
}
break;
-
- case SequenceIncrement:
- ScalarObject fromObj = ec.getScalarInput(input1.getName(), input1.getValueType(), input1.isLiteral());
- ScalarObject toObj = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral());
- double ret = Double.NaN;
- if ( fromObj.getDoubleValue() > toObj.getDoubleValue() )
- ret = -1.0;
- else
- ret = 1.0;
- ScalarObject incrObj = (ScalarObject) new DoubleObject(ret);
- ec.setVariable(output.getName(), incrObj);
-
- break;
-
+
default:
throw new DMLRuntimeException("Unknown opcode: " + opcode );
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java
index 0e9da31..3b4f798 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java
@@ -390,6 +390,9 @@ public class RandSPInstruction extends UnarySPInstruction
throw new DMLRuntimeException("ERROR: While performing seq(" + seq_from + "," + seq_to + "," + seq_incr + ")");
}
+ //handle default 1 to -1 for special case of from>to
+ seq_incr = LibMatrixDatagen.updateSeqIncr(seq_from, seq_to, seq_incr);
+
if( LOG.isTraceEnabled() )
LOG.trace("Process RandSPInstruction seq with seqFrom="+seq_from+", seqTo="+seq_to+", seqIncr"+seq_incr);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java b/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java
index 554b22c..ba933a4 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java
@@ -195,6 +195,9 @@ public class DataGenMR
double from = seqInst.fromValue;
double to = seqInst.toValue;
double incr = seqInst.incrValue;
+
+ //handle default 1 to -1 for special case of from>to
+ incr = LibMatrixDatagen.updateSeqIncr(from, to, incr);
// Correctness checks on (from, to, incr)
boolean neg = (from > to);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java
index 4e3e382..5f2664c 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java
@@ -70,6 +70,18 @@ public class LibMatrixDatagen
}
/**
+ *
+ * @param seq_from
+ * @param seq_to
+ * @param seq_incr
+ * @return
+ */
+ public static double updateSeqIncr(double seq_from, double seq_to, double seq_incr) {
+ //handle default 1 to -1 for special case of from>to
+ return (seq_from>seq_to && seq_incr==1)? -1 : seq_incr;
+ }
+
+ /**
* A matrix of random numbers is generated by using multiple seeds, one for each
* block. Such block-level seeds are produced via Well equidistributed long-period linear
* generator (Well1024a). For a given seed, this function sets up the block-level seeds.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/runtime/matrix/mapred/DataGenMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/DataGenMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/DataGenMapper.java
index 88fcb66..49b2804 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/DataGenMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/DataGenMapper.java
@@ -98,6 +98,9 @@ implements Mapper<Writable, Writable, Writable, Writable>
double to=Double.parseDouble(params[3]);
double incr=Double.parseDouble(params[4]);
+ //handle default 1 to -1 for special case of from>to
+ incr = LibMatrixDatagen.updateSeqIncr(from, to, incr);
+
//sequence data generation
try {
indexes[i].setIndexes(blockRowNumber, blockColNumber);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/runtime/matrix/operators/BinaryOperator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/BinaryOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/BinaryOperator.java
index bb09c76..a268f49 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/operators/BinaryOperator.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/BinaryOperator.java
@@ -96,7 +96,7 @@ public class BinaryOperator extends Operator implements Serializable
//non-supported ops (not required for sparsity estimates):
//PRINT, CONCAT, QUANTILE, INTERQUANTILE, IQM,
- //CENTRALMOMENT, COVARIANCE, APPEND, SEQINCR, SOLVE, MEDIAN,
+ //CENTRALMOMENT, COVARIANCE, APPEND, SOLVE, MEDIAN,
return OpOp2.INVALID;
}
[2/2] incubator-systemml git commit: Fix memory estimates remove
empty intermediates (underestimation)
Posted by mb...@apache.org.
Fix memory estimates remove empty intermediates (underestimation)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/8c3f9727
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/8c3f9727
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/8c3f9727
Branch: refs/heads/master
Commit: 8c3f9727691e2f0441d11c69106b26c0958e33e1
Parents: 9464101
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Dec 6 14:17:35 2015 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sun Dec 6 14:17:35 2015 -0800
----------------------------------------------------------------------
src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/8c3f9727/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
index b527022..89f6959 100644
--- a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
+++ b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
@@ -727,7 +727,7 @@ public class ParameterizedBuiltinOp extends Hop
if( cols )
{
//selection vector: boolean array in the number of columns
- ret += OptimizerUtils.BIT_SIZE * dim2;
+ ret += OptimizerUtils.BOOLEAN_SIZE * dim2;
//removeEmpty-cols has additional memory requirements for intermediate
//data structures in order to make this a cache-friendly operation.
@@ -736,7 +736,7 @@ public class ParameterizedBuiltinOp extends Hop
else //rows
{
//selection vector: boolean array in the number of rows
- ret += OptimizerUtils.BIT_SIZE * dim1;
+ ret += OptimizerUtils.BOOLEAN_SIZE * dim1;
}
}
else if( _op == ParamBuiltinOp.REXPAND )