You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2015/12/11 10:25:59 UTC
incubator-systemml git commit: Generalization mmchain matrix mult
(compiler/runtime/tests), for lasso
Repository: incubator-systemml
Updated Branches:
refs/heads/master faf67cd17 -> 6886e07df
Generalization mmchain matrix mult (compiler/runtime/tests), for lasso
This change introduces a new mmchain type 'XtXvy' for the pattern
t(X)%*%(X%*%v-y). It also includes a fix for the rewrite into type
'XtwXv' as well as some cleanups. Experiments w/ 100 repetitions on an
8GB singlenode use case did not show any slowdown for existing mmchain
types.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/6886e07d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/6886e07d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/6886e07d
Branch: refs/heads/master
Commit: 6886e07dfa906d38b49c26ae9486dbf0135e4380
Parents: faf67cd
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Thu Dec 10 13:53:41 2015 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Thu Dec 10 13:53:41 2015 -0800
----------------------------------------------------------------------
.../java/org/apache/sysml/hops/AggBinaryOp.java | 50 +++++++----
.../org/apache/sysml/lops/MapMultChain.java | 7 +-
.../instructions/cp/MMChainCPInstruction.java | 3 +-
.../mr/MapMultChainInstruction.java | 8 +-
.../spark/MapmmChainSPInstruction.java | 10 ++-
.../spark/utils/RDDConverterUtilsExt.java | 3 -
.../runtime/matrix/data/LibMatrixMult.java | 49 +++++++++--
.../sysml/runtime/matrix/data/MatrixBlock.java | 2 +-
.../binary/matrix/MapMultChainTest.java | 90 +++++++++++++++++---
.../binary/matrix/MapMultChainWeights2.R | 35 ++++++++
.../binary/matrix/MapMultChainWeights2.dml | 28 ++++++
11 files changed, 238 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6886e07d/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
index 415f62a..f8e1541 100644
--- a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
@@ -506,7 +506,7 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop
//check mapmultchain patterns
//t(X)%*%(w*(X%*%v))
- if( in2 instanceof BinaryOp )
+ if( in2 instanceof BinaryOp && ((BinaryOp)in2).getOp()==OpOp2.MULT )
{
Hop in3b = in2.getInput().get(1);
if( in3b instanceof AggBinaryOp )
@@ -516,6 +516,18 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop
chainType = ChainType.XtwXv;
}
}
+ //t(X)%*%((X%*%v)-y)
+ else if( in2 instanceof BinaryOp && ((BinaryOp)in2).getOp()==OpOp2.MINUS )
+ {
+ Hop in3a = in2.getInput().get(0);
+ Hop in3b = in2.getInput().get(1);
+ if( in3a instanceof AggBinaryOp && in3b.getDataType()==DataType.MATRIX )
+ {
+ Hop in4 = in3a.getInput().get(0);
+ if( X == in4 ) //common input
+ chainType = ChainType.XtXvy;
+ }
+ }
//t(X)%*%(X%*%v)
else if( in2 instanceof AggBinaryOp )
{
@@ -565,11 +577,13 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop
Hop hv = getInput().get(1).getInput().get(1);
mapmmchain = new MapMultChain( hX.constructLops(), hv.constructLops(), getDataType(), getValueType(), ExecType.CP);
}
- else { //if( chainType == ChainType.XtwXv )
+ else { //ChainType.XtwXv / ChainType.XtwXvy
+ int wix = (chain == ChainType.XtwXv) ? 0 : 1;
+ int vix = (chain == ChainType.XtwXv) ? 1 : 0;
Hop hX = getInput().get(0).getInput().get(0);
- Hop hw = getInput().get(1).getInput().get(0);
- Hop hv = getInput().get(1).getInput().get(1).getInput().get(1);
- mapmmchain = new MapMultChain( hX.constructLops(), hv.constructLops(), hw.constructLops(), getDataType(), getValueType(), ExecType.CP);
+ Hop hw = getInput().get(1).getInput().get(wix);
+ Hop hv = getInput().get(1).getInput().get(vix).getInput().get(1);
+ mapmmchain = new MapMultChain( hX.constructLops(), hv.constructLops(), hw.constructLops(), chain, getDataType(), getValueType(), ExecType.CP);
}
//set degree of parallelism
@@ -768,11 +782,13 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop
Hop hv = getInput().get(1).getInput().get(1);
mapmmchain = new MapMultChain( hX.constructLops(), hv.constructLops(), getDataType(), getValueType(), ExecType.SPARK);
}
- else { //if( chainType == ChainType.XtwXv )
+ else { //ChainType.XtwXv / ChainType.XtXvy
+ int wix = (chain == ChainType.XtwXv) ? 0 : 1;
+ int vix = (chain == ChainType.XtwXv) ? 1 : 0;
Hop hX = getInput().get(0).getInput().get(0);
- Hop hw = getInput().get(1).getInput().get(0);
- Hop hv = getInput().get(1).getInput().get(1).getInput().get(1);
- mapmmchain = new MapMultChain( hX.constructLops(), hv.constructLops(), hw.constructLops(), getDataType(), getValueType(), ExecType.SPARK);
+ Hop hw = getInput().get(1).getInput().get(wix);
+ Hop hv = getInput().get(1).getInput().get(vix).getInput().get(1);
+ mapmmchain = new MapMultChain( hX.constructLops(), hv.constructLops(), hw.constructLops(), chain, getDataType(), getValueType(), ExecType.SPARK);
}
setOutputDimensions(mapmmchain);
setLineNumbers(mapmmchain);
@@ -1121,12 +1137,14 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop
mapmult.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
setLineNumbers(mapmult);
}
- else //if( chainType == ChainType.XtwXv )
+ else //ChainType.XtwXv / ChainType.XtXvy
{
//v never needs partitioning because always single block
+ int wix = (chainType == ChainType.XtwXv) ? 0 : 1;
+ int vix = (chainType == ChainType.XtwXv) ? 1 : 0;
Hop hX = getInput().get(0).getInput().get(0);
- Hop hw = getInput().get(1).getInput().get(0);
- Hop hv = getInput().get(1).getInput().get(1).getInput().get(1);
+ Hop hw = getInput().get(1).getInput().get(wix);
+ Hop hv = getInput().get(1).getInput().get(vix).getInput().get(1);
double mestW = OptimizerUtils.estimateSize(hw.getDim1(), hw.getDim2());
boolean needPart = !hw.dimsKnown() || hw.getDim1() * hw.getDim2() > DistributedCacheInput.PARTITION_SIZE;
@@ -1140,7 +1158,7 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop
w = hw.constructLops();
//core matrix mult
- mapmult = new MapMultChain( X, v, w, getDataType(), getValueType(), ExecType.MR);
+ mapmult = new MapMultChain( X, v, w, chainType, getDataType(), getValueType(), ExecType.MR);
mapmult.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
setLineNumbers(mapmult);
}
@@ -1607,7 +1625,8 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop
{
return MMultMethod.MAPMM_CHAIN;
}
- else if( chainType==ChainType.XtwXv && m1_rows>=0 && m2_cols>=0 && m1_cols>=0
+ else if( (chainType==ChainType.XtwXv || chainType==ChainType.XtXvy )
+ && m1_rows>=0 && m2_cols>=0 && m1_cols>=0
&& OptimizerUtils.estimateSize(m1_rows, m2_cols )
+ OptimizerUtils.estimateSize(m1_cols, m2_cols) < memBudget )
{
@@ -1751,7 +1770,8 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop
{
return MMultMethod.MAPMM_CHAIN;
}
- else if( chainType==ChainType.XtwXv && m1_rows>=0 && m2_cols>=0 && m1_cols>=0
+ else if( (chainType==ChainType.XtwXv || chainType==ChainType.XtXvy )
+ && m1_rows>=0 && m2_cols>=0 && m1_cols>=0
&& OptimizerUtils.estimateSize(m1_rows, m2_cols)
+ OptimizerUtils.estimateSize(m1_cols, m2_cols) < memBudgetExec
&& 2*(OptimizerUtils.estimateSize(m1_rows, m2_cols)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6886e07d/src/main/java/org/apache/sysml/lops/MapMultChain.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/MapMultChain.java b/src/main/java/org/apache/sysml/lops/MapMultChain.java
index 7b08a90..ba424c0 100644
--- a/src/main/java/org/apache/sysml/lops/MapMultChain.java
+++ b/src/main/java/org/apache/sysml/lops/MapMultChain.java
@@ -35,6 +35,7 @@ public class MapMultChain extends Lop
public enum ChainType {
XtXv, //(t(X) %*% (X %*% v))
XtwXv, //(t(X) %*% (w * (X %*% v)))
+ XtXvy, //(t(X) %*% ((X %*% v) - y))
NONE,
}
@@ -71,7 +72,7 @@ public class MapMultChain extends Lop
* @return
* @throws LopsException
*/
- public MapMultChain(Lop input1, Lop input2, Lop input3, DataType dt, ValueType vt, ExecType et)
+ public MapMultChain(Lop input1, Lop input2, Lop input3, ChainType chain, DataType dt, ValueType vt, ExecType et)
throws LopsException
{
super(Lop.Type.MapMultChain, dt, vt);
@@ -83,7 +84,7 @@ public class MapMultChain extends Lop
input3.addOutput(this);
//setup mapmult parameters
- _chainType = ChainType.XtwXv;
+ _chainType = chain;
setupLopProperties(et);
}
@@ -261,7 +262,7 @@ public class MapMultChain extends Lop
{
if( _chainType == ChainType.XtXv )
return new int[]{2};
- else if( _chainType == ChainType.XtwXv )
+ else if( _chainType == ChainType.XtwXv || _chainType == ChainType.XtXvy )
return new int[]{2,3};
//error
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6886e07d/src/main/java/org/apache/sysml/runtime/instructions/cp/MMChainCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MMChainCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MMChainCPInstruction.java
index 6c4c35e..d2bf270 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MMChainCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MMChainCPInstruction.java
@@ -89,7 +89,8 @@ public class MMChainCPInstruction extends UnaryCPInstruction
//get inputs
MatrixBlock X = ec.getMatrixInput(input1.getName());
MatrixBlock v = ec.getMatrixInput(input2.getName());
- MatrixBlock w = (_type==ChainType.XtwXv) ? ec.getMatrixInput(input3.getName()) : null;
+ MatrixBlock w = (_type==ChainType.XtwXv || _type==ChainType.XtXvy) ?
+ ec.getMatrixInput(input3.getName()) : null;
//execute mmchain operation
MatrixBlock out = (MatrixBlock) X.chainMatrixMultOperations(v, w, new MatrixBlock(), _type, _numThreads);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6886e07d/src/main/java/org/apache/sysml/runtime/instructions/mr/MapMultChainInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/MapMultChainInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/MapMultChainInstruction.java
index 4993bee..cb55600 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/MapMultChainInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/MapMultChainInstruction.java
@@ -157,7 +157,7 @@ public class MapMultChainInstruction extends MRInstruction implements IDistribut
if( _chainType == ChainType.XtXv ){
indexes.add(_input2);
}
- else if( _chainType == ChainType.XtwXv ){
+ else {
indexes.add(_input2);
indexes.add(_input3);
}
@@ -210,7 +210,7 @@ public class MapMultChainInstruction extends MRInstruction implements IDistribut
if( _chainType == ChainType.XtXv )
processXtXvOperations(inIx, inVal, outIx, outVal);
else
- processXtwXvOperations(inIx, inVal, outIx, outVal);
+ processXtwXvOperations(inIx, inVal, outIx, outVal, _chainType);
//put the output value in the cache
if(iout==tempValue)
@@ -253,7 +253,7 @@ public class MapMultChainInstruction extends MRInstruction implements IDistribut
* @throws DMLRuntimeException
* @throws DMLUnsupportedOperationException
*/
- private void processXtwXvOperations(MatrixIndexes inIx, MatrixValue inVal, MatrixIndexes outIx, MatrixValue outVal )
+ private void processXtwXvOperations(MatrixIndexes inIx, MatrixValue inVal, MatrixIndexes outIx, MatrixValue outVal, ChainType chain )
throws DMLRuntimeException, DMLUnsupportedOperationException
{
DistributedCacheInput dcInput2 = MRBaseForCommonInstructions.dcValues.get(_input2); //v
@@ -263,7 +263,7 @@ public class MapMultChainInstruction extends MRInstruction implements IDistribut
MatrixBlock w = (MatrixBlock) dcInput3.getDataBlock((int)inIx.getRowIndex(), 1).getValue();
//process core block operation
- Xi.chainMatrixMultOperations(v, w, (MatrixBlock) outVal, ChainType.XtwXv);
+ Xi.chainMatrixMultOperations(v, w, (MatrixBlock) outVal, chain);
outIx.setIndexes(1, 1);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6886e07d/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmChainSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmChainSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmChainSPInstruction.java
index 90bdf76..84e94bc 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmChainSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmChainSPInstruction.java
@@ -137,9 +137,9 @@ public class MapmmChainSPInstruction extends SPInstruction
JavaPairRDD<MatrixIndexes,MatrixBlock> tmp = inX.mapValues(fmmc);
out = RDDAggregateUtils.sumStable(tmp);
}
- else { // ChainType.XtwXv
+ else { // ChainType.XtwXv / ChainType.XtXvy
PartitionedBroadcastMatrix inW = sec.getBroadcastForVariable( _input3.getName() );
- RDDMapMMChainFunction2 fmmc = new RDDMapMMChainFunction2(inV, inW);
+ RDDMapMMChainFunction2 fmmc = new RDDMapMMChainFunction2(inV, inW, _chainType);
JavaPairRDD<MatrixIndexes,MatrixBlock> tmp = inX.mapToPair(fmmc);
out = RDDAggregateUtils.sumStable(tmp);
}
@@ -189,13 +189,15 @@ public class MapmmChainSPInstruction extends SPInstruction
private PartitionedBroadcastMatrix _pmV = null;
private PartitionedBroadcastMatrix _pmW = null;
+ private ChainType _chainType = null;
- public RDDMapMMChainFunction2( PartitionedBroadcastMatrix bV, PartitionedBroadcastMatrix bW)
+ public RDDMapMMChainFunction2( PartitionedBroadcastMatrix bV, PartitionedBroadcastMatrix bW, ChainType chain)
throws DMLRuntimeException, DMLUnsupportedOperationException
{
//get both broadcast vectors (first always single block)
_pmV = bV;
_pmW = bW;
+ _chainType = chain;
}
@Override
@@ -212,7 +214,7 @@ public class MapmmChainSPInstruction extends SPInstruction
MatrixBlock blkOut = new MatrixBlock();
//execute mapmmchain operation
- blkIn.chainMatrixMultOperations(pmV, _pmW.getMatrixBlock(rowIx,1), blkOut, ChainType.XtwXv);
+ blkIn.chainMatrixMultOperations(pmV, _pmW.getMatrixBlock(rowIx,1), blkOut, _chainType);
//output new tuple
return new Tuple2<MatrixIndexes, MatrixBlock>(ixOut, blkOut);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6886e07d/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java
index ccb6cad..1408d4d 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java
@@ -424,7 +424,6 @@ public class RDDConverterUtilsExt
}
- @SuppressWarnings("unused")
private static class CSVToBinaryBlockFunction implements PairFlatMapFunction<Iterator<Tuple2<Text,Long>>,MatrixIndexes,MatrixBlock> {
private static final long serialVersionUID = 1501589201971233542L;
@@ -472,13 +471,11 @@ public class RDDConverterUtilsExt
_aNnz = aNnz;
}
- @SuppressWarnings("unused")
public RowAnalysisFunctionHelper( Accumulator<Double> aNnz, String delim ) {
_aNnz = aNnz;
_delim = delim;
}
- @SuppressWarnings("unused")
public String analyzeText(Text v1) throws Exception {
//parse input line
String line = v1.toString();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6886e07d/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
index 6def2dd..e234640 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
@@ -1597,7 +1597,8 @@ public class LibMatrixMult
double[] c = ret.denseBlock;
final int cd = mX.clen; //features in X
boolean weights = (ct == ChainType.XtwXv);
-
+ boolean weights2 = (ct == ChainType.XtXvy);
+
//temporary array for cache blocking
//(blocksize chosen to fit b+v in L2 (256KB) for default 1k blocks)
final int blocksize = 24; // constraint: factor of 4
@@ -1611,10 +1612,12 @@ public class LibMatrixMult
for( int j=0, aix=bi*cd; j < blocksize; j++, aix+=cd)
tmp[j] = dotProduct(a, b, aix, 0, cd);
- //multiply weights (in-place), if required
+ //multiply/substract weights (in-place), if required
if( weights )
vectMultiply(w, tmp, bi, 0, blocksize);
-
+ else if( weights2 )
+ vectSubstract(w, tmp, bi, 0, blocksize);
+
//compute 2nd matrix vector for row block and aggregate
for (int j=0, aix=bi*cd; j < blocksize; j+=4, aix+=4*cd)
vectMultiplyAdd4(tmp[j], tmp[j+1], tmp[j+2], tmp[j+3], a, c, aix, aix+cd, aix+2*cd, aix+3*cd, 0, cd);
@@ -1623,7 +1626,8 @@ public class LibMatrixMult
//compute rest (not aligned to blocksize)
for( int i=bn, aix=bn*cd; i < ru; i++, aix+=cd ) {
double val = dotProduct(a, b, aix, 0, cd);
- val *= (weights) ? w[i] : 1;
+ val *= (weights) ? w[i] : 1;
+ val -= (weights2) ? w[i] : 0;
vectMultiplyAdd(val, a, c, aix, 0, cd);
}
}
@@ -1647,6 +1651,7 @@ public class LibMatrixMult
double[] w = (mW!=null) ? mW.denseBlock : null;
double[] c = ret.denseBlock;
boolean weights = (ct == ChainType.XtwXv);
+ boolean weights2 = (ct == ChainType.XtXvy);
//temporary array for cache blocking
//(blocksize chosen to fit b+v in L2 (256KB) for default 1k blocks)
@@ -1673,7 +1678,9 @@ public class LibMatrixMult
//multiply weights (in-place), if required
if( weights )
vectMultiply(w, tmp, bi, 0, tmplen);
-
+ else if( weights2 )
+ vectSubstract(w, tmp, bi, 0, tmplen);
+
//compute 2nd matrix vector for row block and aggregate
for( int j=0; j < tmplen; j++) {
SparseRow arow = a[bi+j];
@@ -3468,6 +3475,38 @@ public class LibMatrixMult
}
}
+ /**
+ *
+ * @param a
+ * @param c
+ * @param ai
+ * @param ci
+ * @param len
+ */
+ private static void vectSubstract( double[] a, double[] c, int ai, int ci, final int len )
+ {
+ final int bn = len%8;
+
+ //rest, not aligned to 8-blocks
+ for( int j = 0; j < bn; j++, ai++, ci++)
+ c[ ci ] += a[ ai ];
+
+ //unrolled 8-block (for better instruction-level parallelism)
+ for( int j = bn; j < len; j+=8, ai+=8, ci+=8)
+ {
+ //read 64B cachelines of a and c
+ //compute c' = c * a
+ //write back 64B cacheline of c = c'
+ c[ ci+0 ] -= a[ ai+0 ];
+ c[ ci+1 ] -= a[ ai+1 ];
+ c[ ci+2 ] -= a[ ai+2 ];
+ c[ ci+3 ] -= a[ ai+3 ];
+ c[ ci+4 ] -= a[ ai+4 ];
+ c[ ci+5 ] -= a[ ai+5 ];
+ c[ ci+6 ] -= a[ ai+6 ];
+ c[ ci+7 ] -= a[ ai+7 ];
+ }
+ }
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6886e07d/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index 7a58103..b7819b8 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -3834,7 +3834,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
throws DMLRuntimeException, DMLUnsupportedOperationException
{
//check for transpose type
- if( !(ctype == ChainType.XtXv || ctype == ChainType.XtwXv) )
+ if( !(ctype == ChainType.XtXv || ctype == ChainType.XtwXv || ctype == ChainType.XtXvy) )
throw new DMLRuntimeException("Invalid mmchain type '"+ctype.toString()+"'.");
//check for matching dimensions
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6886e07d/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MapMultChainTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MapMultChainTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MapMultChainTest.java
index 47c2793..5e597d6 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MapMultChainTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/MapMultChainTest.java
@@ -42,6 +42,7 @@ public class MapMultChainTest extends AutomatedTestBase
private final static String TEST_NAME1 = "MapMultChain";
private final static String TEST_NAME2 = "MapMultChainWeights";
+ private final static String TEST_NAME3 = "MapMultChainWeights2";
private final static String TEST_DIR = "functions/binary/matrix/";
private final static String TEST_CLASS_DIR = TEST_DIR + MapMultChainTest.class.getSimpleName() + "/";
private final static double eps = 1e-8;
@@ -61,7 +62,9 @@ public class MapMultChainTest extends AutomatedTestBase
new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "R" }));
addTestConfiguration(TEST_NAME2,
new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] { "R" }));
-
+ addTestConfiguration(TEST_NAME3,
+ new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] { "R" }));
+
if (TEST_CACHE_ENABLED) {
setOutAndExpectedDeletionDisabled(true);
}
@@ -80,7 +83,7 @@ public class MapMultChainTest extends AutomatedTestBase
TestUtils.clearDirectory(TEST_DATA_DIR + TEST_CLASS_DIR);
}
}
-
+
@Test
public void testMapMultChainNoRewriteDenseCP()
{
@@ -94,6 +97,12 @@ public class MapMultChainTest extends AutomatedTestBase
}
@Test
+ public void testMapMultChainWeights2NoRewriteDenseCP()
+ {
+ runMapMultChainTest(TEST_NAME3, false, false, ExecType.CP);
+ }
+
+ @Test
public void testMapMultChainNoRewriteSparseCP()
{
runMapMultChainTest(TEST_NAME1, true, false, ExecType.CP);
@@ -106,6 +115,12 @@ public class MapMultChainTest extends AutomatedTestBase
}
@Test
+ public void testMapMultChainWeights2NoRewriteSparseCP()
+ {
+ runMapMultChainTest(TEST_NAME3, true, false, ExecType.CP);
+ }
+
+ @Test
public void testMapMultChainRewriteDenseCP()
{
runMapMultChainTest(TEST_NAME1, false, true, ExecType.CP);
@@ -118,6 +133,12 @@ public class MapMultChainTest extends AutomatedTestBase
}
@Test
+ public void testMapMultChainWeights2RewriteDenseCP()
+ {
+ runMapMultChainTest(TEST_NAME3, false, true, ExecType.CP);
+ }
+
+ @Test
public void testMapMultChainRewriteSparseCP()
{
runMapMultChainTest(TEST_NAME1, true, true, ExecType.CP);
@@ -130,6 +151,12 @@ public class MapMultChainTest extends AutomatedTestBase
}
@Test
+ public void testMapMultChainWeights2RewriteSparseCP()
+ {
+ runMapMultChainTest(TEST_NAME3, true, true, ExecType.CP);
+ }
+
+ @Test
public void testMapMultChainNoRewriteDenseMR()
{
runMapMultChainTest(TEST_NAME1, false, false, ExecType.MR);
@@ -142,6 +169,12 @@ public class MapMultChainTest extends AutomatedTestBase
}
@Test
+ public void testMapMultChainWeights2NoRewriteDenseMR()
+ {
+ runMapMultChainTest(TEST_NAME3, false, false, ExecType.MR);
+ }
+
+ @Test
public void testMapMultChainNoRewriteSparseMR()
{
runMapMultChainTest(TEST_NAME1, true, false, ExecType.MR);
@@ -154,6 +187,12 @@ public class MapMultChainTest extends AutomatedTestBase
}
@Test
+ public void testMapMultChainWeights2NoRewriteSparseMR()
+ {
+ runMapMultChainTest(TEST_NAME3, true, false, ExecType.MR);
+ }
+
+ @Test
public void testMapMultChainRewriteDenseMR()
{
runMapMultChainTest(TEST_NAME1, false, true, ExecType.MR);
@@ -166,6 +205,12 @@ public class MapMultChainTest extends AutomatedTestBase
}
@Test
+ public void testMapMultChainWeights2RewriteDenseMR()
+ {
+ runMapMultChainTest(TEST_NAME3, false, true, ExecType.MR);
+ }
+
+ @Test
public void testMapMultChainRewriteSparseMR()
{
runMapMultChainTest(TEST_NAME1, true, true, ExecType.MR);
@@ -178,6 +223,12 @@ public class MapMultChainTest extends AutomatedTestBase
}
@Test
+ public void testMapMultChainWeights2RewriteSparseMR()
+ {
+ runMapMultChainTest(TEST_NAME3, true, true, ExecType.MR);
+ }
+
+ @Test
public void testMapMultChainNoRewriteDenseSpark()
{
runMapMultChainTest(TEST_NAME1, false, false, ExecType.SPARK);
@@ -190,6 +241,12 @@ public class MapMultChainTest extends AutomatedTestBase
}
@Test
+ public void testMapMultChainWeights2NoRewriteDenseSpark()
+ {
+ runMapMultChainTest(TEST_NAME3, false, false, ExecType.SPARK);
+ }
+
+ @Test
public void testMapMultChainNoRewriteSparseSpark()
{
runMapMultChainTest(TEST_NAME1, true, false, ExecType.SPARK);
@@ -202,6 +259,12 @@ public class MapMultChainTest extends AutomatedTestBase
}
@Test
+ public void testMapMultChainWeights2NoRewriteSparseSpark()
+ {
+ runMapMultChainTest(TEST_NAME3, true, false, ExecType.SPARK);
+ }
+
+ @Test
public void testMapMultChainRewriteDenseSpark()
{
runMapMultChainTest(TEST_NAME1, false, true, ExecType.SPARK);
@@ -214,6 +277,12 @@ public class MapMultChainTest extends AutomatedTestBase
}
@Test
+ public void testMapMultChainWeights2RewriteDenseSpark()
+ {
+ runMapMultChainTest(TEST_NAME3, false, true, ExecType.SPARK);
+ }
+
+ @Test
public void testMapMultChainRewriteSparseSpark()
{
runMapMultChainTest(TEST_NAME1, true, true, ExecType.SPARK);
@@ -224,6 +293,12 @@ public class MapMultChainTest extends AutomatedTestBase
{
runMapMultChainTest(TEST_NAME2, true, true, ExecType.SPARK);
}
+
+ @Test
+ public void testMapMultChainWeights2RewriteSparseSpark()
+ {
+ runMapMultChainTest(TEST_NAME3, true, true, ExecType.SPARK);
+ }
/**
*
@@ -253,16 +328,9 @@ public class MapMultChainTest extends AutomatedTestBase
{
String TEST_NAME = testname;
TestConfiguration config = getTestConfiguration(TEST_NAME);
-
- String TEST_CACHE_DIR = "";
- if (TEST_CACHE_ENABLED)
- {
- TEST_CACHE_DIR = TEST_NAME + "_" + sparse + "/";
- }
-
+ String TEST_CACHE_DIR = (TEST_CACHE_ENABLED)? TEST_NAME + "_" + sparse + "/" : "";
loadTestConfiguration(config, TEST_CACHE_DIR);
- /* This is for running the junit test the new way, i.e., construct the arguments directly */
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + TEST_NAME + ".dml";
programArgs = new String[]{"-stats","-args",
@@ -276,7 +344,7 @@ public class MapMultChainTest extends AutomatedTestBase
writeInputMatrixWithMTD("X", X, true);
double[][] v = getRandomMatrix(colsX, 1, 0, 1, sparsity1, 3);
writeInputMatrixWithMTD("v", v, true);
- if( TEST_NAME.equals(TEST_NAME2) ){
+ if( !TEST_NAME.equals(TEST_NAME1) ){
double[][] w = getRandomMatrix(rowsX, 1, 0, 1, sparse?sparsity2:sparsity1, 10);
writeInputMatrixWithMTD("w", w, true);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6886e07d/src/test/scripts/functions/binary/matrix/MapMultChainWeights2.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/MapMultChainWeights2.R b/src/test/scripts/functions/binary/matrix/MapMultChainWeights2.R
new file mode 100644
index 0000000..2c7320d
--- /dev/null
+++ b/src/test/scripts/functions/binary/matrix/MapMultChainWeights2.R
@@ -0,0 +1,35 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args <- commandArgs(TRUE)
+options(digits=22)
+
+library("Matrix")
+
+X <- readMM(paste(args[1], "X.mtx", sep=""))
+v <- readMM(paste(args[1], "v.mtx", sep=""))
+w <- readMM(paste(args[1], "w.mtx", sep=""))
+
+R = t(X) %*% ((X %*% v)-w);
+
+writeMM(as(R, "CsparseMatrix"), paste(args[2], "R", sep=""));
+
+
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6886e07d/src/test/scripts/functions/binary/matrix/MapMultChainWeights2.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/binary/matrix/MapMultChainWeights2.dml b/src/test/scripts/functions/binary/matrix/MapMultChainWeights2.dml
new file mode 100644
index 0000000..e5c2edf
--- /dev/null
+++ b/src/test/scripts/functions/binary/matrix/MapMultChainWeights2.dml
@@ -0,0 +1,28 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+X = read($1);
+v = read($2);
+w = read($3);
+
+R = t(X) %*% ((X %*% v)-w);
+
+write(R, $4, format="text");
\ No newline at end of file