You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/03/29 07:32:19 UTC
[1/2] incubator-systemml git commit: [SYSTEMML-603] Fix rewrite
'transpose-append' w/ multiple consumers
Repository: incubator-systemml
Updated Branches:
refs/heads/master de8d49d34 -> a99fa6eaa
[SYSTEMML-603] Fix rewrite 'transpose-append' w/ multiple consumers
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/1fe10a29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/1fe10a29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/1fe10a29
Branch: refs/heads/master
Commit: 1fe10a29516ab7a91514b87c17bdf9ca0d720c3e
Parents: de8d49d
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Mon Mar 28 22:31:26 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Mon Mar 28 22:31:26 2016 -0700
----------------------------------------------------------------------
.../RewriteAlgebraicSimplificationStatic.java | 29 +++++++-------------
1 file changed, 10 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1fe10a29/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
index 0cc9f40..e40bfd3 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
@@ -955,36 +955,27 @@ public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
&& hi.getInput().get(0) instanceof BinaryOp
&& (((BinaryOp)hi.getInput().get(0)).getOp()==OpOp2.CBIND //append (cbind/rbind)
|| ((BinaryOp)hi.getInput().get(0)).getOp()==OpOp2.RBIND)
- && hi.getInput().get(0).getParent().size() == 1 ) //single consumers of append
+ && hi.getInput().get(0).getParent().size() == 1 ) //single consumer of append
{
BinaryOp bop = (BinaryOp)hi.getInput().get(0);
if( bop.getInput().get(0) instanceof ReorgOp //both inputs transpose ops
&& ((ReorgOp)bop.getInput().get(0)).getOp()==ReOrgOp.TRANSPOSE
+ && bop.getInput().get(0).getParent().size() == 1 //single consumer of transpose
&& bop.getInput().get(1) instanceof ReorgOp
- && ((ReorgOp)bop.getInput().get(1)).getOp()==ReOrgOp.TRANSPOSE )
+ && ((ReorgOp)bop.getInput().get(1)).getOp()==ReOrgOp.TRANSPOSE
+ && bop.getInput().get(1).getParent().size() == 1 ) //single consumer of transpose
{
Hop left = bop.getInput().get(0).getInput().get(0);
Hop right = bop.getInput().get(1).getInput().get(0);
- //rewire links from parent, transpose, and binary
+ //create new subdag (no in-place dag update to prevent anomalies with
+ //multiple consumers during rewrite process)
HopRewriteUtils.removeChildReferenceByPos(parent, hi, pos);
- HopRewriteUtils.addChildReference(parent, bop, pos);
- HopRewriteUtils.removeAllChildReferences(hi);
- HopRewriteUtils.removeAllChildReferences(bop);
-
- //change append type (safe due to single parent check)
- if( bop.getOp()==OpOp2.CBIND )
- bop.setOp(OpOp2.RBIND);
- else
- bop.setOp(OpOp2.CBIND);
+ OpOp2 binop = (bop.getOp()==OpOp2.CBIND) ? OpOp2.RBIND : OpOp2.CBIND;
+ BinaryOp bopnew = HopRewriteUtils.createBinary(left, right, binop);
+ HopRewriteUtils.addChildReference(parent, bopnew, pos);
- //relink new childs to binary op
- HopRewriteUtils.addChildReference(bop, left, 0);
- HopRewriteUtils.addChildReference(bop, right, 1);
- bop.refreshSizeInformation();
-
- hi = bop;
-
+ hi = bopnew;
LOG.debug("Applied simplifyTransposedAppend (line "+hi.getBeginLine()+").");
}
}
[2/2] incubator-systemml git commit: [HOTFIX] Fix mr leftindexing
(shifting, dimensions), incl cleanup spark
Posted by mb...@apache.org.
[HOTFIX] Fix mr leftindexing (shifting, dimensions), incl cleanup spark
Commit 42e471079b296529c2e54a72961e11d28ab70dda made a couple of
cleanups for block/cell computation utils. This introduced an mr
leftindexing bug as the old blocksize computation specifically
compensated for an incorrect leftindexing implementation (which allowed
reuse with right indexing). It did not showup during local test runs
because a larger memory budget changed plans to cp indexing, whereas on
jenkins for example ARIMA used mr indexing. Instead of reverting the
mentioned commit, this patch actually fixes the mr leftindexing
implementation by consolidating it with the spark leftindexing
functionality. Furthermore, this change also fixes issues with dimension
inference for mr leftindexing as well as hardens the related tests by
explicitly checking for the output characteristics of test results.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/a99fa6ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/a99fa6ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/a99fa6ea
Branch: refs/heads/master
Commit: a99fa6eaa6ef097f1a0452f3c33c7c271c3a1761
Parents: 1fe10a2
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Mon Mar 28 22:31:56 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Mon Mar 28 22:31:56 2016 -0700
----------------------------------------------------------------------
.../mr/RangeBasedReIndexInstruction.java | 86 ++++++-------
.../spark/MatrixIndexingSPInstruction.java | 125 ++++++-------------
.../runtime/matrix/MatrixCharacteristics.java | 10 +-
.../matrix/data/OperationsOnMatrixValues.java | 64 ++++++++++
.../indexing/LeftIndexingScalarTest.java | 3 +-
.../indexing/LeftIndexingSparseDenseTest.java | 3 +-
.../indexing/LeftIndexingSparseSparseTest.java | 3 +-
7 files changed, 150 insertions(+), 144 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a99fa6ea/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java
index 0368e35..5ca5aa3 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.DMLUnsupportedOperationException;
import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.MatrixValue;
import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
import org.apache.sysml.runtime.matrix.mapred.CachedValueMap;
@@ -36,41 +37,48 @@ import org.apache.sysml.runtime.util.UtilFunctions;
public class RangeBasedReIndexInstruction extends UnaryMRInstructionBase
{
+ private IndexRange _ixrange = null;
+ private boolean _forLeft = false;
+ private long _rlenLhs = -1;
+ private long _clenLhs = -1;
- protected boolean forLeftIndexing=false;
- protected long leftMatrixNRows=0;
- protected long leftMatrixNCols=0;
-
- private IndexRange indexRange=null;
-
- public RangeBasedReIndexInstruction(Operator op, byte in, byte out, IndexRange rng, String istr) {
+ public RangeBasedReIndexInstruction(Operator op, byte in, byte out, IndexRange rng, boolean forleft, long rlen, long clen, String istr) {
super(op, in, out);
mrtype = MRINSTRUCTION_TYPE.RangeReIndex;
instString = istr;
- indexRange=rng;
+ _ixrange = rng;
+ _forLeft = forleft;
+ _rlenLhs = rlen;
+ _clenLhs = clen;
}
- public RangeBasedReIndexInstruction(Operator op, byte in, byte out, IndexRange rng, boolean forleft, long leftNRows, long leftNCols, String istr) {
- super(op, in, out);
- mrtype = MRINSTRUCTION_TYPE.RangeReIndex;
- instString = istr;
- indexRange=rng;
- this.forLeftIndexing=forleft;
- this.leftMatrixNRows=leftNRows;
- this.leftMatrixNCols=leftNCols;
+ /**
+ *
+ * @param mcIn
+ * @param mcOut
+ */
+ public void computeOutputCharacteristics(MatrixCharacteristics mcIn, MatrixCharacteristics mcOut) {
+ if( _forLeft )
+ mcOut.set(_rlenLhs, _clenLhs, mcIn.getRowsPerBlock(), mcIn.getColsPerBlock(), -1);
+ else
+ mcOut.set(_ixrange.rowEnd-_ixrange.rowStart+1, _ixrange.colEnd-_ixrange.colStart+1,
+ mcIn.getRowsPerBlock(), mcIn.getColsPerBlock(), -1);
}
- public IndexRange getIndexRange() {
- return indexRange;
- }
-
- public static RangeBasedReIndexInstruction parseInstruction ( String str ) throws DMLRuntimeException {
-
+ /**
+ *
+ * @param str
+ * @return
+ * @throws DMLRuntimeException
+ */
+ public static RangeBasedReIndexInstruction parseInstruction( String str )
+ throws DMLRuntimeException
+ {
InstructionUtils.checkNumFields ( str, 8 );
String[] parts = InstructionUtils.getInstructionParts ( str );
String opcode = parts[0];
- boolean forLeft=false;
+ boolean forLeft = false;
if(opcode.equalsIgnoreCase("rangeReIndexForLeft"))
forLeft=true;
else if(!opcode.equalsIgnoreCase("rangeReIndex"))
@@ -79,37 +87,22 @@ public class RangeBasedReIndexInstruction extends UnaryMRInstructionBase
IndexRange rng=new IndexRange(UtilFunctions.parseToLong(parts[2]),
UtilFunctions.parseToLong(parts[3]),
UtilFunctions.parseToLong(parts[4]),
- UtilFunctions.parseToLong(parts[5]));
+ UtilFunctions.parseToLong(parts[5]));
byte out = Byte.parseByte(parts[6]);
- long leftIndexingNrow=Long.parseLong(parts[7]);
- long leftIndexingNcol=Long.parseLong(parts[8]);
+ long rlen = Long.parseLong(parts[7]);
+ long clen = Long.parseLong(parts[8]);
- //recalculate the index range for left indexing
- if(forLeft)
- {
- long a=rng.rowStart;
- long b=rng.colStart;
- rng.rowStart=2-a;
- rng.colStart=2-b;
- //don't need to extend to the whole left matrix dimension
- rng.rowEnd=leftIndexingNrow-a+1;
- rng.colEnd=leftIndexingNcol-b+1;
- return new RangeBasedReIndexInstruction(new ReIndexOperator(), in, out, rng, forLeft, leftIndexingNrow, leftIndexingNcol, str);
- }else
- return new RangeBasedReIndexInstruction(new ReIndexOperator(), in, out, rng, str);
+ return new RangeBasedReIndexInstruction(new ReIndexOperator(), in, out, rng, forLeft, rlen, clen, str);
}
-
-
@Override
public void processInstruction(Class<? extends MatrixValue> valueClass,
CachedValueMap cachedValues, IndexedMatrixValue tempValue,
IndexedMatrixValue zeroInput, int blockRowFactor, int blockColFactor)
throws DMLUnsupportedOperationException, DMLRuntimeException
- {
-
+ {
if(input==output)
- throw new DMLRuntimeException("input cannot be the same for output for "+this.instString);
+ throw new DMLRuntimeException("input cannot be the same for output for "+instString);
ArrayList<IndexedMatrixValue> blkList = cachedValues.get(input);
if( blkList != null ) {
@@ -120,7 +113,10 @@ public class RangeBasedReIndexInstruction extends UnaryMRInstructionBase
//process instruction (incl block allocation)
ArrayList<IndexedMatrixValue> outlist = new ArrayList<IndexedMatrixValue>();
- OperationsOnMatrixValues.performSlice(in, indexRange, blockRowFactor, blockColFactor, outlist);
+ if( _forLeft )
+ OperationsOnMatrixValues.performShift(in, _ixrange, blockRowFactor, blockColFactor, _rlenLhs, _clenLhs, outlist);
+ else
+ OperationsOnMatrixValues.performSlice(in, _ixrange, blockRowFactor, blockColFactor, outlist);
//put blocks into result cache
for( IndexedMatrixValue ret : outlist )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a99fa6ea/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
index 3bf4d98..b379d2c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
@@ -210,15 +210,13 @@ public class MatrixIndexingSPInstruction extends UnarySPInstruction
out = in1.mapPartitionsToPair(
new LeftIndexPartitionFunction(broadcastIn2, ixrange, mcOut), true);
}
- else {
- // Zero-out LHS
- in1 = in1.mapToPair(new ZeroOutLHS(false, mcLeft.getRowsPerBlock(),
- mcLeft.getColsPerBlock(), rl, ru, cl, cu));
+ else { //general case
+ // zero-out lhs
+ in1 = in1.mapToPair(new ZeroOutLHS(false, ixrange, mcLeft));
- // Slice RHS to merge for LHS
+ // slice rhs, shift and merge with lhs
in2 = sec.getBinaryBlockRDDHandleForVariable( input2.getName() )
- .flatMapToPair(new SliceRHSForLeftIndexing(rl, cl, mcLeft.getRowsPerBlock(), mcLeft.getColsPerBlock(), mcLeft.getRows(), mcLeft.getCols()));
-
+ .flatMapToPair(new SliceRHSForLeftIndexing(ixrange, mcLeft));
out = RDDAggregateUtils.mergeByKey(in1.union(in2));
}
@@ -267,74 +265,29 @@ public class MatrixIndexingSPInstruction extends UnarySPInstruction
{
private static final long serialVersionUID = 5724800998701216440L;
- private long rl;
- private long cl;
- private int brlen;
- private int bclen;
- private long lhs_rlen;
- private long lhs_clen;
+ private IndexRange _ixrange = null;
+ private int _brlen = -1;
+ private int _bclen = -1;
+ private long _rlen = -1;
+ private long _clen = -1;
- public SliceRHSForLeftIndexing(long rl, long cl, int brlen, int bclen, long lhs_rlen, long lhs_clen) {
- this.rl = rl;
- this.cl = cl;
- this.brlen = brlen;
- this.bclen = bclen;
- this.lhs_rlen = lhs_rlen;
- this.lhs_clen = lhs_clen;
+ public SliceRHSForLeftIndexing(IndexRange ixrange, MatrixCharacteristics mcLeft) {
+ _ixrange = ixrange;
+ _rlen = mcLeft.getRows();
+ _clen = mcLeft.getCols();
+ _brlen = mcLeft.getRowsPerBlock();
+ _bclen = mcLeft.getColsPerBlock();
}
@Override
public Iterable<Tuple2<MatrixIndexes, MatrixBlock>> call(Tuple2<MatrixIndexes, MatrixBlock> rightKV)
throws Exception
{
- ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> retVal = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>();
-
- long start_lhs_globalRowIndex = rl + (rightKV._1.getRowIndex()-1)*brlen;
- long start_lhs_globalColIndex = cl + (rightKV._1.getColumnIndex()-1)*bclen;
- long end_lhs_globalRowIndex = start_lhs_globalRowIndex + rightKV._2.getNumRows() - 1;
- long end_lhs_globalColIndex = start_lhs_globalColIndex + rightKV._2.getNumColumns() - 1;
-
- long start_lhs_rowIndex = UtilFunctions.computeBlockIndex(start_lhs_globalRowIndex, brlen);
- long end_lhs_rowIndex = UtilFunctions.computeBlockIndex(end_lhs_globalRowIndex, brlen);
- long start_lhs_colIndex = UtilFunctions.computeBlockIndex(start_lhs_globalColIndex, bclen);
- long end_lhs_colIndex = UtilFunctions.computeBlockIndex(end_lhs_globalColIndex, bclen);
-
- for(long leftRowIndex = start_lhs_rowIndex; leftRowIndex <= end_lhs_rowIndex; leftRowIndex++) {
- for(long leftColIndex = start_lhs_colIndex; leftColIndex <= end_lhs_colIndex; leftColIndex++) {
-
- // Calculate global index of right hand side block
- long lhs_rl = Math.max((leftRowIndex-1)*brlen+1, start_lhs_globalRowIndex);
- long lhs_ru = Math.min(leftRowIndex*brlen, end_lhs_globalRowIndex);
- long lhs_cl = Math.max((leftColIndex-1)*bclen+1, start_lhs_globalColIndex);
- long lhs_cu = Math.min(leftColIndex*bclen, end_lhs_globalColIndex);
-
- int lhs_lrl = UtilFunctions.computeCellInBlock(lhs_rl, brlen);
- int lhs_lru = UtilFunctions.computeCellInBlock(lhs_ru, brlen);
- int lhs_lcl = UtilFunctions.computeCellInBlock(lhs_cl, bclen);
- int lhs_lcu = UtilFunctions.computeCellInBlock(lhs_cu, bclen);
-
- long rhs_rl = lhs_rl - rl + 1;
- long rhs_ru = rhs_rl + (lhs_ru - lhs_rl);
- long rhs_cl = lhs_cl - cl + 1;
- long rhs_cu = rhs_cl + (lhs_cu - lhs_cl);
-
- int rhs_lrl = UtilFunctions.computeCellInBlock(rhs_rl, brlen);
- int rhs_lru = UtilFunctions.computeCellInBlock(rhs_ru, brlen);
- int rhs_lcl = UtilFunctions.computeCellInBlock(rhs_cl, bclen);
- int rhs_lcu = UtilFunctions.computeCellInBlock(rhs_cu, bclen);
-
- MatrixBlock slicedRHSBlk = rightKV._2.sliceOperations(rhs_lrl, rhs_lru, rhs_lcl, rhs_lcu, new MatrixBlock());
-
- int lbrlen = UtilFunctions.computeBlockSize(lhs_rlen, leftRowIndex, brlen);
- int lbclen = UtilFunctions.computeBlockSize(lhs_clen, leftColIndex, bclen);
- MatrixBlock resultBlock = new MatrixBlock(lbrlen, lbclen, false);
- resultBlock = resultBlock.leftIndexingOperations(slicedRHSBlk, lhs_lrl, lhs_lru, lhs_lcl, lhs_lcu, null, false);
- retVal.add(new Tuple2<MatrixIndexes, MatrixBlock>(new MatrixIndexes(leftRowIndex, leftColIndex), resultBlock));
- }
- }
- return retVal;
- }
-
+ IndexedMatrixValue in = SparkUtils.toIndexedMatrixBlock(rightKV);
+ ArrayList<IndexedMatrixValue> out = new ArrayList<IndexedMatrixValue>();
+ OperationsOnMatrixValues.performShift(in, _ixrange, _brlen, _bclen, _rlen, _clen, out);
+ return SparkUtils.fromIndexedMatrixBlock(out);
+ }
}
/**
@@ -344,39 +297,34 @@ public class MatrixIndexingSPInstruction extends UnarySPInstruction
{
private static final long serialVersionUID = -3581795160948484261L;
- private boolean complementary = false;
- private int brlen; int bclen;
- private IndexRange indexRange;
- private long rl; long ru; long cl; long cu;
+ private boolean _complement = false;
+ private IndexRange _ixrange = null;
+ private int _brlen = -1;
+ private int _bclen = -1;
- public ZeroOutLHS(boolean complementary, int brlen, int bclen, long rl, long ru, long cl, long cu) {
- this.complementary = complementary;
- this.brlen = brlen;
- this.bclen = bclen;
- this.rl = rl;
- this.ru = ru;
- this.cl = cl;
- this.cu = cu;
- this.indexRange = new IndexRange(rl, ru, cl, cu);
+ public ZeroOutLHS(boolean complement, IndexRange range, MatrixCharacteristics mcLeft) {
+ _complement = complement;
+ _ixrange = range;
+ _brlen = mcLeft.getRowsPerBlock();
+ _bclen = mcLeft.getColsPerBlock();
}
@Override
public Tuple2<MatrixIndexes, MatrixBlock> call(Tuple2<MatrixIndexes, MatrixBlock> kv)
throws Exception
{
- if( !UtilFunctions.isInBlockRange(kv._1(), brlen, bclen, rl, ru, cl, cu) ) {
+ if( !UtilFunctions.isInBlockRange(kv._1(), _brlen, _bclen, _ixrange) ) {
return kv;
}
- IndexRange range = UtilFunctions.getSelectedRangeForZeroOut(new IndexedMatrixValue(kv._1, kv._2), brlen, bclen, indexRange);
+ IndexRange range = UtilFunctions.getSelectedRangeForZeroOut(new IndexedMatrixValue(kv._1, kv._2), _brlen, _bclen, _ixrange);
if(range.rowStart == -1 && range.rowEnd == -1 && range.colStart == -1 && range.colEnd == -1) {
throw new Exception("Error while getting range for zero-out");
}
- MatrixBlock zeroBlk = (MatrixBlock) kv._2.zeroOutOperations(new MatrixBlock(), range, complementary);
+ MatrixBlock zeroBlk = (MatrixBlock) kv._2.zeroOutOperations(new MatrixBlock(), range, _complement);
return new Tuple2<MatrixIndexes, MatrixBlock>(kv._1, zeroBlk);
}
-
}
/**
@@ -387,10 +335,9 @@ public class MatrixIndexingSPInstruction extends UnarySPInstruction
private static final long serialVersionUID = 1757075506076838258L;
private PartitionedBroadcastMatrix _binput;
- private IndexRange _ixrange;
- private int _brlen;
- private int _bclen;
-
+ private IndexRange _ixrange = null;
+ private int _brlen = -1;
+ private int _bclen = -1;
public LeftIndexPartitionFunction(PartitionedBroadcastMatrix binput, IndexRange ixrange, MatrixCharacteristics mc)
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a99fa6ea/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java b/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java
index 4618099..8151465 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java
@@ -64,7 +64,6 @@ import org.apache.sysml.runtime.instructions.mr.ZeroOutInstruction;
import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator;
import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator;
import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
-import org.apache.sysml.runtime.util.IndexRange;
public class MatrixCharacteristics implements Serializable
@@ -385,12 +384,9 @@ public class MatrixCharacteristics implements Serializable
}
else if(ins instanceof RangeBasedReIndexInstruction)
{
- RangeBasedReIndexInstruction realIns=(RangeBasedReIndexInstruction)ins;
- MatrixCharacteristics in_dim=dims.get(realIns.input);
- IndexRange ixrange = realIns.getIndexRange();
- long nrow=ixrange.rowEnd-ixrange.rowStart+1;
- long ncol=ixrange.colEnd-ixrange.colStart+1;
- dimOut.set(nrow, ncol, in_dim.numRowsPerBlock, in_dim.numColumnsPerBlock);
+ RangeBasedReIndexInstruction realIns = (RangeBasedReIndexInstruction)ins;
+ MatrixCharacteristics dimIn = dims.get(realIns.input);
+ realIns.computeOutputCharacteristics(dimIn, dimOut);
}
else if (ins instanceof TernaryInstruction) {
TernaryInstruction realIns = (TernaryInstruction)ins;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a99fa6ea/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
index ea33de3..9201795 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
@@ -38,6 +38,7 @@ import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
import org.apache.sysml.runtime.util.IndexRange;
import org.apache.sysml.runtime.util.UtilFunctions;
+
public class OperationsOnMatrixValues
{
@@ -362,6 +363,69 @@ public class OperationsOnMatrixValues
//execute actual slice operation
in.getValue().sliceOperations(outlist, tmpRange, rowCut, colCut, brlen, bclen, boundaryRlen, boundaryClen);
}
+
+ /**
+ *
+ * @param in
+ * @param ixrange
+ * @param brlen
+ * @param bclen
+ * @param rlen
+ * @param clen
+ * @param outlist
+ * @throws DMLUnsupportedOperationException
+ * @throws DMLRuntimeException
+ */
+ public static void performShift(IndexedMatrixValue in, IndexRange ixrange, int brlen, int bclen, long rlen, long clen, ArrayList<IndexedMatrixValue> outlist)
+ throws DMLUnsupportedOperationException, DMLRuntimeException
+ {
+ MatrixIndexes ix = in.getIndexes();
+ MatrixBlock mb = (MatrixBlock)in.getValue();
+
+ long start_lhs_globalRowIndex = ixrange.rowStart + (ix.getRowIndex()-1)*brlen;
+ long start_lhs_globalColIndex = ixrange.colStart + (ix.getColumnIndex()-1)*bclen;
+ long end_lhs_globalRowIndex = start_lhs_globalRowIndex + mb.getNumRows() - 1;
+ long end_lhs_globalColIndex = start_lhs_globalColIndex + mb.getNumColumns() - 1;
+
+ long start_lhs_rowIndex = UtilFunctions.computeBlockIndex(start_lhs_globalRowIndex, brlen);
+ long end_lhs_rowIndex = UtilFunctions.computeBlockIndex(end_lhs_globalRowIndex, brlen);
+ long start_lhs_colIndex = UtilFunctions.computeBlockIndex(start_lhs_globalColIndex, bclen);
+ long end_lhs_colIndex = UtilFunctions.computeBlockIndex(end_lhs_globalColIndex, bclen);
+
+ for(long leftRowIndex = start_lhs_rowIndex; leftRowIndex <= end_lhs_rowIndex; leftRowIndex++) {
+ for(long leftColIndex = start_lhs_colIndex; leftColIndex <= end_lhs_colIndex; leftColIndex++) {
+
+ // Calculate global index of right hand side block
+ long lhs_rl = Math.max((leftRowIndex-1)*brlen+1, start_lhs_globalRowIndex);
+ long lhs_ru = Math.min(leftRowIndex*brlen, end_lhs_globalRowIndex);
+ long lhs_cl = Math.max((leftColIndex-1)*bclen+1, start_lhs_globalColIndex);
+ long lhs_cu = Math.min(leftColIndex*bclen, end_lhs_globalColIndex);
+
+ int lhs_lrl = UtilFunctions.computeCellInBlock(lhs_rl, brlen);
+ int lhs_lru = UtilFunctions.computeCellInBlock(lhs_ru, brlen);
+ int lhs_lcl = UtilFunctions.computeCellInBlock(lhs_cl, bclen);
+ int lhs_lcu = UtilFunctions.computeCellInBlock(lhs_cu, bclen);
+
+ long rhs_rl = lhs_rl - ixrange.rowStart + 1;
+ long rhs_ru = rhs_rl + (lhs_ru - lhs_rl);
+ long rhs_cl = lhs_cl - ixrange.colStart + 1;
+ long rhs_cu = rhs_cl + (lhs_cu - lhs_cl);
+
+ int rhs_lrl = UtilFunctions.computeCellInBlock(rhs_rl, brlen);
+ int rhs_lru = UtilFunctions.computeCellInBlock(rhs_ru, brlen);
+ int rhs_lcl = UtilFunctions.computeCellInBlock(rhs_cl, bclen);
+ int rhs_lcu = UtilFunctions.computeCellInBlock(rhs_cu, bclen);
+
+ MatrixBlock slicedRHSBlk = mb.sliceOperations(rhs_lrl, rhs_lru, rhs_lcl, rhs_lcu, new MatrixBlock());
+
+ int lbrlen = UtilFunctions.computeBlockSize(rlen, leftRowIndex, brlen);
+ int lbclen = UtilFunctions.computeBlockSize(clen, leftColIndex, bclen);
+ MatrixBlock resultBlock = new MatrixBlock(lbrlen, lbclen, false);
+ resultBlock = resultBlock.leftIndexingOperations(slicedRHSBlk, lhs_lrl, lhs_lru, lhs_lcl, lhs_lcu, null, false);
+ outlist.add(new IndexedMatrixValue(new MatrixIndexes(leftRowIndex, leftColIndex), resultBlock));
+ }
+ }
+ }
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a99fa6ea/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingScalarTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingScalarTest.java b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingScalarTest.java
index 8710d09..d70d2c8 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingScalarTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingScalarTest.java
@@ -22,10 +22,10 @@ package org.apache.sysml.test.integration.functions.indexing;
import java.util.HashMap;
import org.junit.Test;
-
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
import org.apache.sysml.test.integration.AutomatedTestBase;
import org.apache.sysml.test.integration.TestConfiguration;
@@ -106,6 +106,7 @@ public class LeftIndexingScalarTest extends AutomatedTestBase
HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("A");
HashMap<CellIndex, Double> rfile = readRMatrixFromFS("A");
TestUtils.compareMatrices(dmlfile, rfile, epsilon, "A-DML", "A-R");
+ checkDMLMetaDataFile("A", new MatrixCharacteristics(rows,cols,1,1));
}
finally
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a99fa6ea/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseDenseTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseDenseTest.java b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseDenseTest.java
index 778b649..b55b737 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseDenseTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseDenseTest.java
@@ -24,12 +24,12 @@ import java.util.HashMap;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
import org.apache.sysml.hops.LeftIndexingOp;
import org.apache.sysml.hops.LeftIndexingOp.LeftIndexingMethod;
import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
import org.apache.sysml.test.integration.AutomatedTestBase;
import org.apache.sysml.test.integration.TestConfiguration;
@@ -226,6 +226,7 @@ public class LeftIndexingSparseDenseTest extends AutomatedTestBase
HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
TestUtils.compareMatrices(dmlfile, rfile, 0, "DML", "R");
+ checkDMLMetaDataFile("R", new MatrixCharacteristics(rows1,cols1,1,1));
}
finally {
rtplatform = oldRTP;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/a99fa6ea/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseSparseTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseSparseTest.java b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseSparseTest.java
index e458c8a..80b941d 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseSparseTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/indexing/LeftIndexingSparseSparseTest.java
@@ -24,12 +24,12 @@ import java.util.HashMap;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
import org.apache.sysml.hops.LeftIndexingOp;
import org.apache.sysml.hops.LeftIndexingOp.LeftIndexingMethod;
import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
import org.apache.sysml.test.integration.AutomatedTestBase;
import org.apache.sysml.test.integration.TestConfiguration;
@@ -233,6 +233,7 @@ public class LeftIndexingSparseSparseTest extends AutomatedTestBase
HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
TestUtils.compareMatrices(dmlfile, rfile, 0, "DML", "R");
+ checkDMLMetaDataFile("R", new MatrixCharacteristics(rows1,cols1,1,1));
}
}