You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/06/08 04:43:22 UTC
[1/2] systemml git commit: [SYSTEMML-2371] New nary min/max
operations (language, CP dense/sparse)
Repository: systemml
Updated Branches:
refs/heads/master 15971f797 -> 5f580f02e
[SYSTEMML-2371] New nary min/max operations (language, CP dense/sparse)
This patch introduces new nary min/max operations to avoid unnecessary
nesting of binary operations at script level. As a by-product this also
significantly improves performance if codegen is not enabled.
On a scenario of 10 iterations of min(X, Y, Z, 3, 7) this patch improved
performance compared to the equivalent formulation with binary
operations as follows:
10 x 100K x 1K, dense: 45.1s -> 27.2
10 x 100K x 1K, sparse 0.1: 113.7s -> 8s.
Furthermore, this also includes a refactoring of the existing nary
cbind/rbind operations into a separate package.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/231a3648
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/231a3648
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/231a3648
Branch: refs/heads/master
Commit: 231a36489685d8057f6129ed7ca25dba46f999fe
Parents: 15971f7
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Jun 7 12:04:22 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Jun 7 20:30:35 2018 -0700
----------------------------------------------------------------------
src/main/java/org/apache/sysml/hops/Hop.java | 4 +-
src/main/java/org/apache/sysml/hops/NaryOp.java | 5 +
src/main/java/org/apache/sysml/lops/Nary.java | 6 +-
.../sysml/parser/BuiltinFunctionExpression.java | 37 +++-
.../org/apache/sysml/parser/DMLTranslator.java | 13 +-
.../org/apache/sysml/parser/Expression.java | 11 +-
.../instructions/CPInstructionParser.java | 4 +-
.../cp/BuiltinNaryCPInstruction.java | 5 +
.../cp/MatrixBuiltinNaryCPInstruction.java | 48 ++++-
.../sysml/runtime/matrix/data/MatrixBlock.java | 100 ++++++++++
.../functions/append/NaryCBindTest.java | 192 -------------------
.../functions/append/NaryRBindTest.java | 192 -------------------
.../functions/nary/NaryCBindTest.java | 192 +++++++++++++++++++
.../functions/nary/NaryMinMaxTest.java | 123 ++++++++++++
.../functions/nary/NaryRBindTest.java | 192 +++++++++++++++++++
src/test/scripts/functions/append/NaryCbind.R | 32 ----
src/test/scripts/functions/append/NaryCbind.dml | 26 ---
src/test/scripts/functions/append/NaryRbind.R | 32 ----
src/test/scripts/functions/append/NaryRbind.dml | 26 ---
src/test/scripts/functions/nary/NaryCbind.R | 32 ++++
src/test/scripts/functions/nary/NaryCbind.dml | 26 +++
src/test/scripts/functions/nary/NaryMinMax.R | 37 ++++
src/test/scripts/functions/nary/NaryMinMax.dml | 31 +++
src/test/scripts/functions/nary/NaryRbind.R | 32 ++++
src/test/scripts/functions/nary/NaryRbind.dml | 26 +++
.../functions/append/ZPackageSuite.java | 2 -
.../functions/nary/ZPackageSuite.java | 38 ++++
27 files changed, 933 insertions(+), 531 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index 5d789df..bfff50a 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -1089,7 +1089,7 @@ public abstract class Hop implements ParseInfo
// Operations that require a variable number of operands
public enum OpOpN {
- PRINTF, CBIND, RBIND, EVAL, LIST
+ PRINTF, CBIND, RBIND, MIN, MAX, EVAL, LIST
}
public enum AggOp {
@@ -1375,6 +1375,8 @@ public abstract class Hop implements ParseInfo
HopsOpOpNLops.put(OpOpN.PRINTF, Nary.OperationType.PRINTF);
HopsOpOpNLops.put(OpOpN.CBIND, Nary.OperationType.CBIND);
HopsOpOpNLops.put(OpOpN.RBIND, Nary.OperationType.RBIND);
+ HopsOpOpNLops.put(OpOpN.MIN, Nary.OperationType.MIN);
+ HopsOpOpNLops.put(OpOpN.MAX, Nary.OperationType.MAX);
HopsOpOpNLops.put(OpOpN.EVAL, Nary.OperationType.EVAL);
HopsOpOpNLops.put(OpOpN.LIST, Nary.OperationType.LIST);
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/main/java/org/apache/sysml/hops/NaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/NaryOp.java b/src/main/java/org/apache/sysml/hops/NaryOp.java
index 8927087..0847659 100644
--- a/src/main/java/org/apache/sysml/hops/NaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/NaryOp.java
@@ -195,6 +195,11 @@ public class NaryOp extends Hop {
setDim1(HopRewriteUtils.getSumValidInputDims(this, true));
setDim2(HopRewriteUtils.getMaxInputDim(this, false));
break;
+ case MIN:
+ case MAX:
+ setDim1(getDataType().isScalar() ? 0 : HopRewriteUtils.getMaxInputDim(this, true));
+ setDim2(getDataType().isScalar() ? 0 : HopRewriteUtils.getMaxInputDim(this, false));
+ break;
case LIST:
setDim1(getInput().size());
setDim2(1);
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/main/java/org/apache/sysml/lops/Nary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Nary.java b/src/main/java/org/apache/sysml/lops/Nary.java
index f7d5952..568b1e7 100644
--- a/src/main/java/org/apache/sysml/lops/Nary.java
+++ b/src/main/java/org/apache/sysml/lops/Nary.java
@@ -32,7 +32,7 @@ import org.apache.sysml.parser.Expression.ValueType;
public class Nary extends Lop {
public enum OperationType {
- PRINTF, CBIND, RBIND, EVAL, LIST
+ PRINTF, CBIND, RBIND, MIN, MAX, EVAL, LIST
}
private OperationType operationType;
@@ -121,6 +121,10 @@ public class Nary extends Lop {
case EVAL:
case LIST:
return operationType.name().toLowerCase();
+ case MIN:
+ case MAX:
+ //need to differentiate from binary min/max operations
+ return "n"+operationType.name().toLowerCase();
default:
throw new UnsupportedOperationException(
"Nary operation type (" + operationType + ") is not defined.");
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
index eb9e2c3..d38f445 100644
--- a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
+++ b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java
@@ -20,6 +20,7 @@
package org.apache.sysml.parser;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -566,11 +567,14 @@ public class BuiltinFunctionExpression extends DataIdentifier
case BITWXOR:
case BITWSHIFTL:
case BITWSHIFTR:
+ checkNumParameters(2);
+ setBinaryOutputProperties(output);
+ break;
+
case MIN:
case MAX:
//min(X), min(X,s), min(s,X), min(s,r), min(X,Y)
-
- //unary aggregate
+ //unary
if (getSecondExpr() == null) {
checkNumParameters(1);
checkMatrixParam(getFirstExpr());
@@ -579,12 +583,13 @@ public class BuiltinFunctionExpression extends DataIdentifier
output.setDimensions(0, 0);
output.setBlockDimensions (0, 0);
}
- //binary operation
+
+ //nary operation
else {
- checkNumParameters(2);
- setBinaryOutputProperties(output);
+ for( Expression e : getAllExpr() )
+ checkMatrixScalarParam(e);
+ setNaryOutputProperties(output);
}
-
break;
case CUMSUM:
@@ -1420,7 +1425,7 @@ public class BuiltinFunctionExpression extends DataIdentifier
DataType dt1 = getFirstExpr().getOutput().getDataType();
DataType dt2 = getSecondExpr().getOutput().getDataType();
DataType dtOut = (dt1==DataType.MATRIX || dt2==DataType.MATRIX) ?
- DataType.MATRIX : DataType.SCALAR;
+ DataType.MATRIX : DataType.SCALAR;
if( dt1==DataType.MATRIX && dt2==DataType.MATRIX )
checkMatchingDimensions(getFirstExpr(), getSecondExpr(), true);
MatrixCharacteristics dims = getBinaryMatrixCharacteristics(getFirstExpr(), getSecondExpr());
@@ -1453,6 +1458,24 @@ public class BuiltinFunctionExpression extends DataIdentifier
Math.max(dims1.getColsPerBlock(), dims2.getColsPerBlock()));
}
+ private void setNaryOutputProperties(DataIdentifier output) {
+ DataType dt = Arrays.stream(getAllExpr()).allMatch(
+ e -> e.getOutput().getDataType().isScalar()) ? DataType.SCALAR : DataType.MATRIX;
+ Expression firstM = dt.isMatrix() ? Arrays.stream(getAllExpr()).filter(
+ e -> e.getOutput().getDataType().isMatrix()).findFirst().get() : null;
+ ValueType vt = dt.isMatrix() ? ValueType.DOUBLE : ValueType.BOOLEAN;
+ for( Expression e : getAllExpr() ) {
+ vt = computeValueType(e, e.getOutput().getValueType(), vt, true);
+ if( e.getOutput().getDataType().isMatrix() )
+ checkMatchingDimensions(firstM, e, true);
+ }
+ output.setDataType(dt);
+ output.setValueType(vt);
+ output.setDimensions(firstM.getOutput().getDim1(), firstM.getOutput().getDim2());
+ output.setBlockDimensions (
+ firstM.getOutput().getRowsInBlock(), firstM.getOutput().getColumnsInBlock());
+ }
+
private void expandArguments() {
if ( _args == null ) {
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/main/java/org/apache/sysml/parser/DMLTranslator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
index 3dce294..2349b56 100644
--- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java
+++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
@@ -2408,10 +2408,15 @@ public class DMLTranslator
case MIN:
case MAX:
- //construct AggUnary for min(X) but BinaryOp for min(X,Y)
- currBuiltinOp = (expr2 == null) ? new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(),
- AggOp.valueOf(source.getOpCode().name()), Direction.RowCol, expr) : new BinaryOp(target.getName(),
- target.getDataType(), target.getValueType(), OpOp2.valueOf(source.getOpCode().name()), expr, expr2);
+ //construct AggUnary for min(X) but BinaryOp for min(X,Y) and NaryOp for min(X,Y,Z)
+ currBuiltinOp = (expr2 == null) ?
+ new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(),
+ AggOp.valueOf(source.getOpCode().name()), Direction.RowCol, expr) :
+ (source.getAllExpr().length == 2) ?
+ new BinaryOp(target.getName(), target.getDataType(), target.getValueType(),
+ OpOp2.valueOf(source.getOpCode().name()), expr, expr2) :
+ new NaryOp(target.getName(), target.getDataType(), target.getValueType(),
+ OpOpN.valueOf(source.getOpCode().name()), processAllExpressions(source.getAllExpr(), hops));
break;
case PPRED:
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/main/java/org/apache/sysml/parser/Expression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/Expression.java b/src/main/java/org/apache/sysml/parser/Expression.java
index 4355bd5..2808609 100644
--- a/src/main/java/org/apache/sysml/parser/Expression.java
+++ b/src/main/java/org/apache/sysml/parser/Expression.java
@@ -453,9 +453,10 @@ public abstract class Expression implements ParseInfo
* @return The value type ({@link ValueType})
*/
public static ValueType computeValueType(Identifier identifier1, Identifier identifier2, boolean cast) {
- ValueType v1 = identifier1.getValueType();
- ValueType v2 = identifier2.getValueType();
-
+ return computeValueType(identifier1, identifier1.getValueType(), identifier2.getValueType(), cast);
+ }
+
+ public static ValueType computeValueType(Expression expr1, ValueType v1, ValueType v2, boolean cast) {
if (v1 == v2)
return v1;
@@ -472,8 +473,8 @@ public abstract class Expression implements ParseInfo
}
//raise error with id1 location
- identifier1.raiseValidateError("Invalid Valuetypes for operation "+v1+" "+v2, false,
- LanguageException.LanguageErrorCodes.INVALID_PARAMETERS);
+ expr1.raiseValidateError("Invalid Valuetypes for operation "+v1+" "+v2, false,
+ LanguageException.LanguageErrorCodes.INVALID_PARAMETERS);
return null; //never reached because unconditional
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
index ece9d65..f01d3ae 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java
@@ -146,9 +146,11 @@ public class CPInstructionParser extends InstructionParser
String2CPInstructionType.put( "log" , CPType.Builtin);
String2CPInstructionType.put( "log_nz" , CPType.Builtin);
+ String2CPInstructionType.put( "solve" , CPType.Binary);
String2CPInstructionType.put( "max" , CPType.Binary);
String2CPInstructionType.put( "min" , CPType.Binary);
- String2CPInstructionType.put( "solve" , CPType.Binary);
+ String2CPInstructionType.put( "nmax" , CPType.BuiltinNary);
+ String2CPInstructionType.put( "nmin" , CPType.BuiltinNary);
String2CPInstructionType.put( "exp" , CPType.Unary);
String2CPInstructionType.put( "abs" , CPType.Unary);
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinNaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinNaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinNaryCPInstruction.java
index eac9c32..8077575 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinNaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinNaryCPInstruction.java
@@ -65,6 +65,11 @@ public abstract class BuiltinNaryCPInstruction extends CPInstruction
}
else if( opcode.equals("cbind") || opcode.equals("rbind") ) {
return new MatrixBuiltinNaryCPInstruction(null,
+ opcode, str, outputOperand, inputOperands);
+ }
+ else if( opcode.equals("nmin") || opcode.equals("nmax") ) {
+ ValueFunction func = Builtin.getBuiltinFnObject(opcode.substring(1));
+ return new MatrixBuiltinNaryCPInstruction(new SimpleOperator(func),
opcode, str, outputOperand, inputOperands);
}
else if (Nary.OperationType.EVAL.name().equalsIgnoreCase(opcode)) {
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinNaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinNaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinNaryCPInstruction.java
index 3f16447..50e3721 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinNaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinNaryCPInstruction.java
@@ -19,6 +19,10 @@
package org.apache.sysml.runtime.instructions.cp;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -32,24 +36,48 @@ public class MatrixBuiltinNaryCPInstruction extends BuiltinNaryCPInstruction {
@Override
public void processInstruction(ExecutionContext ec) {
- //pin input matrix blocks
- MatrixBlock in1 = ec.getMatrixInput(inputs[0].getName());
- MatrixBlock[] in2 = new MatrixBlock[inputs.length-1];
- for( int i=1; i<inputs.length; i++ )
- in2[i-1] = ec.getMatrixInput(inputs[i].getName());
+ //separate scalars and matrices and pin all input matrices
+ List<MatrixBlock> matrices = getMatrices(ec);
+ List<ScalarObject> scalars = getScalars(ec);
MatrixBlock outBlock = null;
if( "cbind".equals(getOpcode()) || "rbind".equals(getOpcode()) ) {
boolean cbind = "cbind".equals(getOpcode());
- outBlock = in1.append(in2, new MatrixBlock(), cbind);
+ outBlock = matrices.get(0).append(matrices.subList(1, matrices.size())
+ .toArray(new MatrixBlock[0]), new MatrixBlock(), cbind);
+ }
+
+ else if( "nmin".equals(getOpcode()) || "nmax".equals(getOpcode()) ) {
+ outBlock = MatrixBlock.naryOperations(_optr, matrices.toArray(new MatrixBlock[0]),
+ scalars.toArray(new ScalarObject[0]), new MatrixBlock());
}
else {
throw new DMLRuntimeException("Unknown opcode: "+getOpcode());
}
- //release inputs and set output
- for( int i=0; i<inputs.length; i++ )
- ec.releaseMatrixInput(inputs[i].getName());
- ec.setMatrixOutput(output.getName(), outBlock);
+ //release inputs and set output matrix or scalar
+ releaseInputs(ec);
+ if( output.getDataType().isMatrix() ) {
+ ec.setMatrixOutput(output.getName(), outBlock);
+ }
+ else {
+ ec.setVariable(output.getName(), ScalarObjectFactory.createScalarObject(
+ output.getValueType(), outBlock.quickGetValue(0, 0)));
+ }
+ }
+
+ private List<MatrixBlock> getMatrices(ExecutionContext ec) {
+ return Arrays.stream(inputs).filter(in -> in.getDataType().isMatrix())
+ .map(in -> ec.getMatrixInput(in.getName())).collect(Collectors.toList());
+ }
+
+ private List<ScalarObject> getScalars(ExecutionContext ec) {
+ return Arrays.stream(inputs).filter(in -> in.getDataType().isScalar())
+ .map(in -> ec.getScalarInput(in)).collect(Collectors.toList());
+ }
+
+ private void releaseInputs(ExecutionContext ec) {
+ Arrays.stream(inputs).filter(in -> in.getDataType().isMatrix())
+ .forEach(in -> ec.releaseMatrixInput(in.getName()));
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index 78f4fde..755bca1 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -44,6 +44,7 @@ import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.caching.CacheBlock;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType;
import org.apache.sysml.runtime.functionobjects.Builtin;
+import org.apache.sysml.runtime.functionobjects.Builtin.BuiltinCode;
import org.apache.sysml.runtime.functionobjects.CM;
import org.apache.sysml.runtime.functionobjects.CTable;
import org.apache.sysml.runtime.functionobjects.DiagIndex;
@@ -83,6 +84,7 @@ import org.apache.sysml.runtime.matrix.operators.Operator;
import org.apache.sysml.runtime.matrix.operators.QuaternaryOperator;
import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
import org.apache.sysml.runtime.matrix.operators.TernaryOperator;
import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
import org.apache.sysml.runtime.util.DataConverter;
@@ -3458,6 +3460,104 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
return result;
}
+ public static MatrixBlock naryOperations(Operator op, MatrixBlock[] matrices, ScalarObject[] scalars, MatrixBlock ret) {
+ //note: currently only min and max supported and hence specialized implementation
+
+ //prepare operator
+ Builtin fn = (Builtin)((SimpleOperator)op).fn;
+
+ //process all scalars
+ double init = (fn.getBuiltinCode() == BuiltinCode.MIN) ?
+ Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY;
+ for( ScalarObject so : scalars )
+ init = fn.execute(init, so.getDoubleValue());
+
+ //compute output dimensions and estimate sparsity
+ final int m = matrices.length > 0 ? matrices[0].rlen : 1;
+ final int n = matrices.length > 0 ? matrices[0].clen : 1;
+ final long mn = (long) m * n;
+ final long nnz = (fn.getBuiltinCode()==BuiltinCode.MIN && init < 0)
+ || (fn.getBuiltinCode()==BuiltinCode.MAX && init > 0) ? mn :
+ Math.min(Arrays.stream(matrices).mapToLong(mb -> mb.nonZeros).sum(), mn);
+ boolean sp = evalSparseFormatInMemory(m, n, nnz);
+
+ //init result matrix
+ if( ret == null )
+ ret = new MatrixBlock(m, n, sp, nnz);
+ else
+ ret.reset(m, n, sp, nnz);
+
+ //main processing
+ if( matrices.length > 0 ) {
+ ret.allocateBlock();
+ int[] cnt = Arrays.stream(matrices).allMatch(mb ->
+ mb.sparse || mb.isEmpty()) ? new int[n] : null;
+ if( ret.isInSparseFormat() ) {
+ double[] tmp = new double[n];
+ for(int i = 0; i < m; i++) {
+ //reset tmp and compute row output
+ Arrays.fill(tmp, 0);
+ processMinMaxRow(fn, matrices, init, tmp, 0, n, i, cnt);
+ //copy to sparse output
+ for(int j = 0; j < n; j++)
+ if( tmp[j] != 0 )
+ ret.appendValue(i, j, tmp[j]);
+ }
+ }
+ else {
+ DenseBlock c = ret.getDenseBlock();
+ for(int i = 0; i < m; i++) {
+ processMinMaxRow(fn, matrices, init,
+ c.values(i), c.pos(i), n, i, cnt);
+ }
+ }
+ }
+ else {
+ ret.quickSetValue(0, 0, init);
+ }
+
+ return ret;
+ }
+
+ private static void processMinMaxRow(Builtin fn, MatrixBlock[] inputs, double init, double[] c, int cix, int n, int i, int[] cnt) {
+ //always init entire output vector
+ Arrays.fill(c, cix, cix+n, init);
+ if( cnt != null )
+ Arrays.fill(cnt, 0);
+
+ //sparse-safe update over all input matrices
+ for( MatrixBlock in : inputs ) {
+ if( in.isEmptyBlock(false) )
+ continue;
+ if( in.isInSparseFormat() ) {
+ SparseBlock a = in.sparseBlock;
+ int alen = a.size(i);
+ int apos = a.pos(i);
+ int[] aix = a.indexes(i);
+ double[] avals = a.values(i);
+ for( int k=apos; k<apos+alen; k++ ) {
+ c[aix[k]] = fn.execute(c[aix[k]], avals[k]);
+ if( cnt != null ) //maintain seen values
+ cnt[aix[k]]++;
+ }
+ }
+ else {
+ double[] avals = in.getDenseBlock().values(i);
+ int aix = in.getDenseBlock().pos(i);
+ for( int j=0; j<n; j++ )
+ c[cix+j] = fn.execute(c[cix+j], avals[aix+j]);
+ }
+ }
+
+ //corrections for all sparse inputs
+ if( Arrays.stream(inputs).allMatch(m -> m.sparse || m.isEmpty()) ) {
+ for( int j=0; j<n; j++ ) {
+ if( cnt[j]!=inputs.length )
+ c[cix+j] = fn.execute(c[cix+j], 0);
+ }
+ }
+ }
+
public MatrixBlock transposeSelfMatrixMultOperations( MatrixBlock out, MMTSJType tstype ) {
return transposeSelfMatrixMultOperations(out, tstype, 1);
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/test/java/org/apache/sysml/test/integration/functions/append/NaryCBindTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/append/NaryCBindTest.java b/src/test/java/org/apache/sysml/test/integration/functions/append/NaryCBindTest.java
deleted file mode 100644
index 13ba799..0000000
--- a/src/test/java/org/apache/sysml/test/integration/functions/append/NaryCBindTest.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.test.integration.functions.append;
-
-import java.util.HashMap;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.sysml.api.DMLScript;
-import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
-import org.apache.sysml.lops.LopProperties.ExecType;
-import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
-import org.apache.sysml.test.integration.AutomatedTestBase;
-import org.apache.sysml.test.integration.TestConfiguration;
-import org.apache.sysml.test.utils.TestUtils;
-
-public class NaryCBindTest extends AutomatedTestBase
-{
- private final static String TEST_NAME = "NaryCbind";
- private final static String TEST_DIR = "functions/append/";
- private final static String TEST_CLASS_DIR = TEST_DIR + NaryCBindTest.class.getSimpleName() + "/";
-
- private final static double epsilon=0.0000000001;
-
- private final static int rows = 1101;
- private final static int cols1 = 101;
- private final static int cols2 = 79;
- private final static int cols3 = 123;
-
- private final static double sparsity1 = 0.7;
- private final static double sparsity2 = 0.07;
-
- @Override
- public void setUp() {
- TestUtils.clearAssertionInformation();
- addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"R"}));
- }
-
- @Test
- public void testNaryCbindDenseDenseDenseCP() {
- runCbindTest(false, false, false, ExecType.CP);
- }
-
- @Test
- public void testNaryCbindDenseDenseSparseCP() {
- runCbindTest(false, false, true, ExecType.CP);
- }
-
- @Test
- public void testNaryCbindDenseSparseDenseCP() {
- runCbindTest(false, true, false, ExecType.CP);
- }
-
- @Test
- public void testNaryCbindDenseSparseSparseCP() {
- runCbindTest(false, true, true, ExecType.CP);
- }
-
- @Test
- public void testNaryCbindSparseDenseDenseCP() {
- runCbindTest(true, false, false, ExecType.CP);
- }
-
- @Test
- public void testNaryCbindSparseDenseSparseCP() {
- runCbindTest(true, false, true, ExecType.CP);
- }
-
- @Test
- public void testNaryCbindSparseSparseDenseCP() {
- runCbindTest(true, true, false, ExecType.CP);
- }
-
- @Test
- public void testNaryCbindSparseSparseSparseCP() {
- runCbindTest(true, true, true, ExecType.CP);
- }
-
- @Test
- public void testNaryCbindDenseDenseDenseSP() {
- runCbindTest(false, false, false, ExecType.SPARK);
- }
-
- @Test
- public void testNaryCbindDenseDenseSparseSP() {
- runCbindTest(false, false, true, ExecType.SPARK);
- }
-
- @Test
- public void testNaryCbindDenseSparseDenseSP() {
- runCbindTest(false, true, false, ExecType.SPARK);
- }
-
- @Test
- public void testNaryCbindDenseSparseSparseSP() {
- runCbindTest(false, true, true, ExecType.SPARK);
- }
-
- @Test
- public void testNaryCbindSparseDenseDenseSP() {
- runCbindTest(true, false, false, ExecType.SPARK);
- }
-
- @Test
- public void testNaryCbindSparseDenseSparseSP() {
- runCbindTest(true, false, true, ExecType.SPARK);
- }
-
- @Test
- public void testNaryCbindSparseSparseDenseSP() {
- runCbindTest(true, true, false, ExecType.SPARK);
- }
-
- @Test
- public void testNaryCbindSparseSparseSparseSP() {
- runCbindTest(true, true, true, ExecType.CP);
- }
-
-
- public void runCbindTest(boolean sparse1, boolean sparse2, boolean sparse3, ExecType et)
- {
- RUNTIME_PLATFORM platformOld = rtplatform;
- switch( et ) {
- case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break;
- case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
- default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break;
- }
-
- boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
- if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK )
- DMLScript.USE_LOCAL_SPARK_CONFIG = true;
-
- try
- {
- TestConfiguration config = getAndLoadTestConfiguration(TEST_NAME);
- loadTestConfiguration(config);
-
- String RI_HOME = SCRIPT_DIR + TEST_DIR;
- fullDMLScriptName = RI_HOME + TEST_NAME + ".dml";
- programArgs = new String[]{"-stats", "-args", input("A"),
- input("B"), input("C"), output("R") };
- fullRScriptName = RI_HOME + TEST_NAME + ".R";
- rCmd = "Rscript" + " " + fullRScriptName + " " +
- inputDir() + " "+ expectedDir();
-
- //generate input data
- double sp1 = sparse1 ? sparsity2 : sparsity1;
- double sp2 = sparse2 ? sparsity2 : sparsity1;
- double sp3 = sparse3 ? sparsity2 : sparsity1;
- double[][] A = getRandomMatrix(rows, cols1, -1, 1, sp1, 711);
- double[][] B = getRandomMatrix(rows, cols2, -1, 1, sp2, 722);
- double[][] C = getRandomMatrix(rows, cols3, -1, 1, sp3, 733);
- writeInputMatrixWithMTD("A", A, true);
- writeInputMatrixWithMTD("B", B, true);
- writeInputMatrixWithMTD("C", C, true);
-
- //run tests
- runTest(true, false, null, -1);
- runRScript(true);
-
- //compare result data
- HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
- HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
- TestUtils.compareMatrices(dmlfile, rfile, epsilon, "DML", "R");
-
- //check for spark instructions
- Assert.assertTrue(heavyHittersContainsSubString("sp_cbind")==(et==ExecType.SPARK));
- }
- finally {
- rtplatform = platformOld;
- DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/test/java/org/apache/sysml/test/integration/functions/append/NaryRBindTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/append/NaryRBindTest.java b/src/test/java/org/apache/sysml/test/integration/functions/append/NaryRBindTest.java
deleted file mode 100644
index f820bf5..0000000
--- a/src/test/java/org/apache/sysml/test/integration/functions/append/NaryRBindTest.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.test.integration.functions.append;
-
-import java.util.HashMap;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.sysml.api.DMLScript;
-import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
-import org.apache.sysml.lops.LopProperties.ExecType;
-import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
-import org.apache.sysml.test.integration.AutomatedTestBase;
-import org.apache.sysml.test.integration.TestConfiguration;
-import org.apache.sysml.test.utils.TestUtils;
-
-public class NaryRBindTest extends AutomatedTestBase
-{
- private final static String TEST_NAME = "NaryRbind";
- private final static String TEST_DIR = "functions/append/";
- private final static String TEST_CLASS_DIR = TEST_DIR + NaryRBindTest.class.getSimpleName() + "/";
-
- private final static double epsilon=0.0000000001;
-
- private final static int cols = 101;
- private final static int rows1 = 1101;
- private final static int rows2 = 1179;
- private final static int rows3 = 1123;
-
- private final static double sparsity1 = 0.7;
- private final static double sparsity2 = 0.07;
-
- @Override
- public void setUp() {
- TestUtils.clearAssertionInformation();
- addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"R"}));
- }
-
- @Test
- public void testNaryRbindDenseDenseDenseCP() {
- runRbindTest(false, false, false, ExecType.CP);
- }
-
- @Test
- public void testNaryRbindDenseDenseSparseCP() {
- runRbindTest(false, false, true, ExecType.CP);
- }
-
- @Test
- public void testNaryRbindDenseSparseDenseCP() {
- runRbindTest(false, true, false, ExecType.CP);
- }
-
- @Test
- public void testNaryRbindDenseSparseSparseCP() {
- runRbindTest(false, true, true, ExecType.CP);
- }
-
- @Test
- public void testNaryRbindSparseDenseDenseCP() {
- runRbindTest(true, false, false, ExecType.CP);
- }
-
- @Test
- public void testNaryRbindSparseDenseSparseCP() {
- runRbindTest(true, false, true, ExecType.CP);
- }
-
- @Test
- public void testNaryRbindSparseSparseDenseCP() {
- runRbindTest(true, true, false, ExecType.CP);
- }
-
- @Test
- public void testNaryRbindSparseSparseSparseCP() {
- runRbindTest(true, true, true, ExecType.CP);
- }
-
- @Test
- public void testNaryRbindDenseDenseDenseSP() {
- runRbindTest(false, false, false, ExecType.SPARK);
- }
-
- @Test
- public void testNaryRbindDenseDenseSparseSP() {
- runRbindTest(false, false, true, ExecType.SPARK);
- }
-
- @Test
- public void testNaryRbindDenseSparseDenseSP() {
- runRbindTest(false, true, false, ExecType.SPARK);
- }
-
- @Test
- public void testNaryRbindDenseSparseSparseSP() {
- runRbindTest(false, true, true, ExecType.SPARK);
- }
-
- @Test
- public void testNaryRbindSparseDenseDenseSP() {
- runRbindTest(true, false, false, ExecType.SPARK);
- }
-
- @Test
- public void testNaryRbindSparseDenseSparseSP() {
- runRbindTest(true, false, true, ExecType.SPARK);
- }
-
- @Test
- public void testNaryRbindSparseSparseDenseSP() {
- runRbindTest(true, true, false, ExecType.SPARK);
- }
-
- @Test
- public void testNaryRbindSparseSparseSparseSP() {
- runRbindTest(true, true, true, ExecType.CP);
- }
-
-
- public void runRbindTest(boolean sparse1, boolean sparse2, boolean sparse3, ExecType et)
- {
- RUNTIME_PLATFORM platformOld = rtplatform;
- switch( et ) {
- case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break;
- case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
- default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break;
- }
-
- boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
- if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK )
- DMLScript.USE_LOCAL_SPARK_CONFIG = true;
-
- try
- {
- TestConfiguration config = getAndLoadTestConfiguration(TEST_NAME);
- loadTestConfiguration(config);
-
- String RI_HOME = SCRIPT_DIR + TEST_DIR;
- fullDMLScriptName = RI_HOME + TEST_NAME + ".dml";
- programArgs = new String[]{"-stats", "-args", input("A"),
- input("B"), input("C"), output("R") };
- fullRScriptName = RI_HOME + TEST_NAME + ".R";
- rCmd = "Rscript" + " " + fullRScriptName + " " +
- inputDir() + " "+ expectedDir();
-
- //generate input data
- double sp1 = sparse1 ? sparsity2 : sparsity1;
- double sp2 = sparse2 ? sparsity2 : sparsity1;
- double sp3 = sparse3 ? sparsity2 : sparsity1;
- double[][] A = getRandomMatrix(rows1, cols, -1, 1, sp1, 711);
- double[][] B = getRandomMatrix(rows2, cols, -1, 1, sp2, 722);
- double[][] C = getRandomMatrix(rows3, cols, -1, 1, sp3, 733);
- writeInputMatrixWithMTD("A", A, true);
- writeInputMatrixWithMTD("B", B, true);
- writeInputMatrixWithMTD("C", C, true);
-
- //run tests
- runTest(true, false, null, -1);
- runRScript(true);
-
- //compare result data
- HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
- HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
- TestUtils.compareMatrices(dmlfile, rfile, epsilon, "DML", "R");
-
- //check for spark instructions
- Assert.assertTrue(heavyHittersContainsSubString("sp_rbind")==(et==ExecType.SPARK));
- }
- finally {
- rtplatform = platformOld;
- DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/test/java/org/apache/sysml/test/integration/functions/nary/NaryCBindTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/nary/NaryCBindTest.java b/src/test/java/org/apache/sysml/test/integration/functions/nary/NaryCBindTest.java
new file mode 100644
index 0000000..dc4812d
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/nary/NaryCBindTest.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.nary;
+
+import java.util.HashMap;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+
+public class NaryCBindTest extends AutomatedTestBase
+{
+ private final static String TEST_NAME = "NaryCbind";
+ private final static String TEST_DIR = "functions/nary/";
+ private final static String TEST_CLASS_DIR = TEST_DIR + NaryCBindTest.class.getSimpleName() + "/";
+
+ private final static double epsilon=0.0000000001;
+
+ private final static int rows = 1101;
+ private final static int cols1 = 101;
+ private final static int cols2 = 79;
+ private final static int cols3 = 123;
+
+ private final static double sparsity1 = 0.7;
+ private final static double sparsity2 = 0.07;
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"R"}));
+ }
+
+ @Test
+ public void testNaryCbindDenseDenseDenseCP() {
+ runCbindTest(false, false, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryCbindDenseDenseSparseCP() {
+ runCbindTest(false, false, true, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryCbindDenseSparseDenseCP() {
+ runCbindTest(false, true, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryCbindDenseSparseSparseCP() {
+ runCbindTest(false, true, true, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryCbindSparseDenseDenseCP() {
+ runCbindTest(true, false, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryCbindSparseDenseSparseCP() {
+ runCbindTest(true, false, true, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryCbindSparseSparseDenseCP() {
+ runCbindTest(true, true, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryCbindSparseSparseSparseCP() {
+ runCbindTest(true, true, true, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryCbindDenseDenseDenseSP() {
+ runCbindTest(false, false, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryCbindDenseDenseSparseSP() {
+ runCbindTest(false, false, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryCbindDenseSparseDenseSP() {
+ runCbindTest(false, true, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryCbindDenseSparseSparseSP() {
+ runCbindTest(false, true, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryCbindSparseDenseDenseSP() {
+ runCbindTest(true, false, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryCbindSparseDenseSparseSP() {
+ runCbindTest(true, false, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryCbindSparseSparseDenseSP() {
+ runCbindTest(true, true, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryCbindSparseSparseSparseSP() {
+ runCbindTest(true, true, true, ExecType.CP);
+ }
+
+
+ public void runCbindTest(boolean sparse1, boolean sparse2, boolean sparse3, ExecType et)
+ {
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch( et ) {
+ case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break;
+ case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
+ default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break;
+ }
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK )
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try
+ {
+ TestConfiguration config = getAndLoadTestConfiguration(TEST_NAME);
+ loadTestConfiguration(config);
+
+ String RI_HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = RI_HOME + TEST_NAME + ".dml";
+ programArgs = new String[]{"-stats", "-args", input("A"),
+ input("B"), input("C"), output("R") };
+ fullRScriptName = RI_HOME + TEST_NAME + ".R";
+ rCmd = "Rscript" + " " + fullRScriptName + " " +
+ inputDir() + " "+ expectedDir();
+
+ //generate input data
+ double sp1 = sparse1 ? sparsity2 : sparsity1;
+ double sp2 = sparse2 ? sparsity2 : sparsity1;
+ double sp3 = sparse3 ? sparsity2 : sparsity1;
+ double[][] A = getRandomMatrix(rows, cols1, -1, 1, sp1, 711);
+ double[][] B = getRandomMatrix(rows, cols2, -1, 1, sp2, 722);
+ double[][] C = getRandomMatrix(rows, cols3, -1, 1, sp3, 733);
+ writeInputMatrixWithMTD("A", A, true);
+ writeInputMatrixWithMTD("B", B, true);
+ writeInputMatrixWithMTD("C", C, true);
+
+ //run tests
+ runTest(true, false, null, -1);
+ runRScript(true);
+
+ //compare result data
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
+ HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
+ TestUtils.compareMatrices(dmlfile, rfile, epsilon, "DML", "R");
+
+ //check for spark instructions
+ Assert.assertTrue(heavyHittersContainsSubString("sp_cbind")==(et==ExecType.SPARK));
+ }
+ finally {
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/test/java/org/apache/sysml/test/integration/functions/nary/NaryMinMaxTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/nary/NaryMinMaxTest.java b/src/test/java/org/apache/sysml/test/integration/functions/nary/NaryMinMaxTest.java
new file mode 100644
index 0000000..c5a026a
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/nary/NaryMinMaxTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.nary;
+
+import java.util.HashMap;
+
+import org.junit.Test;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+
+public class NaryMinMaxTest extends AutomatedTestBase
+{
+ private final static String TEST_NAME = "NaryMinMax";
+ private final static String TEST_DIR = "functions/nary/";
+ private final static String TEST_CLASS_DIR = TEST_DIR + NaryMinMaxTest.class.getSimpleName() + "/";
+
+ private final static double epsilon=0.0000000001;
+
+ private final static int rows = 2101;
+ private final static int cols = 110;
+
+ private final static double sparsity1 = 0.7;
+ private final static double sparsity2 = 0.07;
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"R"}));
+ }
+
+ @Test
+ public void testNaryMinDenseCP() {
+ runMinMaxTest(true, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryMinSparseCP() {
+ runMinMaxTest(true, true, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryMaxDenseCP() {
+ runMinMaxTest(false, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryMaxSparseCP() {
+ runMinMaxTest(false, true, ExecType.CP);
+ }
+
+ public void runMinMaxTest(boolean min, boolean sparse, ExecType et)
+ {
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch( et ) {
+ case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break;
+ case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
+ default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break;
+ }
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK )
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try
+ {
+ TestConfiguration config = getAndLoadTestConfiguration(TEST_NAME);
+ loadTestConfiguration(config);
+
+ String RI_HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = RI_HOME + TEST_NAME + ".dml";
+ programArgs = new String[]{"-stats", "-args", input("A"),
+ input("B"), input("C"), String.valueOf(min).toUpperCase(), output("R") };
+ fullRScriptName = RI_HOME + TEST_NAME + ".R";
+ rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " "
+ + String.valueOf(min).toUpperCase() + " "+ expectedDir();
+
+ //generate input data
+ double sp = sparse ? sparsity2 : sparsity1;
+ double[][] A = getRandomMatrix(rows, cols, -1, 1, sp, 3);
+ double[][] B = getRandomMatrix(rows, cols, -1, 1, sp, 7);
+ double[][] C = getRandomMatrix(rows, cols, -1, 1, sp, 10);
+ writeInputMatrixWithMTD("A", A, true);
+ writeInputMatrixWithMTD("B", B, true);
+ writeInputMatrixWithMTD("C", C, true);
+
+ //run tests
+ runTest(true, false, null, -1);
+ runRScript(true);
+
+ //compare result data
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
+ HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
+ TestUtils.compareMatrices(dmlfile, rfile, epsilon, "DML", "R");
+ }
+ finally {
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/test/java/org/apache/sysml/test/integration/functions/nary/NaryRBindTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/nary/NaryRBindTest.java b/src/test/java/org/apache/sysml/test/integration/functions/nary/NaryRBindTest.java
new file mode 100644
index 0000000..1e51ac9
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/nary/NaryRBindTest.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.nary;
+
+import java.util.HashMap;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+
+public class NaryRBindTest extends AutomatedTestBase
+{
+ private final static String TEST_NAME = "NaryRbind";
+ private final static String TEST_DIR = "functions/nary/";
+ private final static String TEST_CLASS_DIR = TEST_DIR + NaryRBindTest.class.getSimpleName() + "/";
+
+ private final static double epsilon=0.0000000001;
+
+ private final static int cols = 101;
+ private final static int rows1 = 1101;
+ private final static int rows2 = 1179;
+ private final static int rows3 = 1123;
+
+ private final static double sparsity1 = 0.7;
+ private final static double sparsity2 = 0.07;
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"R"}));
+ }
+
+ @Test
+ public void testNaryRbindDenseDenseDenseCP() {
+ runRbindTest(false, false, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryRbindDenseDenseSparseCP() {
+ runRbindTest(false, false, true, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryRbindDenseSparseDenseCP() {
+ runRbindTest(false, true, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryRbindDenseSparseSparseCP() {
+ runRbindTest(false, true, true, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryRbindSparseDenseDenseCP() {
+ runRbindTest(true, false, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryRbindSparseDenseSparseCP() {
+ runRbindTest(true, false, true, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryRbindSparseSparseDenseCP() {
+ runRbindTest(true, true, false, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryRbindSparseSparseSparseCP() {
+ runRbindTest(true, true, true, ExecType.CP);
+ }
+
+ @Test
+ public void testNaryRbindDenseDenseDenseSP() {
+ runRbindTest(false, false, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryRbindDenseDenseSparseSP() {
+ runRbindTest(false, false, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryRbindDenseSparseDenseSP() {
+ runRbindTest(false, true, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryRbindDenseSparseSparseSP() {
+ runRbindTest(false, true, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryRbindSparseDenseDenseSP() {
+ runRbindTest(true, false, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryRbindSparseDenseSparseSP() {
+ runRbindTest(true, false, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryRbindSparseSparseDenseSP() {
+ runRbindTest(true, true, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryRbindSparseSparseSparseSP() {
+ runRbindTest(true, true, true, ExecType.CP);
+ }
+
+
+ public void runRbindTest(boolean sparse1, boolean sparse2, boolean sparse3, ExecType et)
+ {
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch( et ) {
+ case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break;
+ case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
+ default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break;
+ }
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK )
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try
+ {
+ TestConfiguration config = getAndLoadTestConfiguration(TEST_NAME);
+ loadTestConfiguration(config);
+
+ String RI_HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = RI_HOME + TEST_NAME + ".dml";
+ programArgs = new String[]{"-stats", "-args", input("A"),
+ input("B"), input("C"), output("R") };
+ fullRScriptName = RI_HOME + TEST_NAME + ".R";
+ rCmd = "Rscript" + " " + fullRScriptName + " " +
+ inputDir() + " "+ expectedDir();
+
+ //generate input data
+ double sp1 = sparse1 ? sparsity2 : sparsity1;
+ double sp2 = sparse2 ? sparsity2 : sparsity1;
+ double sp3 = sparse3 ? sparsity2 : sparsity1;
+ double[][] A = getRandomMatrix(rows1, cols, -1, 1, sp1, 711);
+ double[][] B = getRandomMatrix(rows2, cols, -1, 1, sp2, 722);
+ double[][] C = getRandomMatrix(rows3, cols, -1, 1, sp3, 733);
+ writeInputMatrixWithMTD("A", A, true);
+ writeInputMatrixWithMTD("B", B, true);
+ writeInputMatrixWithMTD("C", C, true);
+
+ //run tests
+ runTest(true, false, null, -1);
+ runRScript(true);
+
+ //compare result data
+ HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
+ HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R");
+ TestUtils.compareMatrices(dmlfile, rfile, epsilon, "DML", "R");
+
+ //check for spark instructions
+ Assert.assertTrue(heavyHittersContainsSubString("sp_rbind")==(et==ExecType.SPARK));
+ }
+ finally {
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/test/scripts/functions/append/NaryCbind.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/append/NaryCbind.R b/src/test/scripts/functions/append/NaryCbind.R
deleted file mode 100644
index 4cc8d89..0000000
--- a/src/test/scripts/functions/append/NaryCbind.R
+++ /dev/null
@@ -1,32 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-args <- commandArgs(TRUE)
-options(digits=22)
-library("Matrix")
-
-A = as.matrix(readMM(paste(args[1], "A.mtx", sep="")))
-B = as.matrix(readMM(paste(args[1], "B.mtx", sep="")))
-C = as.matrix(readMM(paste(args[1], "C.mtx", sep="")))
-
-R = cbind(cbind(A, B), C);
-
-writeMM(as(R,"CsparseMatrix"), paste(args[2], "R", sep=""))
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/test/scripts/functions/append/NaryCbind.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/append/NaryCbind.dml b/src/test/scripts/functions/append/NaryCbind.dml
deleted file mode 100644
index 18331ad..0000000
--- a/src/test/scripts/functions/append/NaryCbind.dml
+++ /dev/null
@@ -1,26 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-A = read($1)
-B = read($2)
-C = read($3)
-R = cbind(A, B, C);
-write(R, $4)
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/test/scripts/functions/append/NaryRbind.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/append/NaryRbind.R b/src/test/scripts/functions/append/NaryRbind.R
deleted file mode 100644
index 30ed40b..0000000
--- a/src/test/scripts/functions/append/NaryRbind.R
+++ /dev/null
@@ -1,32 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-args <- commandArgs(TRUE)
-options(digits=22)
-library("Matrix")
-
-A = as.matrix(readMM(paste(args[1], "A.mtx", sep="")))
-B = as.matrix(readMM(paste(args[1], "B.mtx", sep="")))
-C = as.matrix(readMM(paste(args[1], "C.mtx", sep="")))
-
-R = rbind(rbind(A, B), C);
-
-writeMM(as(R,"CsparseMatrix"), paste(args[2], "R", sep=""))
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/test/scripts/functions/append/NaryRbind.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/append/NaryRbind.dml b/src/test/scripts/functions/append/NaryRbind.dml
deleted file mode 100644
index 515c4ce..0000000
--- a/src/test/scripts/functions/append/NaryRbind.dml
+++ /dev/null
@@ -1,26 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-A = read($1)
-B = read($2)
-C = read($3)
-R = rbind(A, B, C);
-write(R, $4)
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/test/scripts/functions/nary/NaryCbind.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/nary/NaryCbind.R b/src/test/scripts/functions/nary/NaryCbind.R
new file mode 100644
index 0000000..4cc8d89
--- /dev/null
+++ b/src/test/scripts/functions/nary/NaryCbind.R
@@ -0,0 +1,32 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+A = as.matrix(readMM(paste(args[1], "A.mtx", sep="")))
+B = as.matrix(readMM(paste(args[1], "B.mtx", sep="")))
+C = as.matrix(readMM(paste(args[1], "C.mtx", sep="")))
+
+R = cbind(cbind(A, B), C);
+
+writeMM(as(R,"CsparseMatrix"), paste(args[2], "R", sep=""))
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/test/scripts/functions/nary/NaryCbind.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/nary/NaryCbind.dml b/src/test/scripts/functions/nary/NaryCbind.dml
new file mode 100644
index 0000000..18331ad
--- /dev/null
+++ b/src/test/scripts/functions/nary/NaryCbind.dml
@@ -0,0 +1,26 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+A = read($1)
+B = read($2)
+C = read($3)
+R = cbind(A, B, C);
+write(R, $4)
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/test/scripts/functions/nary/NaryMinMax.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/nary/NaryMinMax.R b/src/test/scripts/functions/nary/NaryMinMax.R
new file mode 100644
index 0000000..fb34545
--- /dev/null
+++ b/src/test/scripts/functions/nary/NaryMinMax.R
@@ -0,0 +1,37 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+A = as.matrix(readMM(paste(args[1], "A.mtx", sep="")))
+B = as.matrix(readMM(paste(args[1], "B.mtx", sep="")))
+C = as.matrix(readMM(paste(args[1], "C.mtx", sep="")))
+
+#note: R requires that the first argument is a matrix
+if( as.logical(args[2]) ) {
+ R = pmin(A, B, C, 3, 7);
+} else {
+ R = pmax(A, B, C, -3, -7);
+}
+
+writeMM(as(R,"CsparseMatrix"), paste(args[3], "R", sep=""))
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/test/scripts/functions/nary/NaryMinMax.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/nary/NaryMinMax.dml b/src/test/scripts/functions/nary/NaryMinMax.dml
new file mode 100644
index 0000000..a02b09c
--- /dev/null
+++ b/src/test/scripts/functions/nary/NaryMinMax.dml
@@ -0,0 +1,31 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+A = read($1)
+B = read($2)
+C = read($3)
+
+if( as.logical($4) )
+ R = min(3, A, B, 7, C);
+else
+ R = max(-3, A, B, -7, C);
+
+write(R, $5)
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/test/scripts/functions/nary/NaryRbind.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/nary/NaryRbind.R b/src/test/scripts/functions/nary/NaryRbind.R
new file mode 100644
index 0000000..30ed40b
--- /dev/null
+++ b/src/test/scripts/functions/nary/NaryRbind.R
@@ -0,0 +1,32 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args <- commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+A = as.matrix(readMM(paste(args[1], "A.mtx", sep="")))
+B = as.matrix(readMM(paste(args[1], "B.mtx", sep="")))
+C = as.matrix(readMM(paste(args[1], "C.mtx", sep="")))
+
+R = rbind(rbind(A, B), C);
+
+writeMM(as(R,"CsparseMatrix"), paste(args[2], "R", sep=""))
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/test/scripts/functions/nary/NaryRbind.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/nary/NaryRbind.dml b/src/test/scripts/functions/nary/NaryRbind.dml
new file mode 100644
index 0000000..515c4ce
--- /dev/null
+++ b/src/test/scripts/functions/nary/NaryRbind.dml
@@ -0,0 +1,26 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+A = read($1)
+B = read($2)
+C = read($3)
+R = rbind(A, B, C);
+write(R, $4)
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/test_suites/java/org/apache/sysml/test/integration/functions/append/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/append/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/append/ZPackageSuite.java
index df7cdd1..5f0842b 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/append/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/append/ZPackageSuite.java
@@ -29,8 +29,6 @@ import org.junit.runners.Suite;
AppendChainTest.class,
AppendMatrixTest.class,
AppendVectorTest.class,
- NaryCBindTest.class,
- NaryRBindTest.class,
RBindCBindMatrixTest.class,
StringAppendTest.class,
})
http://git-wip-us.apache.org/repos/asf/systemml/blob/231a3648/src/test_suites/java/org/apache/sysml/test/integration/functions/nary/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/nary/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/nary/ZPackageSuite.java
new file mode 100644
index 0000000..5344c3a
--- /dev/null
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/nary/ZPackageSuite.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.nary;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+/** Group together the tests in this package into a single suite so that the Maven build
+ * won't run two of them at once. */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ NaryCBindTest.class,
+ NaryMinMaxTest.class,
+ NaryRBindTest.class,
+})
+
+
+/** This class is just a holder for the above JUnit annotations. */
+public class ZPackageSuite {
+
+}
[2/2] systemml git commit: [SYSTEMML] Distributed spark nary min/max
operations (compiler/runtime)
Posted by mb...@apache.org.
[SYSTEMML] Distributed spark nary min/max operations (compiler/runtime)
This patch adds the necessary compiler and runtime support for
distributed spark nary min/max operations. The basic idea is to join all
matrix inputs similar to codegen spark operations, ship all scalars via
task closures, and then run partitioning-preserving block operations.
Furthermore, this also includes a fix of sparse runtime operations
(robustness for empty rows) and refactors some existing code to avoid
duplications.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/5f580f02
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/5f580f02
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/5f580f02
Branch: refs/heads/master
Commit: 5f580f02ec0181f09806140f54b613ca139151d7
Parents: 231a364
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Jun 7 21:34:02 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Jun 7 21:34:02 2018 -0700
----------------------------------------------------------------------
.../context/ExecutionContext.java | 19 +++-
.../instructions/SPInstructionParser.java | 4 +
.../cp/MatrixBuiltinNaryCPInstruction.java | 23 +---
.../spark/BuiltinNarySPInstruction.java | 108 ++++++++++++++-----
.../instructions/spark/SpoofSPInstruction.java | 24 +----
.../spark/functions/MapInputSignature.java | 33 ++++++
.../spark/functions/MapJoinSignature.java | 36 +++++++
.../sysml/runtime/matrix/data/MatrixBlock.java | 1 +
.../functions/nary/NaryMinMaxTest.java | 20 ++++
9 files changed, 201 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/5f580f02/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
index 6807848..d87f9d9 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
@@ -20,7 +20,9 @@
package org.apache.sysml.runtime.controlprogram.context;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
+import java.util.stream.Collectors;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -72,7 +74,7 @@ public class ExecutionContext {
/**
* List of {@link GPUContext}s owned by this {@link ExecutionContext}
*/
- protected List<GPUContext> _gpuContexts = new ArrayList<>();
+ protected List<GPUContext> _gpuContexts = new ArrayList<>();
protected ExecutionContext()
{
@@ -494,6 +496,21 @@ public class ExecutionContext {
setVariable(varName, fo);
}
+ public List<MatrixBlock> getMatrixInputs(CPOperand[] inputs) {
+ return Arrays.stream(inputs).filter(in -> in.isMatrix())
+ .map(in -> getMatrixInput(in.getName())).collect(Collectors.toList());
+ }
+
+ public List<ScalarObject> getScalarInputs(CPOperand[] inputs) {
+ return Arrays.stream(inputs).filter(in -> in.isScalar())
+ .map(in -> getScalarInput(in)).collect(Collectors.toList());
+ }
+
+ public void releaseMatrixInputs(CPOperand[] inputs) {
+ Arrays.stream(inputs).filter(in -> in.isMatrix())
+ .forEach(in -> releaseMatrixInput(in.getName()));
+ }
+
/**
* Pin a given list of variables i.e., set the "clean up" state in
* corresponding matrix objects, so that the cached data inside these
http://git-wip-us.apache.org/repos/asf/systemml/blob/5f580f02/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
index 5a201cb..dd91b9f 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
@@ -262,8 +262,12 @@ public class SPInstructionParser extends InstructionParser
String2SPInstructionType.put( "rappend", SPType.RAppend);
String2SPInstructionType.put( "gappend", SPType.GAppend);
String2SPInstructionType.put( "galignedappend", SPType.GAlignedAppend);
+
String2SPInstructionType.put( "cbind", SPType.BuiltinNary);
String2SPInstructionType.put( "rbind", SPType.BuiltinNary);
+ String2SPInstructionType.put( "nmin", SPType.BuiltinNary);
+ String2SPInstructionType.put( "nmax", SPType.BuiltinNary);
+
String2SPInstructionType.put( DataGen.RAND_OPCODE , SPType.Rand);
String2SPInstructionType.put( DataGen.SEQ_OPCODE , SPType.Rand);
http://git-wip-us.apache.org/repos/asf/systemml/blob/5f580f02/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinNaryCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinNaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinNaryCPInstruction.java
index 50e3721..6f644ec 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinNaryCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinNaryCPInstruction.java
@@ -19,9 +19,7 @@
package org.apache.sysml.runtime.instructions.cp;
-import java.util.Arrays;
import java.util.List;
-import java.util.stream.Collectors;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
@@ -37,8 +35,8 @@ public class MatrixBuiltinNaryCPInstruction extends BuiltinNaryCPInstruction {
@Override
public void processInstruction(ExecutionContext ec) {
//separate scalars and matrices and pin all input matrices
- List<MatrixBlock> matrices = getMatrices(ec);
- List<ScalarObject> scalars = getScalars(ec);
+ List<MatrixBlock> matrices = ec.getMatrixInputs(inputs);
+ List<ScalarObject> scalars = ec.getScalarInputs(inputs);
MatrixBlock outBlock = null;
if( "cbind".equals(getOpcode()) || "rbind".equals(getOpcode()) ) {
@@ -56,7 +54,7 @@ public class MatrixBuiltinNaryCPInstruction extends BuiltinNaryCPInstruction {
}
//release inputs and set output matrix or scalar
- releaseInputs(ec);
+ ec.releaseMatrixInputs(inputs);
if( output.getDataType().isMatrix() ) {
ec.setMatrixOutput(output.getName(), outBlock);
}
@@ -65,19 +63,4 @@ public class MatrixBuiltinNaryCPInstruction extends BuiltinNaryCPInstruction {
output.getValueType(), outBlock.quickGetValue(0, 0)));
}
}
-
- private List<MatrixBlock> getMatrices(ExecutionContext ec) {
- return Arrays.stream(inputs).filter(in -> in.getDataType().isMatrix())
- .map(in -> ec.getMatrixInput(in.getName())).collect(Collectors.toList());
- }
-
- private List<ScalarObject> getScalars(ExecutionContext ec) {
- return Arrays.stream(inputs).filter(in -> in.getDataType().isScalar())
- .map(in -> ec.getScalarInput(in)).collect(Collectors.toList());
- }
-
- private void releaseInputs(ExecutionContext ec) {
- Arrays.stream(inputs).filter(in -> in.getDataType().isMatrix())
- .forEach(in -> ec.releaseMatrixInput(in.getName()));
- }
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/5f580f02/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinNarySPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinNarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinNarySPInstruction.java
index 583fe96..b0a77fb 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinNarySPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/BuiltinNarySPInstruction.java
@@ -19,18 +19,26 @@
package org.apache.sysml.runtime.instructions.spark;
+import java.util.List;
+
import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.functionobjects.Builtin;
import org.apache.sysml.runtime.instructions.InstructionUtils;
import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.cp.ScalarObject;
import org.apache.sysml.runtime.instructions.spark.AppendGSPInstruction.ShiftMatrix;
+import org.apache.sysml.runtime.instructions.spark.functions.MapInputSignature;
+import org.apache.sysml.runtime.instructions.spark.functions.MapJoinSignature;
import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
import org.apache.sysml.runtime.instructions.spark.utils.SparkUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.SimpleOperator;
import org.apache.sysml.runtime.util.UtilFunctions;
import scala.Tuple2;
@@ -60,53 +68,88 @@ public class BuiltinNarySPInstruction extends SPInstruction
@Override
public void processInstruction(ExecutionContext ec) {
SparkExecutionContext sec = (SparkExecutionContext)ec;
- boolean cbind = getOpcode().equals("cbind");
-
- //compute output characteristics
- MatrixCharacteristics mcOut = computeOutputMatrixCharacteristics(sec, inputs, cbind);
-
- //get consolidated input via union over shifted and padded inputs
- MatrixCharacteristics off = new MatrixCharacteristics(
- 0, 0, mcOut.getRowsPerBlock(), mcOut.getColsPerBlock(), 0);
JavaPairRDD<MatrixIndexes,MatrixBlock> out = null;
- for( CPOperand input : inputs ) {
- MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input.getName());
- JavaPairRDD<MatrixIndexes,MatrixBlock> in = sec
- .getBinaryBlockRDDHandleForVariable( input.getName() )
- .flatMapToPair(new ShiftMatrix(off, mcIn, cbind))
- .mapToPair(new PadBlocksFunction(mcOut)); //just padding
- out = (out != null) ? out.union(in) : in;
- updateMatrixCharacteristics(mcIn, off, cbind);
- }
+ MatrixCharacteristics mcOut = null;
- //aggregate partially overlapping blocks w/ single shuffle
- int numPartOut = SparkUtils.getNumPreferredPartitions(mcOut);
- out = RDDAggregateUtils.mergeByKey(out, numPartOut, false);
+ if( getOpcode().equals("cbind") || getOpcode().equals("rbind") ) {
+ //compute output characteristics
+ boolean cbind = getOpcode().equals("cbind");
+ mcOut = computeAppendOutputMatrixCharacteristics(sec, inputs, cbind);
+
+ //get consolidated input via union over shifted and padded inputs
+ MatrixCharacteristics off = new MatrixCharacteristics(
+ 0, 0, mcOut.getRowsPerBlock(), mcOut.getColsPerBlock(), 0);
+ for( CPOperand input : inputs ) {
+ MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input.getName());
+ JavaPairRDD<MatrixIndexes,MatrixBlock> in = sec
+ .getBinaryBlockRDDHandleForVariable( input.getName() )
+ .flatMapToPair(new ShiftMatrix(off, mcIn, cbind))
+ .mapToPair(new PadBlocksFunction(mcOut)); //just padding
+ out = (out != null) ? out.union(in) : in;
+ updateAppendMatrixCharacteristics(mcIn, off, cbind);
+ }
+
+ //aggregate partially overlapping blocks w/ single shuffle
+ int numPartOut = SparkUtils.getNumPreferredPartitions(mcOut);
+ out = RDDAggregateUtils.mergeByKey(out, numPartOut, false);
+ }
+ else if( getOpcode().equals("nmin") || getOpcode().equals("nmax") ) {
+ //compute output characteristics
+ mcOut = computeMinMaxOutputMatrixCharacteristics(sec, inputs);
+
+ //get scalars and consolidated input via join
+ List<ScalarObject> scalars = sec.getScalarInputs(inputs);
+ JavaPairRDD<MatrixIndexes, MatrixBlock[]> in = null;
+ for( CPOperand input : inputs ) {
+ if( !input.getDataType().isMatrix() ) continue;
+ JavaPairRDD<MatrixIndexes, MatrixBlock> tmp = sec
+ .getBinaryBlockRDDHandleForVariable(input.getName());
+ in = (in == null) ? tmp.mapValues(new MapInputSignature()) :
+ in.join(tmp).mapValues(new MapJoinSignature());
+ }
+
+ //compute nary min/max (partitioning-preserving)
+ out = in.mapValues(new MinMaxFunction(getOpcode(), scalars));
+ }
//set output RDD and add lineage
sec.getMatrixCharacteristics(output.getName()).set(mcOut);
sec.setRDDHandleForVariable(output.getName(), out);
for( CPOperand input : inputs )
- sec.addLineageRDD(output.getName(), input.getName());
+ if( !input.isScalar() )
+ sec.addLineageRDD(output.getName(), input.getName());
}
- private static MatrixCharacteristics computeOutputMatrixCharacteristics(SparkExecutionContext sec, CPOperand[] inputs, boolean cbind) {
+ private static MatrixCharacteristics computeAppendOutputMatrixCharacteristics(SparkExecutionContext sec, CPOperand[] inputs, boolean cbind) {
MatrixCharacteristics mcIn1 = sec.getMatrixCharacteristics(inputs[0].getName());
MatrixCharacteristics mcOut = new MatrixCharacteristics(
0, 0, mcIn1.getRowsPerBlock(), mcIn1.getColsPerBlock(), 0);
for( CPOperand input : inputs ) {
MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input.getName());
- updateMatrixCharacteristics(mcIn, mcOut, cbind);
+ updateAppendMatrixCharacteristics(mcIn, mcOut, cbind);
}
return mcOut;
}
- private static void updateMatrixCharacteristics(MatrixCharacteristics in, MatrixCharacteristics out, boolean cbind) {
+ private static void updateAppendMatrixCharacteristics(MatrixCharacteristics in, MatrixCharacteristics out, boolean cbind) {
out.setDimension(cbind ? Math.max(out.getRows(), in.getRows()) : out.getRows()+in.getRows(),
cbind ? out.getCols()+in.getCols() : Math.max(out.getCols(), in.getCols()));
out.setNonZeros((out.getNonZeros()!=-1 && in.dimsKnown(true)) ? out.getNonZeros()+in.getNonZeros() : -1);
}
+ private static MatrixCharacteristics computeMinMaxOutputMatrixCharacteristics(SparkExecutionContext sec, CPOperand[] inputs) {
+ MatrixCharacteristics mcOut = new MatrixCharacteristics();
+ for( CPOperand input : inputs ) {
+ if( !input.getDataType().isMatrix() ) continue;
+ MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input.getName());
+ mcOut.setRows(Math.max(mcOut.getRows(), mcIn.getRows()));
+ mcOut.setCols(Math.max(mcOut.getCols(), mcIn.getCols()));
+ mcOut.setRowsPerBlock(mcIn.getRowsPerBlock());
+ mcOut.setColsPerBlock(mcIn.getColsPerBlock());
+ }
+ return mcOut;
+ }
+
public static class PadBlocksFunction implements PairFunction<Tuple2<MatrixIndexes,MatrixBlock>,MatrixIndexes,MatrixBlock>
{
private static final long serialVersionUID = 1291358959908299855L;
@@ -135,4 +178,21 @@ public class BuiltinNarySPInstruction extends SPInstruction
return new Tuple2<>(ix, mb);
}
}
+
+ private static class MinMaxFunction implements Function<MatrixBlock[], MatrixBlock> {
+ private static final long serialVersionUID = -4227447915387484397L;
+
+ private final SimpleOperator _op;
+ private final ScalarObject[] _scalars;
+
+ public MinMaxFunction(String opcode, List<ScalarObject> scalars) {
+ _scalars = scalars.toArray(new ScalarObject[0]);
+ _op = new SimpleOperator(Builtin.getBuiltinFnObject(opcode.substring(1)));
+ }
+
+ @Override
+ public MatrixBlock call(MatrixBlock[] v1) throws Exception {
+ return MatrixBlock.naryOperations(_op, v1, _scalars, new MatrixBlock());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/5f580f02/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java
index 15d4de7..8f63427 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java
@@ -27,9 +27,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.stream.IntStream;
-import org.apache.commons.lang3.ArrayUtils;
import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFlatMapFunction;
import org.apache.spark.api.java.function.PairFunction;
@@ -59,6 +57,8 @@ import org.apache.sysml.runtime.instructions.cp.CPOperand;
import org.apache.sysml.runtime.instructions.cp.DoubleObject;
import org.apache.sysml.runtime.instructions.cp.ScalarObject;
import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcast;
+import org.apache.sysml.runtime.instructions.spark.functions.MapInputSignature;
+import org.apache.sysml.runtime.instructions.spark.functions.MapJoinSignature;
import org.apache.sysml.runtime.instructions.spark.functions.ReplicateBlockFunction;
import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
@@ -352,26 +352,6 @@ public class SpoofSPInstruction extends SPInstruction {
}
}
- private static class MapInputSignature implements Function<MatrixBlock, MatrixBlock[]>
- {
- private static final long serialVersionUID = -816443970067626102L;
-
- @Override
- public MatrixBlock[] call(MatrixBlock v1) throws Exception {
- return new MatrixBlock[]{ v1 };
- }
- }
-
- private static class MapJoinSignature implements Function<Tuple2<MatrixBlock[],MatrixBlock>, MatrixBlock[]>
- {
- private static final long serialVersionUID = -704403012606821854L;
-
- @Override
- public MatrixBlock[] call(Tuple2<MatrixBlock[], MatrixBlock> v1) throws Exception {
- return ArrayUtils.add(v1._1(), v1._2());
- }
- }
-
private static class SpoofFunction implements Serializable
{
private static final long serialVersionUID = 2953479427746463003L;
http://git-wip-us.apache.org/repos/asf/systemml/blob/5f580f02/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MapInputSignature.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MapInputSignature.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MapInputSignature.java
new file mode 100644
index 0000000..c7fdad6
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MapInputSignature.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.instructions.spark.functions;
+
+import org.apache.spark.api.java.function.Function;
+
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+
+public class MapInputSignature implements Function<MatrixBlock, MatrixBlock[]> {
+ private static final long serialVersionUID = -816443970067626102L;
+
+ @Override
+ public MatrixBlock[] call(MatrixBlock v1) throws Exception {
+ return new MatrixBlock[]{ v1 };
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/5f580f02/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MapJoinSignature.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MapJoinSignature.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MapJoinSignature.java
new file mode 100644
index 0000000..acfa687
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MapJoinSignature.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.instructions.spark.functions;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.spark.api.java.function.Function;
+
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+
+import scala.Tuple2;
+
+public class MapJoinSignature implements Function<Tuple2<MatrixBlock[],MatrixBlock>, MatrixBlock[]> {
+ private static final long serialVersionUID = -704403012606821854L;
+
+ @Override
+ public MatrixBlock[] call(Tuple2<MatrixBlock[], MatrixBlock> v1) throws Exception {
+ return ArrayUtils.add(v1._1(), v1._2());
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/5f580f02/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index 755bca1..5c285f8 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -3531,6 +3531,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
continue;
if( in.isInSparseFormat() ) {
SparseBlock a = in.sparseBlock;
+ if( a.isEmpty(i) ) continue;
int alen = a.size(i);
int apos = a.pos(i);
int[] aix = a.indexes(i);
http://git-wip-us.apache.org/repos/asf/systemml/blob/5f580f02/src/test/java/org/apache/sysml/test/integration/functions/nary/NaryMinMaxTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/nary/NaryMinMaxTest.java b/src/test/java/org/apache/sysml/test/integration/functions/nary/NaryMinMaxTest.java
index c5a026a..3ef29c9 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/nary/NaryMinMaxTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/nary/NaryMinMaxTest.java
@@ -71,6 +71,26 @@ public class NaryMinMaxTest extends AutomatedTestBase
runMinMaxTest(false, true, ExecType.CP);
}
+ @Test
+ public void testNaryMinDenseSP() {
+ runMinMaxTest(true, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryMinSparseSP() {
+ runMinMaxTest(true, true, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryMaxDenseSP() {
+ runMinMaxTest(false, false, ExecType.SPARK);
+ }
+
+ @Test
+ public void testNaryMaxSparseSP() {
+ runMinMaxTest(false, true, ExecType.SPARK);
+ }
+
public void runMinMaxTest(boolean min, boolean sparse, ExecType et)
{
RUNTIME_PLATFORM platformOld = rtplatform;