You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/06/01 16:49:23 UTC
[3/3] incubator-systemml git commit: [SYSTEMML-562] New spark
frame/matrix casting instructions, tests
[SYSTEMML-562] New spark frame/matrix casting instructions, tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/9607a376
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/9607a376
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/9607a376
Branch: refs/heads/master
Commit: 9607a376ae2346b103c97efe1c4afc9a75a62c08
Parents: 507172a
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Wed Jun 1 01:34:18 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Wed Jun 1 09:48:50 2016 -0700
----------------------------------------------------------------------
src/main/java/org/apache/sysml/hops/Hop.java | 2 +
.../java/org/apache/sysml/hops/MemoTable.java | 6 +
.../sysml/hops/ParameterizedBuiltinOp.java | 14 +-
.../java/org/apache/sysml/hops/UnaryOp.java | 4 +-
src/main/java/org/apache/sysml/lops/Unary.java | 7 +
.../context/SparkExecutionContext.java | 16 +-
.../instructions/SPInstructionParser.java | 7 +
.../instructions/spark/CastSPInstruction.java | 99 ++++++++
.../spark/CheckpointSPInstruction.java | 2 +-
.../instructions/spark/SPInstruction.java | 4 +-
.../functions/frame/FrameMatrixCastingTest.java | 251 +++++++++++++++++++
.../functions/frame/Frame2MatrixCast.dml | 26 ++
.../functions/frame/Matrix2FrameCast.dml | 26 ++
.../functions/frame/ZPackageSuite.java | 1 +
14 files changed, 450 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9607a376/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index 8519055..e0a06e6 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -1288,6 +1288,8 @@ public abstract class Hop
HopsOpOp1LopsU.put(OpOp1.SIGMOID, org.apache.sysml.lops.Unary.OperationTypes.SIGMOID);
HopsOpOp1LopsU.put(OpOp1.SELP, org.apache.sysml.lops.Unary.OperationTypes.SELP);
HopsOpOp1LopsU.put(OpOp1.LOG_NZ, org.apache.sysml.lops.Unary.OperationTypes.LOG_NZ);
+ HopsOpOp1LopsU.put(OpOp1.CAST_AS_MATRIX, org.apache.sysml.lops.Unary.OperationTypes.CAST_AS_MATRIX);
+ HopsOpOp1LopsU.put(OpOp1.CAST_AS_FRAME, org.apache.sysml.lops.Unary.OperationTypes.CAST_AS_FRAME);
}
protected static final HashMap<Hop.OpOp1, org.apache.sysml.lops.UnaryCP.OperationTypes> HopsOpOp1LopsUS;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9607a376/src/main/java/org/apache/sysml/hops/MemoTable.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/MemoTable.java b/src/main/java/org/apache/sysml/hops/MemoTable.java
index ff41f09..cae8507 100644
--- a/src/main/java/org/apache/sysml/hops/MemoTable.java
+++ b/src/main/java/org/apache/sysml/hops/MemoTable.java
@@ -127,6 +127,9 @@ public class MemoTable
*/
public MatrixCharacteristics[] getAllInputStats( ArrayList<Hop> inputs )
{
+ if( inputs == null )
+ return null;
+
MatrixCharacteristics[] ret = new MatrixCharacteristics[inputs.size()];
for( int i=0; i<inputs.size(); i++ )
{
@@ -164,6 +167,9 @@ public class MemoTable
*/
public MatrixCharacteristics getAllInputStats( Hop input )
{
+ if( input == null )
+ return null;
+
MatrixCharacteristics ret = null;
long dim1 = input.getDim1();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9607a376/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
index a793fb1..b2a1807 100644
--- a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
+++ b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
@@ -143,11 +143,9 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop
_outputPermutationMatrix = flag;
}
- public Hop getTargetHop()
- {
- Hop targetHop = getInput().get(_paramIndexMap.get("target"));
-
- return targetHop;
+ public Hop getTargetHop() {
+ return _paramIndexMap.containsKey("target") ?
+ getInput().get(_paramIndexMap.get("target")) : null;
}
@Override
@@ -985,13 +983,11 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop
@Override
protected long[] inferOutputCharacteristics( MemoTable memo )
{
- //CDF always known because
-
- // TOSTRING outputs a string
+ //Notes: CDF, TOSTRING always known because scalar outputs
long[] ret = null;
- Hop input = getInput().get(_paramIndexMap.get("target"));
+ Hop input = getTargetHop();
MatrixCharacteristics mc = memo.getAllInputStats(input);
if( _op == ParamBuiltinOp.GROUPEDAGG )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9607a376/src/main/java/org/apache/sysml/hops/UnaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/UnaryOp.java b/src/main/java/org/apache/sysml/hops/UnaryOp.java
index 4ed0225..f0c8482 100644
--- a/src/main/java/org/apache/sysml/hops/UnaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/UnaryOp.java
@@ -121,8 +121,8 @@ public class UnaryOp extends Hop implements MultiThreadedHop
{
Hop input = getInput().get(0);
- if (getDataType() == DataType.SCALAR
- || _op == OpOp1.CAST_AS_MATRIX || _op == OpOp1.CAST_AS_FRAME ) //TODO generalize frames to distributed ops
+ if( getDataType() == DataType.SCALAR //value type casts or matrix to scalar
+ || (_op == OpOp1.CAST_AS_MATRIX && getInput().get(0).getDataType()==DataType.SCALAR) )
{
if (_op == Hop.OpOp1.IQM) //special handling IQM
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9607a376/src/main/java/org/apache/sysml/lops/Unary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Unary.java b/src/main/java/org/apache/sysml/lops/Unary.java
index 8ef3e7b..bea7352 100644
--- a/src/main/java/org/apache/sysml/lops/Unary.java
+++ b/src/main/java/org/apache/sysml/lops/Unary.java
@@ -43,6 +43,7 @@ public class Unary extends Lop
ROUND, CEIL, FLOOR, MR_IQM, INVERSE, CHOLESKY,
CUMSUM, CUMPROD, CUMMIN, CUMMAX,
SPROP, SIGMOID, SELP, SUBTRACT_NZ, LOG_NZ,
+ CAST_AS_MATRIX, CAST_AS_FRAME,
NOTSUPPORTED
};
@@ -326,6 +327,12 @@ public class Unary extends Lop
case SELP:
return "sel+";
+ case CAST_AS_MATRIX:
+ return UnaryCP.CAST_AS_MATRIX_OPCODE;
+
+ case CAST_AS_FRAME:
+ return UnaryCP.CAST_AS_FRAME_OPCODE;
+
default:
throw new LopsException(
"Instruction not defined for Unary operation: " + op);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9607a376/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
index f282579..956ea4e 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
@@ -540,7 +540,7 @@ public class SparkExecutionContext extends ExecutionContext
* @param rdd
* @throws DMLRuntimeException
*/
- public void setRDDHandleForVariable(String varname, JavaPairRDD<MatrixIndexes,?> rdd)
+ public void setRDDHandleForVariable(String varname, JavaPairRDD<?,?> rdd)
throws DMLRuntimeException
{
MatrixObject mo = getMatrixObject(varname);
@@ -549,6 +549,20 @@ public class SparkExecutionContext extends ExecutionContext
}
/**
+ *
+ * @param varname
+ * @param rdd
+ * @throws DMLRuntimeException
+ */
+ public void setFrameRDDHandleForVariable(String varname, JavaPairRDD<?,?> rdd)
+ throws DMLRuntimeException
+ {
+ FrameObject mo = getFrameObject(varname);
+ RDDObject rddhandle = new RDDObject(rdd, varname);
+ mo.setRDDHandle( rddhandle );
+ }
+
+ /**
* Utility method for creating an RDD out of an in-memory matrix block.
*
* @param sc
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9607a376/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
index fa1a630..430aaf4 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java
@@ -45,6 +45,7 @@ import org.apache.sysml.runtime.instructions.spark.BinUaggChainSPInstruction;
import org.apache.sysml.runtime.instructions.spark.BuiltinBinarySPInstruction;
import org.apache.sysml.runtime.instructions.spark.BuiltinUnarySPInstruction;
import org.apache.sysml.runtime.instructions.spark.CSVReblockSPInstruction;
+import org.apache.sysml.runtime.instructions.spark.CastSPInstruction;
import org.apache.sysml.runtime.instructions.spark.CentralMomentSPInstruction;
import org.apache.sysml.runtime.instructions.spark.CheckpointSPInstruction;
import org.apache.sysml.runtime.instructions.spark.CovarianceSPInstruction;
@@ -256,6 +257,9 @@ public class SPInstructionParser extends InstructionParser
String2SPInstructionType.put( "binuaggchain", SPINSTRUCTION_TYPE.BinUaggChain);
String2SPInstructionType.put( "write" , SPINSTRUCTION_TYPE.Write);
+
+ String2SPInstructionType.put( "castdtm" , SPINSTRUCTION_TYPE.Cast);
+ String2SPInstructionType.put( "castdtf" , SPINSTRUCTION_TYPE.Cast);
}
public static SPInstruction parseSingleInstruction (String str )
@@ -406,6 +410,9 @@ public class SPInstructionParser extends InstructionParser
case Checkpoint:
return CheckpointSPInstruction.parseInstruction(str);
+
+ case Cast:
+ return CastSPInstruction.parseInstruction(str);
case INVALID:
default:
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9607a376/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java
new file mode 100644
index 0000000..8160bd0
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.instructions.spark;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.lops.UnaryCP;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.InstructionUtils;
+import org.apache.sysml.runtime.instructions.cp.CPOperand;
+import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils;
+import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils.LongFrameToLongWritableFrameFunction;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.operators.Operator;
+
+
+public class CastSPInstruction extends UnarySPInstruction
+{
+ public CastSPInstruction(Operator op, CPOperand in, CPOperand out, String opcode, String istr) {
+ super(op, in, out, opcode, istr);
+ _sptype = SPINSTRUCTION_TYPE.Cast;
+ }
+
+ public static CastSPInstruction parseInstruction ( String str )
+ throws DMLRuntimeException
+ {
+ String[] parts = InstructionUtils.getInstructionPartsWithValueType(str);
+ InstructionUtils.checkNumFields(parts, 2);
+
+ String opcode = parts[0];
+ CPOperand in = new CPOperand(parts[1]);
+ CPOperand out = new CPOperand(parts[2]);
+
+ return new CastSPInstruction(null, in, out, opcode, str);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void processInstruction(ExecutionContext ec)
+ throws DMLRuntimeException
+ {
+ SparkExecutionContext sec = (SparkExecutionContext)ec;
+ String opcode = getOpcode();
+
+ //get input RDD and prepare output
+ JavaPairRDD<?,?> in = sec.getRDDHandleForVariable( input1.getName(), InputInfo.BinaryBlockInputInfo );
+ MatrixCharacteristics mcIn = sec.getMatrixCharacteristics( input1.getName() );
+ JavaPairRDD<?,?> out = null;
+
+ //convert frame-matrix / matrix-frame and set output
+ if( opcode.equals(UnaryCP.CAST_AS_MATRIX_OPCODE) ) {
+ //TODO: simplify converter api to allow long indexes to be passed in
+ MatrixCharacteristics mcOut = new MatrixCharacteristics(mcIn);
+ mcOut.setBlockSize(ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
+ in = ((JavaPairRDD<Long, FrameBlock>)in).mapToPair(new LongFrameToLongWritableFrameFunction());
+ out = FrameRDDConverterUtils.binaryBlockToMatrixBlock(
+ (JavaPairRDD<LongWritable, FrameBlock>)in, mcIn, mcOut);
+
+ sec.setRDDHandleForVariable(output.getName(), out);
+ }
+ else if( opcode.equals(UnaryCP.CAST_AS_FRAME_OPCODE) ) {
+ out = FrameRDDConverterUtils.matrixBlockToBinaryBlockLongIndex(sec.getSparkContext(),
+ (JavaPairRDD<MatrixIndexes, MatrixBlock>)in, mcIn);
+
+ sec.setFrameRDDHandleForVariable(output.getName(), out);
+ }
+ else {
+ throw new DMLRuntimeException("Unsupported spark cast operation: "+opcode);
+ }
+
+ //update output statistics and add lineage
+ updateUnaryOutputMatrixCharacteristics(sec, input1.getName(), output.getName());
+ sec.addLineageRDD(output.getName(), input1.getName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9607a376/src/main/java/org/apache/sysml/runtime/instructions/spark/CheckpointSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CheckpointSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CheckpointSPInstruction.java
index 24c614a..03edadd 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CheckpointSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CheckpointSPInstruction.java
@@ -51,7 +51,7 @@ public class CheckpointSPInstruction extends UnarySPInstruction
public CheckpointSPInstruction(Operator op, CPOperand in, CPOperand out, StorageLevel level, String opcode, String istr) {
super(op, in, out, opcode, istr);
- _sptype = SPINSTRUCTION_TYPE.Reorg;
+ _sptype = SPINSTRUCTION_TYPE.Checkpoint;
_level = level;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9607a376/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java
index ae16075..acc2ff6 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/SPInstruction.java
@@ -40,8 +40,8 @@ public abstract class SPInstruction extends Instruction
public enum SPINSTRUCTION_TYPE {
MAPMM, MAPMMCHAIN, CPMM, RMM, TSMM, PMM, ZIPMM, PMAPMM, //matrix multiplication instructions
MatrixIndexing, Reorg, ArithmeticBinary, RelationalBinary, AggregateUnary, AggregateTernary, Reblock, CSVReblock,
- Builtin, BuiltinUnary, BuiltinBinary, Checkpoint,
- CentralMoment, Covariance, QSort, QPick,
+ Builtin, BuiltinUnary, BuiltinBinary, Checkpoint, Cast,
+ CentralMoment, Covariance, QSort, QPick,
ParameterizedBuiltin, MAppend, RAppend, GAppend, GAlignedAppend, Rand,
MatrixReshape, Ternary, Quaternary, CumsumAggregate, CumsumOffset, BinUaggChain, UaggOuterChain,
Write, INVALID,
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9607a376/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameMatrixCastingTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameMatrixCastingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameMatrixCastingTest.java
new file mode 100644
index 0000000..0baf9c2
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameMatrixCastingTest.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.frame;
+
+import java.io.IOException;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.io.FrameReader;
+import org.apache.sysml.runtime.io.FrameReaderFactory;
+import org.apache.sysml.runtime.io.FrameWriter;
+import org.apache.sysml.runtime.io.FrameWriterFactory;
+import org.apache.sysml.runtime.io.MatrixReader;
+import org.apache.sysml.runtime.io.MatrixReaderFactory;
+import org.apache.sysml.runtime.io.MatrixWriter;
+import org.apache.sysml.runtime.io.MatrixWriterFactory;
+import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class FrameMatrixCastingTest extends AutomatedTestBase
+{
+ private final static String TEST_DIR = "functions/frame/";
+ private final static String TEST_NAME1 = "Frame2MatrixCast";
+ private final static String TEST_NAME2 = "Matrix2FrameCast";
+ private final static String TEST_CLASS_DIR = TEST_DIR + FrameMatrixCastingTest.class.getSimpleName() + "/";
+
+ private final static int rows = 2593;
+ private final static int cols1 = 372;
+ private final static int cols2 = 1102;
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"B"}));
+ addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"B"}));
+ }
+
+ @Test
+ public void testStringFrame2MatrixCastSingleCP() {
+ runFrameCastingTest(TEST_NAME1, false, ValueType.STRING, ExecType.CP);
+ }
+
+ @Test
+ public void testStringFrame2MatrixCastMultiCP() {
+ runFrameCastingTest(TEST_NAME1, true, ValueType.STRING, ExecType.CP);
+ }
+
+ @Test
+ public void testDoubleFrame2MatrixCastSingleCP() {
+ runFrameCastingTest(TEST_NAME1, false, ValueType.DOUBLE, ExecType.CP);
+ }
+
+ @Test
+ public void testDoubleFrame2MatrixCastMultiCP() {
+ runFrameCastingTest(TEST_NAME1, true, ValueType.DOUBLE, ExecType.CP);
+ }
+
+ @Test
+ public void testMatrix2FrameCastSingleCP() {
+ runFrameCastingTest(TEST_NAME2, false, null, ExecType.CP);
+ }
+
+ @Test
+ public void testMatrix2FrameCastMultiCP() {
+ runFrameCastingTest(TEST_NAME2, true, null, ExecType.CP);
+ }
+
+ @Test
+ public void testStringFrame2MatrixCastSingleSpark() {
+ runFrameCastingTest(TEST_NAME1, false, ValueType.STRING, ExecType.SPARK);
+ }
+
+ @Test
+ public void testStringFrame2MatrixCastMultiSpark() {
+ runFrameCastingTest(TEST_NAME1, true, ValueType.STRING, ExecType.SPARK);
+ }
+
+ @Test
+ public void testDoubleFrame2MatrixCastSingleSpark() {
+ runFrameCastingTest(TEST_NAME1, false, ValueType.DOUBLE, ExecType.SPARK);
+ }
+
+ @Test
+ public void testDoubleFrame2MatrixCastMultiSpark() {
+ runFrameCastingTest(TEST_NAME1, true, ValueType.DOUBLE, ExecType.SPARK);
+ }
+
+ /*TODO write distributed frame missing
+ @Test
+ public void testMatrix2FrameCastSingleSpark() {
+ runFrameCastingTest(TEST_NAME2, false, null, ExecType.SPARK);
+ }
+
+ @Test
+ public void testMatrix2FrameCastMultiSpark() {
+ runFrameCastingTest(TEST_NAME2, true, null, ExecType.SPARK);
+ }
+ */
+
+ /**
+ *
+ * @param testname
+ * @param schema
+ * @param wildcard
+ */
+ private void runFrameCastingTest( String testname, boolean multColBlks, ValueType vt, ExecType et)
+ {
+ //rtplatform for MR
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch( et ){
+ case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break;
+ case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
+ default: rtplatform = RUNTIME_PLATFORM.HYBRID; break;
+ }
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK )
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try
+ {
+ int cols = multColBlks ? cols2 : cols1;
+
+ TestConfiguration config = getTestConfiguration(testname);
+ loadTestConfiguration(config);
+
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + testname + ".dml";
+ programArgs = new String[]{"-explain","-args", input("A"), output("B") };
+
+ //data generation
+ double[][] A = getRandomMatrix(rows, cols, -1, 1, 0.9, 7);
+ DataType dtin = testname.equals(TEST_NAME1) ? DataType.FRAME : DataType.MATRIX;
+ ValueType vtin = testname.equals(TEST_NAME1) ? vt : ValueType.DOUBLE;
+ writeMatrixOrFrameInput(input("A"), A, rows, cols, dtin, vtin);
+
+ //run testcase
+ runTest(true, false, null, -1);
+
+ //compare matrices
+ DataType dtout = testname.equals(TEST_NAME1) ? DataType.MATRIX : DataType.FRAME;
+ double[][] B = readMatrixOrFrameInput(output("B"), rows, cols, dtout);
+ TestUtils.compareMatrices(A, B, rows, cols, 0);
+ }
+ catch(Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ finally {
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ }
+ }
+
+ /**
+ *
+ * @param fname
+ * @param A
+ * @param rows
+ * @param cols
+ * @param dt
+ * @param vt
+ * @throws DMLRuntimeException
+ * @throws IOException
+ */
+ private void writeMatrixOrFrameInput(String fname, double[][] A, int rows, int cols, DataType dt, ValueType vt)
+ throws DMLRuntimeException, IOException
+ {
+ int blksize = ConfigurationManager.getBlocksize();
+
+ //write input data
+ if( dt == DataType.FRAME ) {
+ FrameBlock fb = DataConverter.convertToFrameBlock(DataConverter.convertToMatrixBlock(A), vt);
+ FrameWriter writer = FrameWriterFactory.createFrameWriter(OutputInfo.BinaryBlockOutputInfo);
+ writer.writeFrameToHDFS(fb, fname, rows, cols);
+ }
+ else {
+ MatrixBlock mb = DataConverter.convertToMatrixBlock(A);
+ MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(OutputInfo.BinaryBlockOutputInfo);
+ writer.writeMatrixToHDFS(mb, fname, (long)rows, (long)cols, blksize, blksize, -1);
+ }
+
+ //write meta data
+ MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, blksize, blksize);
+ MapReduceTool.writeMetaDataFile(fname+".mtd", vt, dt, mc, OutputInfo.BinaryBlockOutputInfo);
+
+ }
+
+ /**
+ *
+ * @param fname
+ * @param rows
+ * @param cols
+ * @param dt
+ * @return
+ * @throws DMLRuntimeException
+ * @throws IOException
+ */
+ private double[][] readMatrixOrFrameInput(String fname, int rows, int cols, DataType dt)
+ throws DMLRuntimeException, IOException
+ {
+ MatrixBlock ret = null;
+
+ //read input data
+ if( dt == DataType.FRAME ) {
+ FrameReader reader = FrameReaderFactory.createFrameReader(InputInfo.BinaryBlockInputInfo);
+ FrameBlock fb = reader.readFrameFromHDFS(fname, rows, cols);
+ ret = DataConverter.convertToMatrixBlock(fb);
+ }
+ else {
+ int blksize = ConfigurationManager.getBlocksize();
+ MatrixReader reader = MatrixReaderFactory.createMatrixReader(InputInfo.BinaryBlockInputInfo);
+ ret = reader.readMatrixFromHDFS(fname, rows, cols, blksize, blksize, -1);
+ }
+
+ return DataConverter.convertToDoubleMatrix(ret);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9607a376/src/test/scripts/functions/frame/Frame2MatrixCast.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/frame/Frame2MatrixCast.dml b/src/test/scripts/functions/frame/Frame2MatrixCast.dml
new file mode 100644
index 0000000..61d2ad4
--- /dev/null
+++ b/src/test/scripts/functions/frame/Frame2MatrixCast.dml
@@ -0,0 +1,26 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+A = read($1);
+
+B = as.matrix(A);
+
+write(B, $2, format="binary");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9607a376/src/test/scripts/functions/frame/Matrix2FrameCast.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/frame/Matrix2FrameCast.dml b/src/test/scripts/functions/frame/Matrix2FrameCast.dml
new file mode 100644
index 0000000..23009d6
--- /dev/null
+++ b/src/test/scripts/functions/frame/Matrix2FrameCast.dml
@@ -0,0 +1,26 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+A = read($1);
+
+B = as.frame(A);
+
+write(B, $2, format="binary");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9607a376/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
index d810534..819225c 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
@@ -32,6 +32,7 @@ import org.junit.runners.Suite;
FrameCopyTest.class,
FrameGetSetTest.class,
FrameIndexingTest.class,
+ FrameMatrixCastingTest.class,
FrameReadWriteTest.class,
FrameSchemaReadTest.class,
FrameSerializationTest.class,