You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/06/10 19:45:34 UTC

incubator-systemml git commit: [SYSTEMML-562] Spark frame reblock instruction (text), tests

Repository: incubator-systemml
Updated Branches:
  refs/heads/master 919d919be -> 9451a0fd8


[SYSTEMML-562] Spark frame reblock instruction (text), tests

This patch generalizes the existing reblock instruction for frames. In
order to stay compatible with the file-based transform, this does not
include CSV yet. Furthermore, it also includes a fix of the distributed
frame to matrix converter to deal with unaligned blocks.

Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/9451a0fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/9451a0fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/9451a0fd

Branch: refs/heads/master
Commit: 9451a0fd89bcf02cc9ba572e8092b9f1447c4d86
Parents: 919d919
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Fri Jun 10 00:59:26 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Fri Jun 10 12:45:17 2016 -0700

----------------------------------------------------------------------
 .../apache/sysml/hops/recompile/Recompiler.java |  36 ++-
 .../RewriteAlgebraicSimplificationStatic.java   |   2 +-
 .../rewrite/RewriteBlockSizeAndReblock.java     |   7 +-
 .../controlprogram/caching/FrameObject.java     |   4 +
 .../context/SparkExecutionContext.java          |  26 +-
 .../spark/CSVReblockSPInstruction.java          |   2 +-
 .../instructions/spark/CastSPInstruction.java   |   5 +-
 .../spark/ReblockSPInstruction.java             |  97 +++++--
 .../spark/utils/FrameRDDConverterUtils.java     |  90 +++---
 .../functions/frame/FrameConverterTest.java     |   8 +-
 .../functions/frame/FrameMatrixReblockTest.java | 277 +++++++++++++++++++
 .../misc/RewritePushdownSumBinaryMult.java      |  38 +--
 .../functions/frame/FrameMatrixReblock.dml      |  25 ++
 13 files changed, 480 insertions(+), 137 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9451a0fd/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
index 9daefdf..ec89775 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
@@ -78,6 +78,8 @@ import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
 import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
 import org.apache.sysml.runtime.controlprogram.ProgramBlock;
 import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
+import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
+import org.apache.sysml.runtime.controlprogram.caching.FrameObject;
 import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
 import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
 import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter;
@@ -94,6 +96,7 @@ import org.apache.sysml.runtime.instructions.mr.RandInstruction;
 import org.apache.sysml.runtime.instructions.mr.SeqInstruction;
 import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
 import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
 import org.apache.sysml.runtime.matrix.data.InputInfo;
 import org.apache.sysml.runtime.matrix.data.MatrixBlock;
 import org.apache.sysml.runtime.util.MapReduceTool;
@@ -1907,8 +1910,8 @@ public class Recompiler
 	public static boolean checkCPReblock(ExecutionContext ec, String varin) 
 		throws DMLRuntimeException
 	{
-		MatrixObject in = ec.getMatrixObject(varin);
-		MatrixCharacteristics mc = in.getMatrixCharacteristics();
+		CacheableData<?> obj = ec.getCacheableData(varin);
+		MatrixCharacteristics mc = ec.getMatrixCharacteristics(varin);
 		
 		long rows = mc.getRows();
 		long cols = mc.getCols();
@@ -1924,8 +1927,8 @@ public class Recompiler
 		//robustness for usage through mlcontext (key/values of input rdds are 
 		//not serializable for text; also bufferpool rdd read only supported for 
 		// binarycell and binaryblock)
-		MatrixFormatMetaData iimd = (MatrixFormatMetaData) in.getMetaData();
-		if( in.getRDDHandle() != null 
+		MatrixFormatMetaData iimd = (MatrixFormatMetaData) obj.getMetaData();
+		if( obj.getRDDHandle() != null 
 			&& iimd.getInputInfo() != InputInfo.BinaryBlockInputInfo 
 			&& iimd.getInputInfo() != InputInfo.BinaryCellInputInfo ) {
 			return false;
@@ -2061,7 +2064,7 @@ public class Recompiler
 	 * @param out
 	 * @throws DMLRuntimeException 
 	 */
-	public static void executeInMemoryReblock(ExecutionContext ec, String varin, String varout) 
+	public static void executeInMemoryMatrixReblock(ExecutionContext ec, String varin, String varout) 
 		throws DMLRuntimeException
 	{
 		MatrixObject in = ec.getMatrixObject(varin);
@@ -2079,6 +2082,29 @@ public class Recompiler
 	
 	/**
 	 * 
+	 * @param ec
+	 * @param varin
+	 * @param varout
+	 * @throws DMLRuntimeException
+	 */
+	public static void executeInMemoryFrameReblock(ExecutionContext ec, String varin, String varout) 
+		throws DMLRuntimeException
+	{
+		FrameObject in = ec.getFrameObject(varin);
+		FrameObject out = ec.getFrameObject(varout);
+
+		//read text input frame (through buffer pool, frame object carries all relevant
+		//information including additional arguments for csv reblock)
+		FrameBlock fb = in.acquireRead(); 
+		
+		//set output (incl update matrix characteristics)
+		out.acquireModify( fb );
+		out.release();
+		in.release();				
+	}
+	
+	/**
+	 * 
 	 * @param fname
 	 * @return
 	 * @throws DMLRuntimeException

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9451a0fd/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
index c36c01f..e903a03 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
@@ -145,7 +145,7 @@ public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
  			hi = simplifyBushyBinaryOperation(hop, hi, i);       //e.g., (X*(Y*(Z%*%v))) -> (X*Y)*(Z%*%v)
  			hi = simplifyUnaryAggReorgOperation(hop, hi, i);     //e.g., sum(t(X)) -> sum(X)
  			hi = pushdownUnaryAggTransposeOperation(hop, hi, i); //e.g., colSums(t(X)) -> t(rowSums(X))
- 			hi = pushdownSumBinaryMult(hop, hi, i);				 //e.g., sum(lamda*X) -> lamda*sum(X)
+ 			hi = pushdownSumBinaryMult(hop, hi, i);              //e.g., sum(lamda*X) -> lamda*sum(X)
  			hi = simplifyUnaryPPredOperation(hop, hi, i);        //e.g., abs(ppred()) -> ppred(), others: round, ceil, floor
  			hi = simplifyTransposedAppend(hop, hi, i);           //e.g., t(cbind(t(A),t(B))) -> rbind(A,B);
  			hi = fuseBinarySubDAGToUnaryOperation(hop, hi, i);   //e.g., X*(1-X)-> sprop(X) || 1/(1+exp(-X)) -> sigmoid(X) || X*(X>0) -> selp(X)

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9451a0fd/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java
index ce09a6f..92dcd69 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java
@@ -27,6 +27,7 @@ import org.apache.sysml.conf.ConfigurationManager;
 import org.apache.sysml.hops.DataOp;
 import org.apache.sysml.hops.FunctionOp;
 import org.apache.sysml.hops.Hop;
+import org.apache.sysml.hops.OptimizerUtils;
 import org.apache.sysml.hops.Hop.DataOpTypes;
 import org.apache.sysml.hops.Hop.FileFormatTypes;
 import org.apache.sysml.hops.Hop.ParamBuiltinOp;
@@ -92,8 +93,9 @@ public class RewriteBlockSizeAndReblock extends HopRewriteRule
 		if (hop instanceof DataOp) 
 		{
 			// if block size does not match
-			if(    canReblock && hop.getDataType() == DataType.MATRIX
-				&& (hop.getRowsInBlock() != GLOBAL_BLOCKSIZE || hop.getColsInBlock() != GLOBAL_BLOCKSIZE) ) 
+			if( canReblock //TODO change frame condition to != BINARY once transform over frames supported
+				&& ((hop.getDataType() == DataType.MATRIX && (hop.getRowsInBlock() != GLOBAL_BLOCKSIZE || hop.getColsInBlock() != GLOBAL_BLOCKSIZE)
+				  ||(hop.getDataType() == DataType.FRAME && OptimizerUtils.isSparkExecutionMode() && ((DataOp)hop).getInputFormatType()==FileFormatTypes.TEXT)))) 
 			{
 				if (((DataOp) hop).getDataOpType() == DataOp.DataOpTypes.PERSISTENTREAD) 
 				{
@@ -141,6 +143,7 @@ public class RewriteBlockSizeAndReblock extends HopRewriteRule
 				}
 			}
 		} 
+		//TODO remove once transform rebased to frames
 		else if ( (hop instanceof ParameterizedBuiltinOp && ((ParameterizedBuiltinOp)hop).getOp() == ParamBuiltinOp.TRANSFORM) ) {
 			
 			// check if there exists a non-csv-write output. If yes, add reblock

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9451a0fd/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
index 84a74dd..b850301 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java
@@ -86,6 +86,10 @@ public class FrameObject extends CacheableData<FrameBlock>
 	public FrameObject(FrameObject fo) {
 		super(fo);
 	}
+	
+	public List<ValueType> getSchema() {
+		return _schema;
+	}
 
 	public void setSchema(String schema) {
 		if( schema.equals("*") ) {

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9451a0fd/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
index 020c49f..1516286 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
@@ -47,6 +47,7 @@ import org.apache.sysml.lops.Checkpoint;
 import org.apache.sysml.parser.Expression.ValueType;
 import org.apache.sysml.runtime.DMLRuntimeException;
 import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
 import org.apache.sysml.runtime.controlprogram.caching.FrameObject;
 import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
 import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
@@ -532,11 +533,8 @@ public class SparkExecutionContext extends ExecutionContext
 	}
 	
 	/**
-	 * Keep the output rdd of spark rdd operations as meta data of matrix objects in the 
-	 * symbol table.
-	 * 
-	 * Spark instructions should call this for all matrix outputs.
-	 * 
+	 * Keep the output rdd of spark rdd operations as meta data of matrix/frame 
+	 * objects in the symbol table.
 	 * 
 	 * @param varname
 	 * @param rdd
@@ -545,23 +543,9 @@ public class SparkExecutionContext extends ExecutionContext
 	public void setRDDHandleForVariable(String varname, JavaPairRDD<?,?> rdd) 
 		throws DMLRuntimeException
 	{
-		MatrixObject mo = getMatrixObject(varname);
-		RDDObject rddhandle = new RDDObject(rdd, varname);
-		mo.setRDDHandle( rddhandle );
-	}
-	
-	/**
-	 * 
-	 * @param varname
-	 * @param rdd
-	 * @throws DMLRuntimeException
-	 */
-	public void setFrameRDDHandleForVariable(String varname, JavaPairRDD<?,?> rdd) 
-		throws DMLRuntimeException
-	{
-		FrameObject mo = getFrameObject(varname);
+		CacheableData<?> obj = getCacheableData(varname);
 		RDDObject rddhandle = new RDDObject(rdd, varname);
-		mo.setRDDHandle( rddhandle );
+		obj.setRDDHandle( rddhandle );
 	}
 	
 	/**

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9451a0fd/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
index 6aa7597..98cc5a0 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CSVReblockSPInstruction.java
@@ -107,7 +107,7 @@ public class CSVReblockSPInstruction extends UnarySPInstruction
 
 		//check for in-memory reblock (w/ lazy spark context, potential for latency reduction)
 		if( Recompiler.checkCPReblock(sec, input1.getName()) ) {
-			Recompiler.executeInMemoryReblock(sec, input1.getName(), output.getName());
+			Recompiler.executeInMemoryMatrixReblock(sec, input1.getName(), output.getName());
 			return;
 		}
 		

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9451a0fd/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java
index 8160bd0..3562d18 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java
@@ -79,20 +79,17 @@ public class CastSPInstruction extends UnarySPInstruction
 			in = ((JavaPairRDD<Long, FrameBlock>)in).mapToPair(new LongFrameToLongWritableFrameFunction());
 			out = FrameRDDConverterUtils.binaryBlockToMatrixBlock(
 				(JavaPairRDD<LongWritable, FrameBlock>)in, mcIn, mcOut);
-			
-			sec.setRDDHandleForVariable(output.getName(), out);
 		}
 		else if( opcode.equals(UnaryCP.CAST_AS_FRAME_OPCODE) ) {
 			out = FrameRDDConverterUtils.matrixBlockToBinaryBlockLongIndex(sec.getSparkContext(), 
 				(JavaPairRDD<MatrixIndexes, MatrixBlock>)in, mcIn);
-		
-			sec.setFrameRDDHandleForVariable(output.getName(), out);
 		}
 		else {
 			throw new DMLRuntimeException("Unsupported spark cast operation: "+opcode);
 		}
 		
 		//update output statistics and add lineage
+		sec.setRDDHandleForVariable(output.getName(), out);
 		updateUnaryOutputMatrixCharacteristics(sec, input1.getName(), output.getName());
 		sec.addLineageRDD(output.getName(), input1.getName());
 	}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9451a0fd/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
index 000bd63..d0128f9 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReblockSPInstruction.java
@@ -23,18 +23,23 @@ import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.spark.api.java.JavaPairRDD;
 import org.apache.sysml.hops.recompile.Recompiler;
+import org.apache.sysml.parser.Expression.DataType;
 import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
+import org.apache.sysml.runtime.controlprogram.caching.FrameObject;
 import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
 import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
 import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
 import org.apache.sysml.runtime.instructions.InstructionUtils;
 import org.apache.sysml.runtime.instructions.cp.CPOperand;
 import org.apache.sysml.runtime.instructions.spark.functions.ExtractBlockForBinaryReblock;
+import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils;
 import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils;
 import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtils;
 import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
 import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
 import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
 import org.apache.sysml.runtime.matrix.data.InputInfo;
 import org.apache.sysml.runtime.matrix.data.MatrixBlock;
 import org.apache.sysml.runtime.matrix.data.MatrixCell;
@@ -77,38 +82,61 @@ public class ReblockSPInstruction extends UnarySPInstruction
 	
 
 	@Override
-	@SuppressWarnings("unchecked")
 	public void processInstruction(ExecutionContext ec)
 		throws DMLRuntimeException 
 	{
 		SparkExecutionContext sec = (SparkExecutionContext)ec;
 
 		//set the output characteristics
-		MatrixObject mo = sec.getMatrixObject(input1.getName());
+		CacheableData<?> obj = sec.getCacheableData(input1.getName());
 		MatrixCharacteristics mc = sec.getMatrixCharacteristics(input1.getName());
 		MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
 		mcOut.set(mc.getRows(), mc.getCols(), brlen, bclen, mc.getNonZeros());
-
+		
 		//get the source format form the meta data
-		MatrixFormatMetaData iimd = (MatrixFormatMetaData) mo.getMetaData();
-		if(iimd == null) {
+		MatrixFormatMetaData iimd = (MatrixFormatMetaData) obj.getMetaData();
+		if(iimd == null)
 			throw new DMLRuntimeException("Error: Metadata not found");
-		}
-		
+		InputInfo iinfo = iimd.getInputInfo();
+
 		//check for in-memory reblock (w/ lazy spark context, potential for latency reduction)
 		if( Recompiler.checkCPReblock(sec, input1.getName()) ) {
-			Recompiler.executeInMemoryReblock(sec, input1.getName(), output.getName());
+			if( input1.getDataType() == DataType.MATRIX )
+				Recompiler.executeInMemoryMatrixReblock(sec, input1.getName(), output.getName());
+			else if( input1.getDataType() == DataType.FRAME )
+				Recompiler.executeInMemoryFrameReblock(sec, input1.getName(), output.getName());	
 			return;
 		}
 		
-		if(iimd.getInputInfo() == InputInfo.TextCellInputInfo || iimd.getInputInfo() == InputInfo.MatrixMarketInputInfo ) 
+		//execute matrix/frame reblock
+		if( input1.getDataType() == DataType.MATRIX )
+			processMatrixReblockInstruction(sec, iinfo);
+		else if( input1.getDataType() == DataType.FRAME )
+			processFrameReblockInstruction(sec, iinfo);
+	}
+	
+	/**
+	 * 
+	 * @param sec
+	 * @param iinfo
+	 * @throws DMLRuntimeException
+	 */
+	@SuppressWarnings("unchecked")
+	protected void processMatrixReblockInstruction(SparkExecutionContext sec, InputInfo iinfo) 
+		throws DMLRuntimeException
+	{
+		MatrixObject mo = sec.getMatrixObject(input1.getName());
+		MatrixCharacteristics mc = sec.getMatrixCharacteristics(input1.getName());
+		MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
+		
+		if(iinfo == InputInfo.TextCellInputInfo || iinfo == InputInfo.MatrixMarketInputInfo ) 
 		{
 			//check jdk version (prevent double.parseDouble contention on <jdk8)
 			sec.checkAndRaiseValidationWarningJDKVersion();
 			
 			//get the input textcell rdd
 			JavaPairRDD<LongWritable, Text> lines = (JavaPairRDD<LongWritable, Text>) 
-					sec.getRDDHandleForVariable(input1.getName(), iimd.getInputInfo());
+					sec.getRDDHandleForVariable(input1.getName(), iinfo);
 			
 			//convert textcell to binary block
 			JavaPairRDD<MatrixIndexes, MatrixBlock> out = 
@@ -118,7 +146,7 @@ public class ReblockSPInstruction extends UnarySPInstruction
 			sec.setRDDHandleForVariable(output.getName(), out);
 			sec.addLineageRDD(output.getName(), input1.getName());
 		}
-		else if(iimd.getInputInfo() == InputInfo.CSVInputInfo) {
+		else if(iinfo == InputInfo.CSVInputInfo) {
 			// HACK ALERT: Until we introduces the rewrite to insert csvrblock for non-persistent read
 			// throw new DMLRuntimeException("CSVInputInfo is not supported for ReblockSPInstruction");
 			CSVReblockSPInstruction csvInstruction = null;
@@ -140,16 +168,16 @@ public class ReblockSPInstruction extends UnarySPInstruction
 			csvInstruction.processInstruction(sec);
 			return;
 		}
-		else if(iimd.getInputInfo()==InputInfo.BinaryCellInputInfo) 
+		else if(iinfo == InputInfo.BinaryCellInputInfo) 
 		{
-			JavaPairRDD<MatrixIndexes, MatrixCell> binaryCells = (JavaPairRDD<MatrixIndexes, MatrixCell>) sec.getRDDHandleForVariable(input1.getName(), iimd.getInputInfo());
+			JavaPairRDD<MatrixIndexes, MatrixCell> binaryCells = (JavaPairRDD<MatrixIndexes, MatrixCell>) sec.getRDDHandleForVariable(input1.getName(), iinfo);
 			JavaPairRDD<MatrixIndexes, MatrixBlock> out = RDDConverterUtils.binaryCellToBinaryBlock(sec.getSparkContext(), binaryCells, mcOut, outputEmptyBlocks);
 			
 			//put output RDD handle into symbol table
 			sec.setRDDHandleForVariable(output.getName(), out);
 			sec.addLineageRDD(output.getName(), input1.getName());
 		}
-		else if(iimd.getInputInfo()==InputInfo.BinaryBlockInputInfo) 
+		else if(iinfo == InputInfo.BinaryBlockInputInfo) 
 		{
 			/// HACK ALERT: Workaround for MLContext 
 			if(mc.getRowsPerBlock() == mcOut.getRowsPerBlock() && mc.getColsPerBlock() == mcOut.getColsPerBlock()) {
@@ -162,7 +190,7 @@ public class ReblockSPInstruction extends UnarySPInstruction
 					return;
 				}
 				else {
-					throw new DMLRuntimeException("Input RDD is not accessible through buffer pool for ReblockSPInstruction:" + iimd.getInputInfo());
+					throw new DMLRuntimeException("Input RDD is not accessible through buffer pool for ReblockSPInstruction:" + iinfo);
 				}
 			}
 			else 
@@ -180,7 +208,42 @@ public class ReblockSPInstruction extends UnarySPInstruction
 			}
 		}
 		else {
-			throw new DMLRuntimeException("The given InputInfo is not implemented for ReblockSPInstruction:" + iimd.getInputInfo());
-		}		
+			throw new DMLRuntimeException("The given InputInfo is not implemented for ReblockSPInstruction:" + iinfo);
+		}
+	}
+	
+	/**
+	 * 
+	 * @param sec
+	 * @param iinfo
+	 * @throws DMLRuntimeException
+	 */
+	@SuppressWarnings("unchecked")
+	protected void processFrameReblockInstruction(SparkExecutionContext sec, InputInfo iinfo) 
+		throws DMLRuntimeException
+	{
+		FrameObject fo = sec.getFrameObject(input1.getName());
+		MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
+		
+		if(iinfo == InputInfo.TextCellInputInfo ) 
+		{
+			//check jdk version (prevent double.parseDouble contention on <jdk8)
+			sec.checkAndRaiseValidationWarningJDKVersion();
+			
+			//get the input textcell rdd
+			JavaPairRDD<LongWritable, Text> lines = (JavaPairRDD<LongWritable, Text>) 
+					sec.getRDDHandleForVariable(input1.getName(), iinfo);
+			
+			//convert textcell to binary block
+			JavaPairRDD<Long, FrameBlock> out = 
+					FrameRDDConverterUtils.textCellToBinaryBlock(sec.getSparkContext(), lines, mcOut, fo.getSchema());
+			
+			//put output RDD handle into symbol table
+			sec.setRDDHandleForVariable(output.getName(), out);
+			sec.addLineageRDD(output.getName(), input1.getName());
+		}
+		else {
+			throw new DMLRuntimeException("The given InputInfo is not implemented for ReblockSPInstruction:" + iinfo);
+		}
 	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9451a0fd/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
index 003b016..7c8c08b 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
@@ -22,6 +22,7 @@ package org.apache.sysml.runtime.instructions.spark.utils;
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
@@ -161,23 +162,20 @@ public class FrameRDDConverterUtils
 	 * @return
 	 * @throws DMLRuntimeException
 	 */
-	public static JavaPairRDD<LongWritable, FrameBlock> textCellToBinaryBlock(JavaSparkContext sc,
+	public static JavaPairRDD<Long, FrameBlock> textCellToBinaryBlock(JavaSparkContext sc,
 			JavaPairRDD<LongWritable, Text> in, MatrixCharacteristics mcOut, List<ValueType> schema ) 
 		throws DMLRuntimeException  
 	{
+		//replicate schema entry if necessary
+		List<ValueType> lschema = (schema.size()==1 && mcOut.getCols()>1) ?
+				Collections.nCopies((int)mcOut.getCols(), schema.get(0)) : schema;
 		
 		//convert input rdd to serializable long/frame block
 		JavaPairRDD<Long,Text> input = 
 				in.mapToPair(new LongWritableTextToLongTextFunction());
 		
-		//Do actual conversion
-		JavaPairRDD<Long,FrameBlock> output = textCellToBinaryBlockLongIndex(sc, input, mcOut, schema);
-		
-		//convert input rdd to serializable long/frame block
-		JavaPairRDD<LongWritable,FrameBlock> out = 
-				output.mapToPair(new LongFrameToLongWritableFrameFunction());
-		
-		return out;
+		//do actual conversion
+		return textCellToBinaryBlockLongIndex(sc, input, mcOut, lschema);
 	}
 
 		
@@ -736,13 +734,12 @@ public class FrameRDDConverterUtils
 	{
 		private static final long serialVersionUID = -2654986510471835933L;
 		
-		MatrixCharacteristics _mcIn, _mcOut;
+		private MatrixCharacteristics _mcIn;
+		private MatrixCharacteristics _mcOut;
 
-		public BinaryBlockToMatrixBlockFunction(MatrixCharacteristics mcIn,
-				MatrixCharacteristics mcOut) {
-				
-				_mcIn = mcIn;		//Frame Characteristics
-				_mcOut = mcOut;		//Matrix Characteristics
+		public BinaryBlockToMatrixBlockFunction(MatrixCharacteristics mcIn, MatrixCharacteristics mcOut) {			
+			_mcIn = mcIn;		//Frame Characteristics
+			_mcOut = mcOut;		//Matrix Characteristics
 		}
 
 		@Override
@@ -753,46 +750,35 @@ public class FrameRDDConverterUtils
 			FrameBlock blk = arg0._2();
 			
 			ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes, MatrixBlock>>();
+			long rlen = _mcIn.getRows();
+			long clen = _mcIn.getCols();
+			int brlen = _mcOut.getRowsPerBlock();
+			int bclen = _mcOut.getColsPerBlock();
 			
-			int _brlenMatrix = _mcOut.getRowsPerBlock();
-			int _bclenMatrix = _mcOut.getColsPerBlock();
-			long _rlen = _mcIn.getRows();
-			long _clen = _mcIn.getCols();
-			
-			long lRowId = 0;
-			while (lRowId < blk.getNumRows()) {
-				// Global Row indices (indexes) across all frame blocks  
-				long endRow = ((rowIndex+lRowId-1)/_brlenMatrix+1) * _brlenMatrix;
-				long begRow = Math.max(endRow-_brlenMatrix+1, 0);
-				endRow = Math.min(endRow, _rlen);
-				
-				// Local Row indices (indexes) within a matrix block  
-				long begRowMat = UtilFunctions.computeCellInBlock(begRow, _brlenMatrix);
-				long endRowMat = UtilFunctions.computeCellInBlock(endRow, _brlenMatrix);
-				
-				long lColId = 0;
-				while (lColId < blk.getNumColumns()) {
-					// Global Column index across all frame blocks  
-					long endCol = Math.min(lColId+_bclenMatrix-1, _clen-1);
-
-					// Local Column indices (indexes) within a matrix block  
-					long begColMat = UtilFunctions.computeCellInBlock(lColId+1, _bclenMatrix);
-					long endColMat = UtilFunctions.computeCellInBlock(endCol+1, _bclenMatrix);
-
-					FrameBlock tmpFrame = new FrameBlock();
-					tmpFrame = blk.sliceOperations((int)lRowId, (int)(lRowId+endRowMat-begRowMat), (int)lColId, (int)endCol, tmpFrame);
-
-					MatrixIndexes matrixIndexes = new MatrixIndexes(UtilFunctions.computeBlockIndex(begRow+1, _brlenMatrix),UtilFunctions.computeBlockIndex(lColId+1, _bclenMatrix));
-
-					MatrixBlock matrixBlocktmp = DataConverter.convertToMatrixBlock(tmpFrame);
-					MatrixBlock matrixBlock = matrixBlocktmp.leftIndexingOperations(matrixBlocktmp, (int)begRowMat, (int)endRowMat, (int)begColMat, (int)endColMat, new MatrixBlock(), UpdateType.INPLACE_PINNED);
-					ret.add(new Tuple2<MatrixIndexes, MatrixBlock>(matrixIndexes, matrixBlock));
-					
-					lColId = endCol+1;
+			//slice aligned matrix blocks out of given frame block
+			long rstartix = UtilFunctions.computeBlockIndex(rowIndex, brlen);
+			long rendix = UtilFunctions.computeBlockIndex(rowIndex+blk.getNumRows()-1, brlen);
+			long cendix = UtilFunctions.computeBlockIndex(blk.getNumColumns(), bclen);
+			for( long rix=rstartix; rix<=rendix; rix++ ) { //for all row blocks
+				long rpos = UtilFunctions.computeCellIndex(rix, brlen, 0);
+				int lrlen = UtilFunctions.computeBlockSize(rlen, rix, brlen);
+				int fix = (int)((rpos-rowIndex>=0) ? rpos-rowIndex : 0);
+				int fix2 = (int)Math.min(rpos+lrlen-rowIndex-1,blk.getNumRows()-1);
+				int mix = UtilFunctions.computeCellInBlock(rowIndex+fix, brlen);
+				int mix2 = mix + (fix2-fix);
+				for( long cix=1; cix<=cendix; cix++ ) { //for all column blocks
+					long cpos = UtilFunctions.computeCellIndex(cix, bclen, 0);
+					int lclen = UtilFunctions.computeBlockSize(clen, cix, bclen);
+					MatrixBlock matrix = new MatrixBlock(lrlen, lclen, false);
+					FrameBlock frame = blk.sliceOperations(fix, fix2, 
+							(int)cpos-1, (int)cpos+lclen-2, new FrameBlock());
+					MatrixBlock mframe = DataConverter.convertToMatrixBlock(frame);
+					ret.add(new Tuple2<MatrixIndexes, MatrixBlock>(new MatrixIndexes(rix, cix), 
+							matrix.leftIndexingOperations(mframe, mix, mix2, 0, lclen-1, 
+							new MatrixBlock(), UpdateType.INPLACE_PINNED)));
 				}
-				lRowId += (endRow-begRow+1);
 			}
-			
+
 			return ret;
 		}
 	}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9451a0fd/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
index 97ac27c..d0a7f88 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
@@ -27,7 +27,6 @@ import java.util.List;
 
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.JavaPairRDD;
 import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.api.java.JavaSparkContext;
@@ -39,6 +38,7 @@ import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
 import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
 import org.apache.sysml.runtime.instructions.spark.functions.CopyFrameBlockPairFunction;
 import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils;
+import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils.LongFrameBlockToLongWritableFrameBlock;
 import org.apache.sysml.runtime.io.FrameReader;
 import org.apache.sysml.runtime.io.FrameReaderFactory;
 import org.apache.sysml.runtime.io.FrameWriter;
@@ -191,9 +191,6 @@ public class FrameConverterTest extends AutomatedTestBase
 		boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
 		DMLScript.USE_LOCAL_SPARK_CONFIG = true;
 
-		SparkConf conf = new SparkConf().setAppName("Frame").setMaster("local");
-		conf.set("spark.kryo.classesToRegister", "org.apache.hadoop.io.LongWritable");
-
 		try
 		{
 			TestConfiguration config = getTestConfiguration(TEST_NAME);
@@ -461,7 +458,8 @@ public class FrameConverterTest extends AutomatedTestBase
 				OutputInfo oinfo = OutputInfo.BinaryBlockOutputInfo;
 				JavaPairRDD<LongWritable,Text> rddIn = sc.hadoopFile(fnameIn, iinfo.inputFormatClass, iinfo.inputKeyClass, iinfo.inputValueClass);
 				JavaPairRDD<LongWritable, FrameBlock> rddOut = FrameRDDConverterUtils
-						.textCellToBinaryBlock(sc, rddIn, mc, schema);
+						.textCellToBinaryBlock(sc, rddIn, mc, schema)
+						.mapToPair(new LongFrameBlockToLongWritableFrameBlock());
 				rddOut.saveAsHadoopFile(fnameOut, LongWritable.class, FrameBlock.class, oinfo.outputFormatClass);
 				break;
 			}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9451a0fd/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameMatrixReblockTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameMatrixReblockTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameMatrixReblockTest.java
new file mode 100644
index 0000000..7da887e
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameMatrixReblockTest.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.frame;
+
+import java.io.IOException;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.io.FrameWriter;
+import org.apache.sysml.runtime.io.FrameWriterFactory;
+import org.apache.sysml.runtime.io.MatrixReader;
+import org.apache.sysml.runtime.io.MatrixReaderFactory;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.junit.Test;
+
+/**
+ * 
+ */
+public class FrameMatrixReblockTest extends AutomatedTestBase
+{
+	private final static String TEST_DIR = "functions/frame/";
+	private final static String TEST_NAME1 = "FrameMatrixReblock";
+	private final static String TEST_CLASS_DIR = TEST_DIR + FrameMatrixReblockTest.class.getSimpleName() + "/";
+
+	private final static int rows = 2593;
+	private final static int cols1 = 372;
+	private final static int cols2 = 1102;
+	private final static double sparsity1 = 0.9;
+	private final static double sparsity2 = 0.3;
+	
+	@Override
+	public void setUp() {
+		TestUtils.clearAssertionInformation();
+		addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"B"}));
+	}
+	
+	@Test
+	public void testFrameWriteSingleDenseBinaryCP() {
+		runFrameReblockTest(TEST_NAME1, false, false, "binary", ExecType.CP);
+	}
+	
+	@Test
+	public void testFrameWriteSingleDenseTextcellCP() {
+		runFrameReblockTest(TEST_NAME1, false, false, "text", ExecType.CP);
+	}
+	
+	@Test
+	public void testFrameWriteSingleDenseCsvCP() {
+		runFrameReblockTest(TEST_NAME1, false, false, "csv", ExecType.CP);
+	}
+	
+	@Test
+	public void testFrameWriteMultipleDenseBinaryCP() {
+		runFrameReblockTest(TEST_NAME1, true,  false, "binary", ExecType.CP);
+	}
+	
+	@Test
+	public void testFrameWriteMultipleDenseTextcellCP() {
+		runFrameReblockTest(TEST_NAME1, true, false, "text", ExecType.CP);
+	}
+	
+	@Test
+	public void testFrameWriteMultipleDenseCsvCP() {
+		runFrameReblockTest(TEST_NAME1, true, false, "csv", ExecType.CP);
+	}
+
+	@Test
+	public void testFrameWriteSingleDenseBinarySpark() {
+		runFrameReblockTest(TEST_NAME1, false, false, "binary", ExecType.SPARK);
+	}
+	
+	@Test
+	public void testFrameWriteSingleDenseTextcellSpark() {
+		runFrameReblockTest(TEST_NAME1, false, false, "text", ExecType.SPARK);
+	}
+
+//TODO enable csv spark tests once transform over frame supported	
+//	@Test
+//	public void testFrameWriteSingleDenseCsvSpark() {
+//		runFrameReblockTest(TEST_NAME1, false, false, "csv", ExecType.SPARK);
+//	}
+	
+	@Test
+	public void testFrameWriteMultipleDenseBinarySpark() {
+		runFrameReblockTest(TEST_NAME1, true, false, "binary", ExecType.SPARK);
+	}
+	
+	@Test
+	public void testFrameWriteMultipleDenseTextcellSpark() {
+		runFrameReblockTest(TEST_NAME1, true, false, "text", ExecType.SPARK);
+	}
+	
+//	@Test
+//	public void testFrameWriteMultipleDenseCsvSpark() {
+//		runFrameReblockTest(TEST_NAME1, true, false, "csv", ExecType.SPARK);
+//	}
+	
+	@Test
+	public void testFrameWriteSingleSparseBinaryCP() {
+		runFrameReblockTest(TEST_NAME1, false, true, "binary", ExecType.CP);
+	}
+	
+	@Test
+	public void testFrameWriteSingleSparseTextcellCP() {
+		runFrameReblockTest(TEST_NAME1, false, true, "text", ExecType.CP);
+	}
+	
+	@Test
+	public void testFrameWriteSingleSparseCsvCP() {
+		runFrameReblockTest(TEST_NAME1, false, true, "csv", ExecType.CP);
+	}
+	
+	@Test
+	public void testFrameWriteMultipleSparseBinaryCP() {
+		runFrameReblockTest(TEST_NAME1, true, true, "binary", ExecType.CP);
+	}
+	
+	@Test
+	public void testFrameWriteMultipleSparseTextcellCP() {
+		runFrameReblockTest(TEST_NAME1, true, true, "text", ExecType.CP);
+	}
+	
+	@Test
+	public void testFrameWriteMultipleSparseCsvCP() {
+		runFrameReblockTest(TEST_NAME1, true, true, "csv", ExecType.CP);
+	}
+
+	@Test
+	public void testFrameWriteSingleSparseBinarySpark() {
+		runFrameReblockTest(TEST_NAME1, false, true, "binary", ExecType.SPARK);
+	}
+	
+	@Test
+	public void testFrameWriteSingleSparseTextcellSpark() {
+		runFrameReblockTest(TEST_NAME1, false, true, "text", ExecType.SPARK);
+	}
+	
+//	@Test
+//	public void testFrameWriteSingleSparseCsvSpark() {
+//		runFrameReblockTest(TEST_NAME1, false, true, "csv", ExecType.SPARK);
+//	}
+	
+	@Test
+	public void testFrameWriteMultipleSparseBinarySpark() {
+		runFrameReblockTest(TEST_NAME1, true, true, "binary", ExecType.SPARK);
+	}
+	
+	@Test
+	public void testFrameWriteMultipleSparseTextcellSpark() {
+		runFrameReblockTest(TEST_NAME1, true, true, "text", ExecType.SPARK);
+	}
+	
+//	@Test
+//	public void testFrameWriteMultipleSparseCsvSpark() {
+//		runFrameReblockTest(TEST_NAME1, true, true, "csv", ExecType.SPARK);
+//	}
+	
+	/**
+	 * 
+	 * @param testname
+	 * @param multColBlks
+	 * @param ofmt
+	 * @param et
+	 */
+	private void runFrameReblockTest( String testname, boolean multColBlks, boolean sparse, String ofmt, ExecType et)
+	{
+		//rtplatform for MR
+		RUNTIME_PLATFORM platformOld = rtplatform;
+		switch( et ){
+			case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break;
+			case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
+			default: rtplatform = RUNTIME_PLATFORM.HYBRID; break;
+		}
+	
+		boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+		if( rtplatform == RUNTIME_PLATFORM.SPARK )
+			DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+		
+		try
+		{
+			int cols = multColBlks ? cols2 : cols1;
+			double sparsity = sparse ? sparsity2 : sparsity1;
+			
+			TestConfiguration config = getTestConfiguration(testname);
+			loadTestConfiguration(config);
+			
+			String HOME = SCRIPT_DIR + TEST_DIR;
+			fullDMLScriptName = HOME + testname + ".dml";
+			programArgs = new String[]{"-explain","-args", input("A"), String.valueOf(rows), 
+					String.valueOf(cols), output("B"), ofmt };
+			
+			//generate input data
+			double[][] A = getRandomMatrix(rows, cols, -1, 1, sparsity, 7);
+			writeFrameInput(input("A"), ofmt, A, rows, cols);
+			
+			//run testcase
+			runTest(true, false, null, -1);
+			
+			//compare matrices
+			double[][] B = readMatrixOutput(output("B"), ofmt, rows, cols);
+			TestUtils.compareMatrices(A, B, rows, cols, 0);
+		}
+		catch(Exception ex) {
+			throw new RuntimeException(ex);
+		}
+		finally {
+			rtplatform = platformOld;
+			DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+		}
+	}
+	
+	/**
+	 * 
+	 * @param fname
+	 * @param ofmt
+	 * @param frame
+	 * @param rows
+	 * @param cols
+	 * @throws DMLRuntimeException
+	 * @throws IOException
+	 */
+	private void writeFrameInput(String fname, String ofmt, double[][] frame, int rows, int cols) 
+		throws DMLRuntimeException, IOException 
+	{
+		MatrixBlock mb = DataConverter.convertToMatrixBlock(frame);
+		FrameBlock fb = DataConverter.convertToFrameBlock(mb);
+		
+		//write input data
+		FrameWriter writer = FrameWriterFactory.createFrameWriter(
+				InputInfo.getMatchingOutputInfo(InputInfo.stringExternalToInputInfo(ofmt)));
+		writer.writeFrameToHDFS(fb, fname, rows, cols);
+	}
+	
+	/**
+	 * 
+	 * @param fname
+	 * @param rows
+	 * @param cols
+	 * @param ofmt
+	 * @return
+	 * @throws DMLRuntimeException
+	 * @throws IOException
+	 */
+	private double[][] readMatrixOutput(String fname, String ofmt, int rows, int cols) 
+		throws DMLRuntimeException, IOException 
+	{
+		MatrixReader reader = MatrixReaderFactory.createMatrixReader(InputInfo.stringExternalToInputInfo(ofmt));
+		MatrixBlock mb = reader.readMatrixFromHDFS(fname, rows, cols, 1000, 1000, -1);
+		
+		return DataConverter.convertToDoubleMatrix(mb); 
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9451a0fd/src/test/java/org/apache/sysml/test/integration/functions/misc/RewritePushdownSumBinaryMult.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/RewritePushdownSumBinaryMult.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/RewritePushdownSumBinaryMult.java
index 9724d1d..bd821b1 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/RewritePushdownSumBinaryMult.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/RewritePushdownSumBinaryMult.java
@@ -21,14 +21,12 @@ package org.apache.sysml.test.integration.functions.misc;
 
 import java.util.HashMap;
 
-import org.junit.Assert;
 import org.junit.Test;
 import org.apache.sysml.hops.OptimizerUtils;
 import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
 import org.apache.sysml.test.integration.AutomatedTestBase;
 import org.apache.sysml.test.integration.TestConfiguration;
 import org.apache.sysml.test.utils.TestUtils;
-import org.apache.sysml.utils.Statistics;
 
 /**
  * Regression test for function recompile-once issue with literal replacement.
@@ -36,57 +34,44 @@ import org.apache.sysml.utils.Statistics;
  */
 public class RewritePushdownSumBinaryMult extends AutomatedTestBase 
 {
-	
 	private static final String TEST_NAME1 = "RewritePushdownSumBinaryMult";
 	private static final String TEST_NAME2 = "RewritePushdownSumBinaryMult2";
 
 	private static final String TEST_DIR = "functions/misc/";
 	private static final String TEST_CLASS_DIR = TEST_DIR + RewritePushdownSumBinaryMult.class.getSimpleName() + "/";
 	
-	//private static final int rows = 1234;
-	//private static final int cols = 567;
-	private static final double eps = Math.pow(10, -10);
-	
 	@Override
-	public void setUp() 
-	{
+	public void setUp() {
 		TestUtils.clearAssertionInformation();
 		addTestConfiguration( TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "R" }) );
 		addTestConfiguration( TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] { "R" }) );
 	}
 	
 	@Test
-	public void testPushdownSumBinaryMultNoRewrite() 
-	{
+	public void testPushdownSumBinaryMultNoRewrite() {
 		testRewritePushdownSumBinaryMult( TEST_NAME1, false );
 	}
 	
 	
 	@Test
-	public void testPushdownSumBinaryMultRewrite() 
-	{
+	public void testPushdownSumBinaryMultRewrite() {
 		testRewritePushdownSumBinaryMult( TEST_NAME1, true );
 	}
 	
-	
 	@Test
-	public void testPushdownSumBinaryMultNoRewrite2() 
-	{
+	public void testPushdownSumBinaryMultNoRewrite2() {
 		testRewritePushdownSumBinaryMult( TEST_NAME2, false );
 	}
 	
 	@Test
-	public void testPushdownSumBinaryMultRewrite2() 
-	{
+	public void testPushdownSumBinaryMultRewrite2() {
 		testRewritePushdownSumBinaryMult( TEST_NAME2, true );
 	}
 	
-	
 	/**
 	 * 
-	 * @param condition
-	 * @param branchRemoval
-	 * @param IPA
+	 * @param testname
+	 * @param rewrites
 	 */
 	private void testRewritePushdownSumBinaryMult( String testname, boolean rewrites )
 	{	
@@ -97,7 +82,6 @@ public class RewritePushdownSumBinaryMult extends AutomatedTestBase
 			TestConfiguration config = getTestConfiguration(testname);
 			loadTestConfiguration(config);
 			
-			
 			String HOME = SCRIPT_DIR + TEST_DIR;
 			fullDMLScriptName = HOME + testname + ".dml";
 			programArgs = new String[]{ "-stats","-args", output("Scalar") };
@@ -114,13 +98,9 @@ public class RewritePushdownSumBinaryMult extends AutomatedTestBase
 			HashMap<CellIndex, Double> dmlfile = readDMLScalarFromHDFS("Scalar");
 			HashMap<CellIndex, Double> rfile  = readRScalarFromFS("Scalar");
 			TestUtils.compareScalars(dmlfile.toString(), rfile.toString());
-			System.out.println("Test case passed");
-			
 		}
-		finally
-		{
+		finally {
 			OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag;
-		}
-		
+		}	
 	}	
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9451a0fd/src/test/scripts/functions/frame/FrameMatrixReblock.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/frame/FrameMatrixReblock.dml b/src/test/scripts/functions/frame/FrameMatrixReblock.dml
new file mode 100644
index 0000000..67eee82
--- /dev/null
+++ b/src/test/scripts/functions/frame/FrameMatrixReblock.dml
@@ -0,0 +1,25 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+A = read($1, rows=$2, cols=$3, data_type="frame", schema="double", format=$5);
+
+B = as.matrix(A);
+write(B, $4, format=$5);