You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by ac...@apache.org on 2016/04/25 06:38:28 UTC

incubator-systemml git commit: [SYSTEMML-560] Frame converter between TextCell and Binary block

Repository: incubator-systemml
Updated Branches:
  refs/heads/master e5aaaf1e8 -> b1b9e838d


[SYSTEMML-560] Frame converter between TextCell and Binary block


Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/b1b9e838
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/b1b9e838
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/b1b9e838

Branch: refs/heads/master
Commit: b1b9e838d81d7ade1a62aaa60b18960fbea7f23b
Parents: e5aaaf1
Author: Arvind Surve <ac...@yahoo.com>
Authored: Sun Apr 24 21:36:59 2016 -0700
Committer: Arvind Surve <ac...@yahoo.com>
Committed: Sun Apr 24 21:36:59 2016 -0700

----------------------------------------------------------------------
 .../functions/ConvertFrameBlockToIJVLines.java  |  55 +++
 .../spark/utils/FrameRDDConverterUtils.java     | 255 ++++++++++++--
 .../spark/utils/RDDAggregateUtils.java          |  41 ++-
 .../instructions/spark/utils/SparkUtils.java    |  28 ++
 .../runtime/io/FrameReaderBinaryBlock.java      |   5 +-
 .../sysml/runtime/matrix/data/FrameBlock.java   |  43 +++
 .../matrix/mapred/FrameReblockBuffer.java       | 351 +++++++++++++++++++
 .../functions/frame/FrameConverterTest.java     |  55 ++-
 8 files changed, 803 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b1b9e838/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertFrameBlockToIJVLines.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertFrameBlockToIJVLines.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertFrameBlockToIJVLines.java
new file mode 100644
index 0000000..b8f8e11
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertFrameBlockToIJVLines.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sysml.runtime.instructions.spark.functions;
+
+import java.util.ArrayList;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.spark.api.java.function.FlatMapFunction;
+
+import scala.Tuple2;
+
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+
+public class ConvertFrameBlockToIJVLines implements FlatMapFunction<Tuple2<LongWritable,FrameBlock>, String> {
+
+	private static final long serialVersionUID = 1803516615963340115L;
+
+	int brlen; int bclen;
+	public ConvertFrameBlockToIJVLines(int brlen, int bclen) {
+		this.brlen = brlen;
+		this.bclen = bclen;
+	}
+
+	@Override
+	public Iterable<String> call(Tuple2<LongWritable, FrameBlock> kv) throws Exception {
+
+		long lRowIndex = kv._1.get();
+		FrameBlock block = kv._2;
+		
+		ArrayList<String> cells = new ArrayList<String>();
+		
+		for (int i=0; i<block.getNumRows(); ++i)
+			for (int j=0; j<block.getNumColumns(); ++j) {
+				Object obj = block.get(i, j);
+				if(obj != null)
+					cells.add(lRowIndex+i+" "+(j+1)+" "+obj.toString());
+			}
+		return cells;
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b1b9e838/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
index 342f1df..02134c8 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
@@ -19,8 +19,11 @@
 
 package org.apache.sysml.runtime.instructions.spark.utils;
 
+import java.io.IOException;
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
@@ -38,39 +41,21 @@ import org.apache.sysml.parser.Expression.ValueType;
 import org.apache.sysml.runtime.DMLRuntimeException;
 import org.apache.sysml.runtime.instructions.spark.data.SerLongWritable;
 import org.apache.sysml.runtime.instructions.spark.data.SerText;
+import org.apache.sysml.runtime.instructions.spark.functions.ConvertFrameBlockToIJVLines;
 import org.apache.sysml.runtime.io.IOUtilFunctions;
 import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
 import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
 import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.Pair;
+import org.apache.sysml.runtime.matrix.mapred.FrameReblockBuffer;
+import org.apache.sysml.runtime.util.FastStringTokenizer;
+import org.apache.sysml.runtime.util.UtilFunctions;
 
 public class FrameRDDConverterUtils 
 {
-	/**
-	 * 
-	 * @param in
-	 * @param mcIn
-	 * @param props
-	 * @param strict
-	 * @return
-	 */
-	public static JavaRDD<String> binaryBlockToCsv(JavaPairRDD<LongWritable,FrameBlock> in, MatrixCharacteristics mcIn, CSVFileFormatProperties props, boolean strict)
-	{
-		//convert input rdd to serializable long/frame block
-		JavaPairRDD<LongWritable,FrameBlock> input = 
-				in.mapToPair(new LongWritableToSerFunction());
-		
-		//sort if required (on blocks/rows)
-		if( strict ) {
-			input = input.sortByKey(true);
-		}
-		
-		//convert binary block to csv (from blocks/rows)
-		JavaRDD<String> out = input
-				.flatMap(new BinaryBlockToCSVFunction(props));
-	
-		return out;
-	}
-	
+	//=====================================
+	// CSV <--> Binary block
+
 	/**
 	 * 
 	 * @param sc
@@ -135,6 +120,104 @@ public class FrameRDDConverterUtils
 	
 	/**
 	 * 
+	 * @param in
+	 * @param mcIn
+	 * @param props
+	 * @param strict
+	 * @return
+	 */
+	public static JavaRDD<String> binaryBlockToCsv(JavaPairRDD<LongWritable,FrameBlock> in, MatrixCharacteristics mcIn, CSVFileFormatProperties props, boolean strict)
+	{
+		//convert input rdd to serializable long/frame block
+		JavaPairRDD<LongWritable,FrameBlock> input = 
+				in.mapToPair(new LongWritableToSerFunction());
+		
+		//sort if required (on blocks/rows)
+		if( strict ) {
+			input = input.sortByKey(true);
+		}
+		
+		//convert binary block to csv (from blocks/rows)
+		JavaRDD<String> out = input
+				.flatMap(new BinaryBlockToCSVFunction(props));
+	
+		return out;
+	}
+	
+	
+	//=====================================
+	// cellText <--> Binary block
+	/**
+	 * 
+	 * @param sc
+	 * @param input
+	 * @param mcOut
+	 * @param schema
+	 * @return
+	 * @throws DMLRuntimeException
+	 */
+	public static JavaPairRDD<LongWritable, FrameBlock> textCellToBinaryBlock(JavaSparkContext sc,
+			JavaPairRDD<LongWritable, Text> in, MatrixCharacteristics mcOut, List<ValueType> schema ) 
+		throws DMLRuntimeException  
+	{
+		
+		//convert input rdd to serializable long/frame block
+		JavaPairRDD<Long,Text> input = 
+				in.mapToPair(new LongWritableTextToLongTextFunction());
+		
+		//Do actual conversion
+		JavaPairRDD<Long,FrameBlock> output = textCellToBinaryBlockLongIndex(sc, input, mcOut, schema);
+		
+		//convert input rdd to serializable long/frame block
+		JavaPairRDD<LongWritable,FrameBlock> out = 
+				output.mapToPair(new LongFrameToLongWritableFrameFunction());
+		
+		return out;
+	}
+
+		
+	/**
+	 * 
+	 * @param sc
+	 * @param input
+	 * @param mcOut
+	 * @param schema
+	 * @return
+	 * @throws DMLRuntimeException
+	 */
+	public static JavaPairRDD<Long, FrameBlock> textCellToBinaryBlockLongIndex(JavaSparkContext sc,
+			JavaPairRDD<Long, Text> input, MatrixCharacteristics mcOut, List<ValueType> schema ) 
+		throws DMLRuntimeException  
+	{
+		
+ 		//convert textcell rdd to binary block rdd (w/ partial blocks)
+		JavaPairRDD<Long, FrameBlock> output = input.values().mapPartitionsToPair(new TextToBinaryBlockFunction( mcOut, schema ));
+		
+		//aggregate partial matrix blocks
+		JavaPairRDD<Long,FrameBlock> out = 
+				RDDAggregateUtils.mergeByFrameKey( output ); 
+
+		return out;
+	}
+
+		
+	// Useful for printing, testing binary blocked RDD and also for external use.
+	public static JavaRDD<String> binaryBlockToStringRDD(JavaPairRDD<LongWritable, FrameBlock> input, MatrixCharacteristics mcIn, String format) throws DMLRuntimeException {
+		if(format.equals("text")) {
+			JavaRDD<String> ijv = input.flatMap(new ConvertFrameBlockToIJVLines(mcIn.getRowsPerBlock(), mcIn.getColsPerBlock()));
+			return ijv;
+		}
+		else {
+			throw new DMLRuntimeException("The output format:" + format + " is not implemented yet.");
+		}
+	}
+	
+
+	/////////////////////////////////
+	// CSV-SPECIFIC FUNCTIONS
+	
+	/**
+	 * 
 	 */
 	private static class StringToSerTextFunction implements PairFunction<String, LongWritable, Text> 
 	{
@@ -162,6 +245,36 @@ public class FrameRDDConverterUtils
 	/**
 	 * 
 	 */
+	public static class LongWritableTextToLongTextFunction implements PairFunction<Tuple2<LongWritable,Text>,Long,Text> 
+	{
+		private static final long serialVersionUID = -5408386071466175348L;
+
+		@Override
+		public Tuple2<Long, Text> call(Tuple2<LongWritable, Text> arg0) throws Exception  {
+			return new Tuple2<Long,Text>(new Long(arg0._1.get()), arg0._2);
+		}
+	}
+	
+	
+	
+	/**
+	 * 
+	 */
+	public static class LongFrameToLongWritableFrameFunction implements PairFunction<Tuple2<Long,FrameBlock>,LongWritable,FrameBlock> 
+	{
+
+		private static final long serialVersionUID = -1467314923206783333L;
+
+		@Override
+		public Tuple2<LongWritable, FrameBlock> call(Tuple2<Long, FrameBlock> arg0) throws Exception  {
+			return new Tuple2<LongWritable, FrameBlock>(new LongWritable(arg0._1), arg0._2);
+		}
+	}
+
+	
+	/**
+	 * 
+	 */
 	private static class TextToStringFunction implements Function<Text,String> 
 	{
 		private static final long serialVersionUID = -2744814934501782747L;
@@ -316,4 +429,94 @@ public class FrameRDDConverterUtils
 			return ret;
 		}
 	}
+	/////////////////////////////////
+	// TEXTCELL-SPECIFIC FUNCTIONS
+	
+	private static abstract class CellToBinaryBlockFunction implements Serializable
+	{
+		private static final long serialVersionUID = -729614449626680946L;
+
+		//internal buffer size (aligned w/ default matrix block size)
+		protected static final int BUFFER_SIZE = 4 * 1000 * 1000; //4M elements (32MB)
+		protected int _bufflen = -1;
+		
+		protected long _rlen = -1;
+		protected long _clen = -1;
+		
+		protected CellToBinaryBlockFunction(MatrixCharacteristics mc)
+		{
+			_rlen = mc.getRows();
+			_clen = mc.getCols();
+			
+			//determine upper bounded buffer len
+			_bufflen = (int) Math.min(_rlen*_clen, BUFFER_SIZE);
+		}
+
+
+		/**
+		 * 
+		 * @param rbuff
+		 * @param ret
+		 * @throws IOException 
+		 * @throws DMLRuntimeException 
+		 */
+		protected void flushBufferToList( FrameReblockBuffer rbuff,  ArrayList<Tuple2<Long,FrameBlock>> ret ) 
+			throws IOException, DMLRuntimeException
+		{
+			//temporary list of indexed matrix values to prevent library dependencies
+			ArrayList<Pair<Long, FrameBlock>> rettmp = new ArrayList<Pair<Long, FrameBlock>>();
+			rbuff.flushBufferToBinaryBlocks(rettmp);
+			ret.addAll(SparkUtils.fromIndexedFrameBlock(rettmp));
+		}
+	}
+	
+	
+	/**
+	 * 
+	 */
+	private static class TextToBinaryBlockFunction extends CellToBinaryBlockFunction implements PairFlatMapFunction<Iterator<Text>,Long,FrameBlock> 
+	{
+		private static final long serialVersionUID = -2042208027876880588L;
+		List<ValueType> _schema = null;
+		
+		protected TextToBinaryBlockFunction(MatrixCharacteristics mc, List<ValueType> schema ) {
+			super(mc);
+			_schema = schema;
+		}
+
+		@Override
+		public Iterable<Tuple2<Long, FrameBlock>> call(Iterator<Text> arg0) 
+			throws Exception 
+		{
+			ArrayList<Tuple2<Long,FrameBlock>> ret = new ArrayList<Tuple2<Long,FrameBlock>>();
+			FrameReblockBuffer rbuff = new FrameReblockBuffer(_bufflen, _rlen, _clen, _schema );
+			FastStringTokenizer st = new FastStringTokenizer(' ');
+			
+			while( arg0.hasNext() )
+			{
+				//get input string (ignore matrix market comments)
+				String strVal = arg0.next().toString();
+				if( strVal.startsWith("%") ) 
+					continue;
+				
+				//parse input ijv triple
+				st.reset( strVal );
+				long row = st.nextLong();
+				long col = st.nextLong();
+				Object val = UtilFunctions.stringToObject(_schema.get((int)col-1), st.nextToken());
+				
+				//flush buffer if necessary
+				if( rbuff.getSize() >= rbuff.getCapacity() )
+					flushBufferToList(rbuff, ret);
+				
+				//add value to reblock buffer
+				rbuff.appendCell(row, col, val);
+			}
+			
+			//final flush buffer
+			flushBufferToList(rbuff, ret);
+		
+			return ret;
+		}
+	}	
 }

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b1b9e838/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDAggregateUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDAggregateUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDAggregateUtils.java
index 5665fce..c545c30 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDAggregateUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDAggregateUtils.java
@@ -22,13 +22,13 @@ package org.apache.sysml.runtime.instructions.spark.utils;
 import org.apache.spark.api.java.JavaPairRDD;
 import org.apache.spark.api.java.function.Function;
 import org.apache.spark.api.java.function.Function2;
-
 import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
 import org.apache.sysml.runtime.DMLRuntimeException;
 import org.apache.sysml.runtime.functionobjects.KahanPlus;
 import org.apache.sysml.runtime.instructions.cp.KahanObject;
 import org.apache.sysml.runtime.instructions.spark.data.CorrMatrixBlock;
 import org.apache.sysml.runtime.instructions.spark.data.RowMatrixBlock;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
 import org.apache.sysml.runtime.matrix.data.MatrixBlock;
 import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
 import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues;
@@ -776,4 +776,43 @@ public class RDDAggregateUtils
 			return v1 + v2;
 		}	
 	}
+	
+	/**
+	 * @param: in
+	 * @return: 
+	 */
+	public static JavaPairRDD<Long, FrameBlock> mergeByFrameKey( JavaPairRDD<Long, FrameBlock> in )
+	{
+		return in.reduceByKey(
+				new MergeFrameBlocksFunction());
+	}
+	
+	/**
+	 * 
+	 */
+	private static class MergeFrameBlocksFunction implements Function2<FrameBlock, FrameBlock, FrameBlock> 
+	{		
+		private static final long serialVersionUID = -8881019027250258850L;
+
+		@Override
+		public FrameBlock call(FrameBlock b1, FrameBlock b2) 
+			throws Exception 
+		{
+			// sanity check input dimensions
+			if (b1.getNumRows() != b2.getNumRows() || b1.getNumColumns() != b2.getNumColumns()) {
+				throw new DMLRuntimeException("Mismatched frame block sizes for: "
+						+ b1.getNumRows() + " " + b1.getNumColumns() + " "
+						+ b2.getNumRows() + " " + b2.getNumColumns());
+			}
+
+			// execute merge (never pass by reference)
+			FrameBlock ret = new FrameBlock(b1);
+			ret.merge(b2);
+			
+			return ret;
+		}
+
+	}
+	
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b1b9e838/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
index 3c898a6..34db095 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
@@ -37,9 +37,11 @@ import org.apache.sysml.runtime.DMLRuntimeException;
 import org.apache.sysml.runtime.instructions.spark.functions.CopyBinaryCellFunction;
 import org.apache.sysml.runtime.instructions.spark.functions.CopyBlockFunction;
 import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
 import org.apache.sysml.runtime.matrix.data.MatrixBlock;
 import org.apache.sysml.runtime.matrix.data.MatrixCell;
 import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysml.runtime.matrix.data.Pair;
 import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue;
 import org.apache.sysml.runtime.util.UtilFunctions;
 
@@ -90,6 +92,32 @@ public class SparkUtils
 		return ret;
 	}
 	
+	
+	/**
+	 * 
+	 * @param in
+	 * @return
+	 */
+	public static Tuple2<Long,FrameBlock> fromIndexedFrameBlock( Pair<Long, FrameBlock> in ){
+		return new Tuple2<Long, FrameBlock>(in.getKey(), in.getValue());
+	}
+	
+	
+	/**
+	 * 
+	 * @param in
+	 * @return
+	 */
+	public static ArrayList<Tuple2<Long,FrameBlock>> fromIndexedFrameBlock( ArrayList<Pair<Long, FrameBlock>> in )
+	{
+		ArrayList<Tuple2<Long, FrameBlock>> ret = new ArrayList<Tuple2<Long, FrameBlock>>();
+		for( Pair<Long, FrameBlock> ifv : in )
+			ret.add(fromIndexedFrameBlock(ifv));
+		
+		return ret;
+	}
+	
+	
 	/**
 	 * 
 	 * @param mb

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b1b9e838/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
index dd1b744..534decf 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
@@ -93,7 +93,7 @@ public class FrameReaderBinaryBlock extends FrameReader
 	private static void readBinaryBlockFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest, long rlen, long clen )
 		throws IOException, DMLRuntimeException
 	{
-		LongWritable key = new LongWritable();
+		LongWritable key = new LongWritable(-1L);
 		FrameBlock value = new FrameBlock();
 		
 		for( Path lpath : getSequenceFilePaths(fs, path) ) //1..N files 
@@ -109,6 +109,9 @@ public class FrameReaderBinaryBlock extends FrameReader
 					
 					int rows = value.getNumRows();
 					int cols = value.getNumColumns();
+
+					if(rows == 0 || cols == 0)	//Empty block, ignore it.
+						continue;
 					
 					//bound check per block
 					if( row_offset + rows < 0 || row_offset + rows > rlen ) {

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b1b9e838/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
index 634e18c..6323fea 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
@@ -678,6 +678,49 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
 		return map;
 	}
 
+	/**
+	 * 
+	 * @param that
+	 * @throws DMLRuntimeException 
+	 */
+	public void merge(FrameBlock that ) 
+		throws DMLRuntimeException
+	{
+		//check for empty input source (nothing to merge)
+		if( that == null || that.getNumRows() == 0 )
+			return;
+		
+		//check dimensions (before potentially copy to prevent implicit dimension change) 
+		if( getNumRows() != that.getNumRows() || getNumColumns() != that.getNumColumns() )
+			throw new DMLRuntimeException("Dimension mismatch on merge disjoint (target="+getNumRows()+"x"+getNumColumns()+", source="+that.getNumRows()+"x"+that.getNumColumns()+")");
+		
+		//core frame block merge through cell copy
+		for( int i=0; i<getNumRows(); i++ ) {
+			for( int j=0; j<getNumColumns(); j++ ) {
+				switch( _schema.get(j) ) {
+					case STRING:  
+						if (that.get(i,j) != null)
+							set(i,j,that.get(i, j));
+						break;
+					case BOOLEAN: 
+						if ((Boolean)that.get(i,j) != Boolean.getBoolean("false"))
+							set(i,j,that.get(i, j));
+						break;
+					case INT:     
+						if ((Long)that.get(i,j) != 0)
+							set(i,j,that.get(i, j));
+						break;
+					case DOUBLE:  
+						if ((Double)that.get(i,j) != 0.0)
+							set(i,j,that.get(i, j));
+						break;
+					default: throw new RuntimeException("Unsupported value type: "+_schema.get(j));
+				}
+			}
+		}
+		
+	}
+
 	
 	///////
 	// row iterators (over strings and boxed objects)

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b1b9e838/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java
new file mode 100644
index 0000000..9b2cd37
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.apache.sysml.runtime.matrix.mapred;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.Pair;
+
+/**
+ * 
+ * 
+ */
+public class FrameReblockBuffer
+{
+	
+	//default buffer size: 5M -> 5M * 3x8B = 120MB 
+	public static final int DEFAULT_BUFFER_SIZE = 5000000;
+	
+	private int _bufflen = -1;
+	private int _count = -1;
+	
+	private FrameCell[] _buff = null;
+	
+	private long _rlen = -1;
+	private long _clen = -1;
+	private int _brlen = -1;
+	private int _bclen = -1;
+	
+	private List<ValueType> _schema;
+
+
+	/**
+	 * @param rlen
+	 * @param clen
+	 * @param schema
+	 * @return
+	 * 
+	 */
+	public FrameReblockBuffer( long rlen, long clen, List<ValueType> schema )
+	{
+		this( DEFAULT_BUFFER_SIZE, rlen, clen, schema );
+	}
+	
+	/**
+	 * @param buffersize
+	 * @param rlen
+	 * @param clen
+	 * @param schema
+	 * @return
+	 * 
+	 */
+	public FrameReblockBuffer( int buffersize,  long rlen, long clen, List<ValueType> schema )
+	{
+		_bufflen = buffersize;
+		_count = 0;
+		
+		_buff = new FrameCell[ _bufflen ];
+		for(int i=0; i< _bufflen; i++)
+			_buff[i] = new FrameCell();
+		
+		_rlen = rlen;
+		_clen = clen;
+		_brlen = Math.max((int)(_bufflen/_clen), 1);
+		_bclen = 1;
+		
+		_schema = schema;
+	}
+	
+
+	/**
+	 * 
+	 * @param ix
+	 * @param blen
+	 * @return
+	 */
+	private long getBlockIndex( long ix, int blen )
+	{
+		return (ix-1)/_brlen+1;
+	}
+	
+	/**
+	 * 
+	 * @param ix
+	 * @param blen
+	 * @return
+	 */
+	private int getIndexInBlock( long ix, int blen )
+	{
+		return (int)((ix-1)%_brlen);
+	}
+	
+	public int getSize()
+	{
+		return _count;
+	}
+	
+	public int getCapacity()
+	{
+		return _bufflen;
+	}
+	
+	/**
+	 * 
+	 * @param r
+	 * @param c
+	 * @param obj
+	 */
+	public void appendCell( long r, long c, Object obj )
+	{
+		_buff[_count].setRow((int)r);
+		_buff[_count].setCol((int)c);
+		_buff[_count].setObjVal(obj);
+		_count++;
+	}
+	
+	/**
+	 * 
+	 * @param r_offset
+	 * @param c_offset
+	 * @param inBlk
+	 * @param out
+	 * @throws IOException
+	 */
+	public void appendBlock(long r_offset, long c_offset, FrameBlock inBlk, OutputCollector<Long, Writable> out ) 
+		throws IOException
+	{
+		{
+			int rlen = inBlk.getNumRows();
+			int clen = inBlk.getNumColumns();
+			for( int i=0; i<rlen; i++ )
+				for( int j=0; j<clen; j++ )
+				{
+					Object obj = inBlk.get(i ,  j );
+					if( obj != null )
+					{
+						appendCell(r_offset+i, c_offset+j, obj);
+						
+						//check and flush if required
+						if( _count ==_bufflen )
+							flushBuffer(out);
+					}
+				}
+		}
+	}
+	
+
+	/**
+	 * 
+	 * @param out
+	 * @throws IOException
+	 */
+	public void flushBuffer( OutputCollector<Long, Writable> out ) 
+		throws IOException
+	{
+		if( _count == 0 )
+			return;
+		
+		//Step 1) sort reblock buffer (blockwise, no in-block sorting!)
+		Arrays.sort( _buff, 0 ,_count, new FrameReblockBufferComparator() );
+		
+		//Step 2) output blocks 
+		Long tmpIx = -1L;
+		//create intermediate blocks
+		FrameBlock tmpBlock = new FrameBlock();
+		
+		//put values into block and output
+		long cbi = -1, cbj = -1; //current block indexes
+		for( int i=0; i<_count; i++ )
+		{
+			long bi = getBlockIndex(_buff[i].getRow(), _brlen);
+			long bj = getBlockIndex(_buff[i].getCol(), _bclen);
+			
+			//output block and switch to next index pair
+			if( bi != cbi || bj != cbj ) {
+				outputBlock(out, tmpIx, tmpBlock);
+				cbi = bi;
+				cbj = bj;					
+				tmpIx = bi;
+				tmpBlock.reset(Math.min(_brlen, (int)(_rlen-(bi-1)*_brlen)));
+			}
+			
+			int ci = getIndexInBlock(_buff[i].getRow(), _brlen);
+			int cj = getIndexInBlock(_buff[i].getCol(), _bclen);
+			tmpBlock.set(ci, cj, _buff[i].getObjVal());
+		}
+		
+		//output last block 
+		outputBlock(out, tmpIx, tmpBlock);
+			
+		_count = 0;
+	}
+	
+	/**
+	 * 
+	 * @param outList
+	 * @throws IOException
+	 * @throws DMLRuntimeException 
+	 */
+	public void flushBufferToBinaryBlocks( ArrayList<Pair<Long, FrameBlock>> outList ) 
+		throws IOException, DMLRuntimeException
+	{
+		if( _count == 0 )
+			return;
+		
+		//Step 1) sort reblock buffer (blockwise, no in-block sorting!)
+		Arrays.sort( _buff, 0 ,_count, new FrameReblockBufferComparator() );
+				
+		//Step 2) output blocks 
+		Long tmpIx = -1L;
+		FrameBlock tmpBlock = new FrameBlock(_schema);
+		
+		//put values into block and output
+		long cbi = -1, cbj = -1; //current block indexes
+		for( int i=0; i<_count; i++ )
+		{
+			long bi = getBlockIndex(_buff[i].getRow(), _brlen);
+			long bj = getBlockIndex(_buff[i].getCol(), _bclen);
+			
+			//output block and switch to next index pair
+			if( bi != cbi || bj != cbj ) {
+				if( cbi != -1 && cbj != -1)
+					outputBlock(outList, tmpIx, tmpBlock);
+				cbi = bi;
+				cbj = bj;					
+				tmpIx = (bi-1)*_brlen+1;
+				tmpBlock = new FrameBlock(_schema);
+				tmpBlock.ensureAllocatedColumns(Math.min(_brlen, (int)(_rlen-(bi-1)*_brlen)));
+				
+			}
+			
+			int ci = getIndexInBlock(_buff[i].getRow(), _brlen);
+			int cj = getIndexInBlock(_buff[i].getCol(), _bclen);
+			tmpBlock.set(ci, cj, _buff[i].getObjVal());
+		}
+		
+		//output last block 
+		if( cbi != -1 && cbj != -1)
+			outputBlock(outList, tmpIx, tmpBlock);
+		
+		_count = 0;
+	}
+	
+	/**
+	 * 
+	 * @param out
+	 * @param key
+	 * @param block
+	 * @throws IOException
+	 */
+	private static void outputBlock( OutputCollector<Long, Writable> out, Long key, FrameBlock block ) 
+		throws IOException
+	{
+		//skip output of unassigned blocks
+		if( key == -1)
+			return;
+		
+		//output block
+		out.collect(key, block);
+	}
+	
+	/**
+	 * 
+	 * @param out
+	 * @param key
+	 * @param value
+	 * @throws IOException
+	 * @throws DMLRuntimeException 
+	 */
+	private static void outputBlock( ArrayList<Pair<Long, FrameBlock>> out, Long key, FrameBlock value ) 
+		throws IOException, DMLRuntimeException
+	{
+		//skip output of unassigned blocks
+		if( key == -1 )
+			return;
+		
+		//output block
+		out.add(new Pair<Long, FrameBlock>(new Long(key), value));
+	}
+	
+	private static class FrameCell 
+	{	
+		private int iRow;
+		private int iCol;
+		private Object objVal;
+		public int getRow() {
+			return iRow;
+		}
+		public void setRow(int iRow) {
+			this.iRow = iRow;
+		}
+		public int getCol() {
+			return iCol;
+		}
+		public void setCol(int iCol) {
+			this.iCol = iCol;
+		}
+		public Object getObjVal() {
+			return objVal;
+		}
+		public void setObjVal(Object objVal) {
+			this.objVal = objVal;
+		}
+	}
+	
+	/**
+	 * Comparator to sort the reblock buffer by block indexes, where we 
+	 * compute the block indexes on-the-fly based on the given cell indexes.
+	 * 
+	 */
+	private class FrameReblockBufferComparator implements Comparator<FrameCell> 
+	{	
+		@Override
+		public int compare(FrameCell arg0, FrameCell arg1) 
+		{
+			long bi0 = arg0.getRow();
+			long bj0 = arg0.getCol();
+			long bi1 = arg1.getRow();
+			long bj1 = arg1.getCol();
+			
+			return ( bi0 < bi1 || (bi0 == bi1 && bj0 < bj1) ) ? -1 :
+                   (( bi0 == bi1 && bj0 == bj1)? 0 : 1);		
+		}		
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b1b9e838/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
index 7c0d4f0..05d752c 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
@@ -25,6 +25,7 @@ import java.util.List;
 
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
+import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.JavaPairRDD;
 import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.api.java.JavaSparkContext;
@@ -64,6 +65,8 @@ public class FrameConverterTest extends AutomatedTestBase
 	private enum ConvType {
 		CSV2BIN,
 		BIN2CSV,
+		TXTCELL2BIN,
+		BIN2TXTCELL
 	}
 	
 	@Override
@@ -92,6 +95,26 @@ public class FrameConverterTest extends AutomatedTestBase
 		runFrameConverterTest(schemaMixed, ConvType.BIN2CSV);
 	}
 
+	@Test
+	public void testFrameStringsTxtCellBinSpark()  {
+		runFrameConverterTest(schemaStrings, ConvType.TXTCELL2BIN);
+	}
+	
+	@Test
+	public void testFrameMixedTxtCellBinSpark()  {
+		runFrameConverterTest(schemaMixed, ConvType.TXTCELL2BIN);
+	}
+	
+	@Test
+	public void testFrameStringsBinTxtCellSpark()  {
+		runFrameConverterTest(schemaStrings, ConvType.BIN2TXTCELL);
+	}
+	
+	@Test
+	public void testFrameMixedBinTxtCellSpark()  {
+		runFrameConverterTest(schemaMixed, ConvType.BIN2TXTCELL);
+	}
+
 	
 	/**
 	 * 
@@ -106,6 +129,9 @@ public class FrameConverterTest extends AutomatedTestBase
 		boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
 		DMLScript.USE_LOCAL_SPARK_CONFIG = true;
 
+		SparkConf conf = new SparkConf().setAppName("Frame").setMaster("local");
+		conf.set("spark.kryo.classesToRegister", "org.apache.hadoop.io.LongWritable");
+
 		try
 		{
 			TestConfiguration config = getTestConfiguration(TEST_NAME);
@@ -126,6 +152,14 @@ public class FrameConverterTest extends AutomatedTestBase
 					oinfo = OutputInfo.BinaryBlockOutputInfo;
 					iinfo = InputInfo.CSVInputInfo;
 					break;
+				case TXTCELL2BIN:
+					oinfo = OutputInfo.TextCellOutputInfo;
+					iinfo = InputInfo.BinaryBlockInputInfo;
+					break;
+				case BIN2TXTCELL:
+					oinfo = OutputInfo.BinaryBlockOutputInfo;
+					iinfo = InputInfo.TextCellInputInfo;
+					break;
 				default: 
 					throw new RuntimeException("Unsuported converter type: "+type.toString());
  			}
@@ -141,7 +175,7 @@ public class FrameConverterTest extends AutomatedTestBase
 
 			//run converter under test
 			MatrixCharacteristics mc = new MatrixCharacteristics(rows, schema.length, -1, -1, -1);
-			runConverter(type, mc, input("A"), output("B"));
+			runConverter(type, mc, Arrays.asList(schema), input("A"), output("B"));
 			
 			//read frame data from hdfs
 			FrameReader reader = FrameReaderFactory.createFrameReader(iinfo);
@@ -200,12 +234,13 @@ public class FrameConverterTest extends AutomatedTestBase
 	 * @param frame1
 	 * @param frame2
 	 * @param fprop
+	 * @param schema
 	 * @return 
 	 * @throws DMLRuntimeException, IOException
 	 */
 
 	@SuppressWarnings("unchecked")
-	private void runConverter(ConvType type, MatrixCharacteristics mc, String fnameIn, String fnameOut)
+	private void runConverter(ConvType type, MatrixCharacteristics mc, List<ValueType> schema, String fnameIn, String fnameOut)
 		throws DMLRuntimeException, IOException
 	{
 		SparkExecutionContext sec = (SparkExecutionContext) ExecutionContextFactory.createContext();		
@@ -231,6 +266,22 @@ public class FrameConverterTest extends AutomatedTestBase
 				rddOut.saveAsTextFile(fnameOut);
 				break;
 			}
+			case TXTCELL2BIN: {
+				InputInfo iinfo = InputInfo.TextCellInputInfo;
+				OutputInfo oinfo = OutputInfo.BinaryBlockOutputInfo;
+				JavaPairRDD<LongWritable,Text> rddIn = sc.hadoopFile(fnameIn, iinfo.inputFormatClass, iinfo.inputKeyClass, iinfo.inputValueClass);
+				JavaPairRDD<LongWritable, FrameBlock> rddOut = FrameRDDConverterUtils
+						.textCellToBinaryBlock(sc, rddIn, mc, schema);
+				rddOut.saveAsHadoopFile(fnameOut, LongWritable.class, FrameBlock.class, oinfo.outputFormatClass);
+				break;
+			}
+			case BIN2TXTCELL: {
+				InputInfo iinfo = InputInfo.BinaryBlockInputInfo;
+				JavaPairRDD<LongWritable, FrameBlock> rddIn = sc.hadoopFile(fnameIn, iinfo.inputFormatClass, LongWritable.class, FrameBlock.class);
+				JavaRDD<String> rddOut = FrameRDDConverterUtils.binaryBlockToStringRDD(rddIn, mc, "text");
+				rddOut.saveAsTextFile(fnameOut);
+				break;
+			}
 		}
 		
 		sec.close();