You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/06/08 19:17:44 UTC

[1/4] incubator-systemml git commit: [SYSTEMML-630] New parallel csv/textcell frame writers, cleanup

Repository: incubator-systemml
Updated Branches:
  refs/heads/master 6bbae619a -> dea42de1f


[SYSTEMML-630] New parallel csv/textcell frame writers, cleanup

This patch adds, similar to the parallel binary block frame writer,
parallel frame writers for csv and textcell. On a 1Mx1k (8GB binary,
~25-30GB in csv/text) scenario, multi-threaded write achieved the
following improvements:

* Textcell: 588s (single-threaded) -> 59s (multi-threaded)
* CSV: 518s (single-threaded) -> 47s (multi-threaded) 


Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/fdf55181
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/fdf55181
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/fdf55181

Branch: refs/heads/master
Commit: fdf551813572d6fd4f0d8076a525d4b1b3ab1b53
Parents: 6bbae61
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Tue Jun 7 13:31:15 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Tue Jun 7 13:31:15 2016 -0700

----------------------------------------------------------------------
 .../io/FrameWriterBinaryBlockParallel.java      |   7 +-
 .../sysml/runtime/io/FrameWriterFactory.java    |  10 +-
 .../sysml/runtime/io/FrameWriterTextCSV.java    |  62 ++++++---
 .../runtime/io/FrameWriterTextCSVParallel.java  | 133 +++++++++++++++++++
 .../sysml/runtime/io/FrameWriterTextCell.java   |  68 ++++++----
 .../runtime/io/FrameWriterTextCellParallel.java | 128 ++++++++++++++++++
 .../sysml/runtime/matrix/data/FrameBlock.java   |  44 +++++-
 7 files changed, 396 insertions(+), 56 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/fdf55181/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlockParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlockParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlockParallel.java
index f0b5bd1..2ef7692 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlockParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlockParallel.java
@@ -56,13 +56,12 @@ public class FrameWriterBinaryBlockParallel extends FrameWriterBinaryBlock
 	 * @throws DMLRuntimeException
 	 */
 	protected void writeBinaryBlockFrameToHDFS( Path path, JobConf job, FrameBlock src, long rlen, long clen )
-			throws IOException, DMLRuntimeException
+		throws IOException, DMLRuntimeException
 	{
 		//estimate output size and number of output blocks (min 1)
 		int blen = ConfigurationManager.getBlocksize();
-		int numPartFiles = (int)(OptimizerUtils.estimatePartitionedSizeExactSparsity(rlen, clen, blen, blen, rlen*clen) 
-						   / InfrastructureAnalyzer.getHDFSBlockSize());
-		numPartFiles = Math.max(numPartFiles, 1);
+		int numPartFiles = Math.max((int)(OptimizerUtils.estimatePartitionedSizeExactSparsity(rlen, clen, blen, blen, rlen*clen) 
+						   / InfrastructureAnalyzer.getHDFSBlockSize()), 1);
 		
 		//determine degree of parallelism
 		int numThreads = OptimizerUtils.getParallelBinaryWriteParallelism();

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/fdf55181/src/main/java/org/apache/sysml/runtime/io/FrameWriterFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterFactory.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterFactory.java
index a7f132f..c1021e6 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterFactory.java
@@ -57,12 +57,18 @@ public class FrameWriterFactory
 		FrameWriter writer = null;
 		
 		if( oinfo == OutputInfo.TextCellOutputInfo ) {
-			writer = new FrameWriterTextCell();
+			if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS) )
+				writer = new FrameWriterTextCellParallel();
+			else
+				writer = new FrameWriterTextCell();
 		}
 		else if( oinfo == OutputInfo.CSVOutputInfo ) {
 			if( props!=null && !(props instanceof CSVFileFormatProperties) )
 				throw new DMLRuntimeException("Wrong type of file format properties for CSV writer.");
-			writer = new FrameWriterTextCSV((CSVFileFormatProperties)props);
+			if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS) )
+				writer = new FrameWriterTextCSVParallel((CSVFileFormatProperties)props);
+			else
+				writer = new FrameWriterTextCSV((CSVFileFormatProperties)props);	
 		}
 		else if( oinfo == OutputInfo.BinaryBlockOutputInfo ) {
 			if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_WRITE_BINARYFORMATS) )

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/fdf55181/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
index 455f44d..addf798 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
@@ -34,6 +34,7 @@ import org.apache.sysml.runtime.matrix.data.FrameBlock;
 import org.apache.sysml.runtime.util.MapReduceTool;
 
 /**
+ * Single-threaded frame text csv writer.
  * 
  */
 public class FrameWriterTextCSV extends FrameWriter
@@ -58,22 +59,23 @@ public class FrameWriterTextCSV extends FrameWriter
 	 * @throws DMLRuntimeException  
 	 */
 	@Override
-	public void writeFrameToHDFS(FrameBlock src, String fname, long rlen, long clen) 
+	public final void writeFrameToHDFS(FrameBlock src, String fname, long rlen, long clen) 
 		throws IOException, DMLRuntimeException 
 	{
-		//validity check frame dimensions
-		if( src.getNumRows() != rlen || src.getNumColumns() != clen ) {
-			throw new IOException("Frame dimensions mismatch with metadata: "+src.getNumRows()+"x"+src.getNumColumns()+" vs "+rlen+"x"+clen+".");
-		}
-		
 		//prepare file access
 		JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
 		Path path = new Path( fname );
 
 		//if the file already exists on HDFS, remove it.
 		MapReduceTool.deleteFileIfExistOnHDFS( fname );
-			
-		//core write
+	
+		//validity check frame dimensions
+		if( src.getNumRows() != rlen || src.getNumColumns() != clen ) {
+			throw new IOException("Frame dimensions mismatch with metadata: " + 
+					src.getNumRows()+"x"+src.getNumColumns()+" vs "+rlen+"x"+clen+".");
+		}
+		
+		//core write (sequential/parallel)
 		writeCSVFrameToHDFS(path, job, src, rlen, clen, _props);
 	}
 
@@ -84,16 +86,36 @@ public class FrameWriterTextCSV extends FrameWriter
 	 * @param src
 	 * @param rlen
 	 * @param clen
+	 * @param csvprops
+	 * @throws IOException
+	 */
+	protected void writeCSVFrameToHDFS( Path path, JobConf job, FrameBlock src, long rlen, long clen, CSVFileFormatProperties csvprops ) 
+		throws IOException
+	{
+		FileSystem fs = FileSystem.get(job);
+        
+		//sequential write to single text file
+		writeCSVFrameToFile(path, job, fs, src, 0, (int)rlen, csvprops);	
+	}
+	
+	/**
+	 * 
+	 * @param path
+	 * @param job
+	 * @param src
+	 * @param rlen
+	 * @param clen
 	 * @param props
 	 * @return
 	 * @throws IOException
 	 */
-	protected void writeCSVFrameToHDFS( Path path, JobConf job, FrameBlock src, long rlen, long clen, CSVFileFormatProperties props )
+	protected final void writeCSVFrameToFile( Path path, JobConf job, FileSystem fs, FrameBlock src, int rl, int ru, CSVFileFormatProperties props )
 		throws IOException
 	{
-		FileSystem fs = FileSystem.get(job);
-        BufferedWriter br=new BufferedWriter(new OutputStreamWriter(fs.create(path,true)));		
-		
+    	//create buffered writer
+		BufferedWriter br=new BufferedWriter(new OutputStreamWriter(fs.create(path,true)));		
+    	int cols = src.getNumColumns();
+	
 		try
 		{
 			//for obj reuse and preventing repeated buffer re-allocations
@@ -103,13 +125,13 @@ public class FrameWriterTextCSV extends FrameWriter
 			String delim = props.getDelim();
 			
 			// Write header line, if needed
-			if( props.hasHeader() ) 
+			if( props.hasHeader() && rl==0 ) 
 			{
 				//write row chunk-wise to prevent OOM on large number of columns
-				for( int bj=0; bj<clen; bj+=BLOCKSIZE_J ) {
-					for( int j=bj; j < Math.min(clen,bj+BLOCKSIZE_J); j++) {
+				for( int bj=0; bj<cols; bj+=BLOCKSIZE_J ) {
+					for( int j=bj; j < Math.min(cols,bj+BLOCKSIZE_J); j++) {
 						sb.append("C"+ (j+1));
-						if ( j < clen-1 )
+						if ( j < cols-1 )
 							sb.append(delim);
 					}
 					br.write( sb.toString() );
@@ -121,15 +143,15 @@ public class FrameWriterTextCSV extends FrameWriter
 			}
 			
 			// Write data lines
-			Iterator<String[]> iter = src.getStringRowIterator();
+			Iterator<String[]> iter = src.getStringRowIterator(rl, ru);
 			while( iter.hasNext() ) {
 				//write row chunk-wise to prevent OOM on large number of columns
 				String[] row = iter.next();
-				for( int bj=0; bj<clen; bj+=BLOCKSIZE_J ) {
-					for( int j=bj; j<Math.min(clen,bj+BLOCKSIZE_J); j++ ) {
+				for( int bj=0; bj<cols; bj+=BLOCKSIZE_J ) {
+					for( int j=bj; j<Math.min(cols,bj+BLOCKSIZE_J); j++ ) {
 						if(row[j] != null)
 							sb.append(row[j]);					
-						if( j != clen-1 )
+						if( j != cols-1 )
 							sb.append(delim);
 					}
 					br.write( sb.toString() );

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/fdf55181/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSVParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSVParallel.java
new file mode 100644
index 0000000..de492e2
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSVParallel.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.MapReduceTool;
+
+/**
+ * Single-threaded frame text csv writer.
+ * 
+ */
+public class FrameWriterTextCSVParallel extends FrameWriterTextCSV
+{
+	public FrameWriterTextCSVParallel( CSVFileFormatProperties props ) {
+		super(props);
+	}
+
+	/**
+	 * 
+	 * @param path
+	 * @param job
+	 * @param src
+	 * @param rlen
+	 * @param clen
+	 * @param csvprops
+	 * @throws IOException
+	 */
+	@Override
+	protected void writeCSVFrameToHDFS( Path path, JobConf job, FrameBlock src, long rlen, long clen, CSVFileFormatProperties csvprops ) 
+		throws IOException
+	{
+		//estimate output size and number of output blocks (min 1)
+		int numPartFiles = Math.max((int)(OptimizerUtils.estimateSizeTextOutput(rlen, clen, rlen*clen, 
+				              OutputInfo.CSVOutputInfo)  / InfrastructureAnalyzer.getHDFSBlockSize()), 1);
+		
+		//determine degree of parallelism
+		int numThreads = OptimizerUtils.getParallelTextWriteParallelism();
+		numThreads = Math.min(numThreads, numPartFiles);
+		
+		//fall back to sequential write if dop is 1 (e.g., <128MB) in order to create single file
+		if( numThreads <= 1 ) {
+			super.writeCSVFrameToHDFS(path, job, src, rlen, clen, csvprops);
+			return;
+		}
+		
+		//create directory for concurrent tasks
+		MapReduceTool.createDirIfNotExistOnHDFS(path.toString(), DMLConfig.DEFAULT_SHARED_DIR_PERMISSION);
+		FileSystem fs = FileSystem.get(job);
+		
+		//create and execute tasks
+		try 
+		{
+			ExecutorService pool = Executors.newFixedThreadPool(numThreads);
+			ArrayList<WriteFileTask> tasks = new ArrayList<WriteFileTask>();
+			int blklen = (int)Math.ceil((double)rlen / numThreads);
+			for(int i=0; i<numThreads & i*blklen<rlen; i++) {
+				Path newPath = new Path(path, String.format("0-m-%05d",i));
+				tasks.add(new WriteFileTask(newPath, job, fs, src, i*blklen, (int)Math.min((i+1)*blklen, rlen), csvprops));
+			}
+
+			//wait until all tasks have been executed
+			List<Future<Object>> rt = pool.invokeAll(tasks);	
+			pool.shutdown();
+			
+			//check for exceptions 
+			for( Future<Object> task : rt )
+				task.get();
+		} 
+		catch (Exception e) {
+			throw new IOException("Failed parallel write of csv output.", e);
+		}
+	}
+	
+	private class WriteFileTask implements Callable<Object> 
+	{
+		private Path _path = null;
+		private JobConf _job = null;
+		private FileSystem _fs = null;
+		private FrameBlock _src = null;
+		private int _rl = -1;
+		private int _ru = -1;
+		private CSVFileFormatProperties _csvprops = null;
+		
+		public WriteFileTask(Path path, JobConf job, FileSystem fs, FrameBlock src, int rl, int ru, CSVFileFormatProperties csvprops) {
+			_path = path;
+			_fs = fs;
+			_job = job;
+			_src = src;
+			_rl = rl;
+			_ru = ru;
+			_csvprops = csvprops;
+		}
+	
+		@Override
+		public Object call() throws Exception  {
+			writeCSVFrameToFile(_path, _job, _fs, _src, _rl, _ru, _csvprops);
+			return null;
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/fdf55181/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
index 8064ce5..38348ad 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
@@ -32,6 +32,10 @@ import org.apache.sysml.runtime.DMLRuntimeException;
 import org.apache.sysml.runtime.matrix.data.FrameBlock;
 import org.apache.sysml.runtime.util.MapReduceTool;
 
+/**
+ * Single-threaded frame text cell writer.
+ * 
+ */
 public class FrameWriterTextCell extends FrameWriter
 {
 	/**
@@ -42,22 +46,23 @@ public class FrameWriterTextCell extends FrameWriter
 	 * @throws DMLRuntimeException 
 	 */
 	@Override
-	public void writeFrameToHDFS( FrameBlock src, String fname, long rlen, long clen )
+	public final void writeFrameToHDFS( FrameBlock src, String fname, long rlen, long clen )
 		throws IOException, DMLRuntimeException 
 	{
-		//validity check frame dimensions
-		if( src.getNumRows() != rlen || src.getNumColumns() != clen ) {
-			throw new IOException("Frame dimensions mismatch with metadata: "+src.getNumRows()+"x"+src.getNumColumns()+" vs "+rlen+"x"+clen+".");
-		}
-				
 		//prepare file access
 		JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
 		Path path = new Path( fname );
 
 		//if the file already exists on HDFS, remove it.
 		MapReduceTool.deleteFileIfExistOnHDFS( fname );
-			
-		//core write
+		
+		//validity check frame dimensions
+		if( src.getNumRows() != rlen || src.getNumColumns() != clen ) {
+			throw new IOException("Frame dimensions mismatch with metadata: " + 
+					src.getNumRows()+"x"+src.getNumColumns()+" vs "+rlen+"x"+clen+".");
+		}
+		
+		//core write (sequential/parallel)
 		writeTextCellFrameToHDFS(path, job, src, src.getNumRows(), src.getNumColumns());
 	}
 
@@ -68,38 +73,51 @@ public class FrameWriterTextCell extends FrameWriter
 	 * @param src
 	 * @param rlen
 	 * @param clen
-	 * @param brlen
-	 * @param bclen
 	 * @throws IOException
 	 */
 	protected void writeTextCellFrameToHDFS( Path path, JobConf job, FrameBlock src, long rlen, long clen )
 		throws IOException
 	{
-		boolean entriesWritten = false;
 		FileSystem fs = FileSystem.get(job);
-        BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(path,true)));		
 		
-    	int rows = src.getNumRows();
-		int cols = src.getNumColumns();
+		//sequential write to single text file
+		writeTextCellFrameToFile(path, job, fs, src, 0, (int)rlen);	
+	}	
+	
+	/**
+	 * Internal primitive to write a row range of a frame to a single text file, 
+	 * which is used for both single- and multi-threaded writers (for consistency). 
+	 *  
+	 * @param path
+	 * @param job
+	 * @param fs
+	 * @param src
+	 * @param rl
+	 * @param ru
+	 * @throws IOException 
+	 */
+	protected final void writeTextCellFrameToFile( Path path, JobConf job, FileSystem fs, FrameBlock src, int rl, int ru ) 
+		throws IOException
+	{
+		boolean entriesWritten = false;
+    	int cols = src.getNumColumns();
+
+    	//create buffered writer 
+    	BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(path,true)));		
 
-		//bound check per block
-		if( rows > rlen || cols > clen ) {
-			throw new IOException("Frame block [1:"+rows+",1:"+cols+"] " +
-					              "out of overall frame range [1:"+rlen+",1:"+clen+"].");
-		}
-		
 		try
 		{
 			//for obj reuse and preventing repeated buffer re-allocations
 			StringBuilder sb = new StringBuilder();
 			
-			Iterator<String[]> iter = src.getStringRowIterator();
-			for( int i=0; iter.hasNext(); i++ ) { //for all rows
+			//write frame row range to output
+			Iterator<String[]> iter = src.getStringRowIterator(rl, ru);
+			for( int i=rl; iter.hasNext(); i++ ) { //for all rows
 				String rowIndex = Integer.toString(i+1);
 				String[] row = iter.next();
 				for( int j=0; j<cols; j++ ) {
 					if( row[j] != null ) {
-						sb.append(rowIndex);
+						sb.append( rowIndex );
 						sb.append(' ');
 						sb.append( j+1 );
 						sb.append(' ');
@@ -119,6 +137,6 @@ public class FrameWriterTextCell extends FrameWriter
 		}
 		finally {
 			IOUtilFunctions.closeSilently(br);
-		}
-	}	
+		}		
+	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/fdf55181/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCellParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCellParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCellParallel.java
new file mode 100644
index 0000000..82a139d
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCellParallel.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.MapReduceTool;
+
+/**
+ * Multi-threaded frame text cell writer.
+ * 
+ */
+public class FrameWriterTextCellParallel extends FrameWriterTextCell
+{
+	/**
+	 * 
+	 * @param path
+	 * @param job
+	 * @param src
+	 * @param rlen
+	 * @param clen
+	 * @throws IOException
+	 */
+	@Override
+	protected void writeTextCellFrameToHDFS( Path path, JobConf job, FrameBlock src, long rlen, long clen )
+		throws IOException
+	{
+		//estimate output size and number of output blocks (min 1)
+		int numPartFiles = Math.max((int)(OptimizerUtils.estimateSizeTextOutput(rlen, clen, rlen*clen, 
+				              OutputInfo.TextCellOutputInfo)  / InfrastructureAnalyzer.getHDFSBlockSize()), 1);
+		
+		//determine degree of parallelism
+		int numThreads = OptimizerUtils.getParallelTextWriteParallelism();
+		numThreads = Math.min(numThreads, numPartFiles);
+		
+		//fall back to sequential write if dop is 1 (e.g., <128MB) in order to create single file
+		if( numThreads <= 1 ) {
+			super.writeTextCellFrameToHDFS(path, job, src, rlen, clen);
+			return;
+		}
+		
+		//create directory for concurrent tasks
+		MapReduceTool.createDirIfNotExistOnHDFS(path.toString(), DMLConfig.DEFAULT_SHARED_DIR_PERMISSION);
+		FileSystem fs = FileSystem.get(job);
+		
+		//create and execute tasks
+		try 
+		{
+			ExecutorService pool = Executors.newFixedThreadPool(numThreads);
+			ArrayList<WriteFileTask> tasks = new ArrayList<WriteFileTask>();
+			int blklen = (int)Math.ceil((double)rlen / numThreads);
+			for(int i=0; i<numThreads & i*blklen<rlen; i++) {
+				Path newPath = new Path(path, String.format("0-m-%05d",i));
+				tasks.add(new WriteFileTask(newPath, job, fs, src, i*blklen, (int)Math.min((i+1)*blklen, rlen)));
+			}
+
+			//wait until all tasks have been executed
+			List<Future<Object>> rt = pool.invokeAll(tasks);	
+			pool.shutdown();
+			
+			//check for exceptions 
+			for( Future<Object> task : rt )
+				task.get();
+		} 
+		catch (Exception e) {
+			throw new IOException("Failed parallel write of text output.", e);
+		}
+	}	
+	
+	/**
+	 * 
+	 */
+	private class WriteFileTask implements Callable<Object> 
+	{
+		private Path _path = null;
+		private JobConf _job = null;
+		private FileSystem _fs = null;
+		private FrameBlock _src = null;
+		private int _rl = -1;
+		private int _ru = -1;
+		
+		public WriteFileTask(Path path, JobConf job, FileSystem fs, FrameBlock src, int rl, int ru) {
+			_path = path;
+			_fs = fs;
+			_job = job;
+			_src = src;
+			_rl = rl;
+			_ru = ru;
+		}
+	
+		@Override
+		public Object call() throws Exception  {
+			writeTextCellFrameToFile(_path, _job, _fs, _src, _rl, _ru);
+			return null;
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/fdf55181/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
index 268ba44..b851704 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
@@ -348,7 +348,19 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
 	 * @return
 	 */
 	public Iterator<String[]> getStringRowIterator() {
-		return new StringRowIterator();
+		return new StringRowIterator(0, _numRows);
+	}
+	
+	/**
+	 * Get a row iterator over the frame where all fields are encoded
+	 * as strings independent of their value types.  
+	 * 
+	 * @param rl
+	 * @param ru
+	 * @return
+	 */
+	public Iterator<String[]> getStringRowIterator(int rl, int ru) {
+		return new StringRowIterator(rl, ru);
 	}
 	
 	/**
@@ -358,7 +370,19 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
 	 * @return
 	 */
 	public Iterator<Object[]> getObjectRowIterator() {
-		return new ObjectRowIterator();
+		return new ObjectRowIterator(0, _numRows);
+	}
+	
+	/**
+	 * Get a row iterator over the frame where all fields are encoded
+	 * as boxed objects according to their value types.  
+	 * 
+	 * @param rl
+	 * @param ru
+	 * @return
+	 */
+	public Iterator<Object[]> getObjectRowIterator(int rl, int ru) {
+		return new ObjectRowIterator(rl, ru);
 	}
 
 	///////
@@ -744,15 +768,17 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
 	private abstract class RowIterator<T> implements Iterator<T[]> {
 		protected T[] _curRow = null;
 		protected int _curPos = -1;
+		protected int _maxPos = -1;
 		
-		protected RowIterator() {
-			_curPos = 0;
+		protected RowIterator(int rl, int ru) {
+			_curPos = rl;
+			_maxPos = ru;
 			_curRow = createRow(getNumColumns());
 		}
 		
 		@Override
 		public boolean hasNext() {
-			return (_curPos < _numRows);
+			return (_curPos < _maxPos);
 		}
 
 		@Override
@@ -767,6 +793,10 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
 	 * 
 	 */
 	private class StringRowIterator extends RowIterator<String> {
+		public StringRowIterator(int rl, int ru) {
+			super(rl, ru);
+		}
+		
 		@Override
 		protected String[] createRow(int size) {
 			return new String[size];
@@ -787,6 +817,10 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
 	 * 
 	 */
 	private class ObjectRowIterator extends RowIterator<Object> {
+		public ObjectRowIterator(int rl, int ru) {
+			super(rl, ru);
+		}
+		
 		@Override
 		protected Object[] createRow(int size) {
 			return new Object[size];


[4/4] incubator-systemml git commit: [SYSTEMML-630] New parallel csv/textcell frame readers, cleanup/tests

Posted by mb...@apache.org.
[SYSTEMML-630] New parallel csv/textcell frame readers, cleanup/tests

This patch introduces parallel frame readers for csv and textcell. On a
1Mx1k (8GB binary,~25-30GB in csv/text) scenario, multi-threaded read
led to the following performance improvements:

* Textcell: 930s (single-threaded) -> 83s (multi-threaded)
* CSV: 538s (single-threaded) -> 51s (multi-threaded) 

Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/dea42de1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/dea42de1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/dea42de1

Branch: refs/heads/master
Commit: dea42de1ff7e8ada41ad50e4e941f080f82d2346
Parents: 288438b
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Wed Jun 8 00:08:27 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Wed Jun 8 01:24:02 2016 -0700

----------------------------------------------------------------------
 .../io/FrameReaderBinaryBlockParallel.java      |  11 +-
 .../sysml/runtime/io/FrameReaderFactory.java    |  10 +-
 .../sysml/runtime/io/FrameReaderTextCSV.java    | 217 ++++++++++-------
 .../runtime/io/FrameReaderTextCSVParallel.java  | 235 +++++++++++++++++++
 .../sysml/runtime/io/FrameReaderTextCell.java   | 132 ++++++-----
 .../runtime/io/FrameReaderTextCellParallel.java | 114 +++++++++
 .../sysml/runtime/io/IOUtilFunctions.java       |  28 +++
 .../sysml/runtime/io/ReaderTextCSVParallel.java |  19 +-
 .../functions/frame/FrameReadWriteTest.java     |  89 +++++--
 9 files changed, 662 insertions(+), 193 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java
index d684af0..3571c37 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java
@@ -63,16 +63,14 @@ public class FrameReaderBinaryBlockParallel extends FrameReaderBinaryBlock
 			//create read tasks for all files
 			ExecutorService pool = Executors.newFixedThreadPool(numThreads);
 			ArrayList<ReadFileTask> tasks = new ArrayList<ReadFileTask>();
-			for( Path lpath : getSequenceFilePaths(fs, path) ){
-				ReadFileTask t = new ReadFileTask(lpath, job, fs, dest);
-				tasks.add(t);
-			}
+			for( Path lpath : getSequenceFilePaths(fs, path) )
+				tasks.add(new ReadFileTask(lpath, job, fs, dest));
 
 			//wait until all tasks have been executed
 			List<Future<Object>> rt = pool.invokeAll(tasks);	
 			pool.shutdown();
 			
-			//check for exceptions and aggregate nnz
+			//check for exceptions
 			for( Future<Object> task : rt )
 				task.get();
 		} 
@@ -99,8 +97,7 @@ public class FrameReaderBinaryBlockParallel extends FrameReaderBinaryBlock
 		}
 
 		@Override
-		public Object call() throws Exception 
-		{
+		public Object call() throws Exception {
 			readBinaryBlockFrameFromSequenceFile(_path, _job, _fs, _dest);
 			return null;
 		}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
index 51606df..b768450 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
@@ -80,12 +80,18 @@ public class FrameReaderFactory
 		FrameReader reader = null;
 
 		if( iinfo == InputInfo.TextCellInputInfo ) {
-			reader = new FrameReaderTextCell();
+			if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_BINARYFORMATS) )
+				reader = new FrameReaderTextCellParallel();
+			else	
+				reader = new FrameReaderTextCell();
 		}
 		else if( iinfo == InputInfo.CSVInputInfo ) {
 			if( props!=null && !(props instanceof CSVFileFormatProperties) )
 				throw new DMLRuntimeException("Wrong type of file format properties for CSV writer.");
-			reader = new FrameReaderTextCSV( (CSVFileFormatProperties)props);
+			if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_BINARYFORMATS) )
+				reader = new FrameReaderTextCSVParallel( (CSVFileFormatProperties)props );
+			else
+				reader = new FrameReaderTextCSV( (CSVFileFormatProperties)props );
 		}
 		else if( iinfo == InputInfo.BinaryBlockInputInfo ) {
 			if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_BINARYFORMATS) )

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
index 2086ea3..d282aef 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
@@ -19,32 +19,37 @@
 
 package org.apache.sysml.runtime.io;
 
-import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 
-import org.apache.hadoop.fs.FileStatus;
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TextInputFormat;
 import org.apache.sysml.conf.ConfigurationManager;
 import org.apache.sysml.parser.Expression.ValueType;
 import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.matrix.CSVReblockMR;
 import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
 import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.Pair;
 import org.apache.sysml.runtime.util.UtilFunctions;
 
+/**
+ * Single-threaded frame text csv reader.
+ * 
+ */
 public class FrameReaderTextCSV extends FrameReader
 {
-
-	private CSVFileFormatProperties _props = null;
+	protected CSVFileFormatProperties _props = null;
 	
-	public FrameReaderTextCSV(CSVFileFormatProperties props)
-	{
+	public FrameReaderTextCSV(CSVFileFormatProperties props) {
 		_props = props;
 	}
 	
@@ -61,15 +66,10 @@ public class FrameReaderTextCSV extends FrameReader
 	 * @throws IOException 
 	 */
 	@Override
-	public FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names,
+	public final FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names,
 			long rlen, long clen)
 		throws IOException, DMLRuntimeException 
 	{
-		//allocate output frame block
-		FrameBlock ret = null;
-		if( rlen>0 && clen>0 ) //otherwise CSV reblock based on file size for frame w/ unknown dimensions
-			ret = createOutputFrameBlock(schema, names, rlen);
-		
 		//prepare file access
 		JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());	
 		FileSystem fs = FileSystem.get(job);
@@ -77,13 +77,52 @@ public class FrameReaderTextCSV extends FrameReader
 		
 		//check existence and non-empty file
 		checkValidInputFile(fs, path); 
+		
+		//compute size if necessary
+		if( rlen <= 0 || clen <= 0 ) {
+			Pair<Integer,Integer> size = computeCSVSize(path, job, fs);
+			rlen = size.getKey();
+			clen = size.getValue();
+		}
+		
+		//allocate output frame block
+		List<ValueType> lschema = createOutputSchema(schema, clen);
+		List<String> lnames = createOutputNames(names, clen);
+		FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
 	
-		//core read 
-		ret = readCSVFrameFromHDFS(path, job, fs, ret, schema, names, rlen, clen,  
-				   _props.hasHeader(), _props.getDelim(), _props.isFill() );
+		//core read (sequential/parallel) 
+		readCSVFrameFromHDFS(path, job, fs, ret, lschema, lnames, rlen, clen);
 		
 		return ret;
 	}
+
+	/**
+	 * 
+	 * @param path
+	 * @param job
+	 * @param fs
+	 * @param dest
+	 * @param schema
+	 * @param names
+	 * @param rlen
+	 * @param clen
+	 * @return
+	 * @throws IOException 
+	 */
+	protected void readCSVFrameFromHDFS( Path path, JobConf job, FileSystem fs, 
+			FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen) 
+		throws IOException
+	{
+		FileInputFormat.addInputPath(job, path);
+		TextInputFormat informat = new TextInputFormat();
+		informat.configure(job);
+		InputSplit[] splits = informat.getSplits(job, 1);
+		splits = IOUtilFunctions.sortInputSplits(splits);
+		for( int i=0; i<splits.length; i++ )
+			readCSVFrameFrameFromInputSplit(splits[i], informat, job, dest, schema, names, rlen, clen, 0, i==0);
+	}
+	
+	
 	
 	/**
 	 * 
@@ -99,72 +138,60 @@ public class FrameReaderTextCSV extends FrameReader
 	 * @return
 	 * @throws IOException
 	 */
-	@SuppressWarnings("unchecked")
-	private FrameBlock readCSVFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest, 
-			List<ValueType> schema, List<String> names, long rlen, long clen, boolean hasHeader, String delim, boolean fill)
+	protected final void readCSVFrameFrameFromInputSplit( InputSplit split, TextInputFormat informat, JobConf job, 
+			FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen, int rl, boolean first)
 		throws IOException
 	{
-		ArrayList<Path> files=new ArrayList<Path>();
-		if(fs.isDirectory(path)) {
-			for(FileStatus stat: fs.listStatus(path, CSVReblockMR.hiddenFileFilter))
-				files.add(stat.getPath());
-			Collections.sort(files);
-		}
-		else
-			files.add(path);
+		boolean hasHeader = _props.hasHeader();
+		boolean isFill = _props.isFill();
+		double dfillValue = _props.getFillValue();
+		String sfillValue = String.valueOf(_props.getFillValue());
+		String delim = _props.getDelim();
 		
-		if ( dest == null ) {
-			dest = computeCSVSize(files, fs, schema, names, hasHeader, delim);
-			clen = dest.getNumColumns();
-		}
-		
-		/////////////////////////////////////////
-		String value = null;
-		int row = 0;
+		//create record reader
+		RecordReader<LongWritable, Text> reader = informat.getRecordReader(split, job, Reporter.NULL);
+		LongWritable key = new LongWritable();
+		Text value = new Text();
+		int row = rl;
 		int col = -1;
 		
-		for(int fileNo=0; fileNo<files.size(); fileNo++)
-		{
-			BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(files.get(fileNo))));
-			if(fileNo==0 && hasHeader ) 
-				br.readLine(); //ignore header
+		//handle header if existing
+		if(first && hasHeader ) 
+			reader.next(key, value); //ignore header
 			
-			// Read the data
-			boolean emptyValuesFound = false;
-			try
+		// Read the data
+		boolean emptyValuesFound = false;
+		try
+		{
+			while( reader.next(key, value) ) //foreach line
 			{
-				while( (value=br.readLine())!=null ) //foreach line
+				String cellStr = value.toString().trim();
+				emptyValuesFound = false; col = 0;
+				String[] parts = IOUtilFunctions.split(cellStr, delim);
+				
+				for( String part : parts ) //foreach cell
 				{
-					String cellStr = value.toString().trim();
-					emptyValuesFound = false;
-					String[] parts = IOUtilFunctions.split(cellStr, delim);
-					col = 0;
-					
-					for( String part : parts ) //foreach cell
-					{
-						part = part.trim();
-						if ( part.isEmpty() ) {
-							//TODO: Do we need to handle empty cell condition?
-							emptyValuesFound = true;
-						}
-						else {
-							dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), part));
-						}
-						col++;
+					part = part.trim();
+					if ( part.isEmpty() ) {
+						if( isFill && dfillValue!=0 )
+							dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), sfillValue));
+						emptyValuesFound = true;
 					}
-					
-					//sanity checks for empty values and number of columns
-					IOUtilFunctions.checkAndRaiseErrorCSVEmptyField(cellStr, fill, emptyValuesFound);
-					IOUtilFunctions.checkAndRaiseErrorCSVNumColumns(path.toString(), cellStr, parts, clen);
-					row++;
+					else {
+						dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), part));
+					}
+					col++;
 				}
-			}
-			finally {
-				IOUtilFunctions.closeSilently(br);
+				
+				//sanity checks for empty values and number of columns
+				IOUtilFunctions.checkAndRaiseErrorCSVEmptyField(cellStr, isFill, emptyValuesFound);
+				IOUtilFunctions.checkAndRaiseErrorCSVNumColumns("", cellStr, parts, clen);
+				row++;
 			}
 		}
-		
-		return dest;
+		finally {
+			IOUtilFunctions.closeSilently(reader);
+		}
 	}
 	
 	/**
@@ -178,35 +205,45 @@ public class FrameReaderTextCSV extends FrameReader
 	 * @return
 	 * @throws IOException
 	 */
-	private FrameBlock computeCSVSize ( List<Path> files, FileSystem fs, List<ValueType> schema, List<String> names, boolean hasHeader, String delim) 
+	protected Pair<Integer,Integer> computeCSVSize( Path path, JobConf job, FileSystem fs) 
 		throws IOException 
 	{		
-		int nrow = 0;
-		for(int fileNo=0; fileNo<files.size(); fileNo++)
+		FileInputFormat.addInputPath(job, path);
+		TextInputFormat informat = new TextInputFormat();
+		informat.configure(job);
+		InputSplit[] splits = informat.getSplits(job, 1);
+		splits = IOUtilFunctions.sortInputSplits(splits);
+		
+		boolean first = true;
+		int ncol = -1;
+		int nrow = -1;
+		
+		for( InputSplit split : splits ) 
 		{
-			BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(files.get(fileNo))));	
+			RecordReader<LongWritable, Text> reader = informat.getRecordReader(split, job, Reporter.NULL);
+			LongWritable key = new LongWritable();
+			Text value = new Text();
+			
 			try
 			{
-				// Read the header line, if there is one.
-				if(fileNo==0)
-				{
-					if ( hasHeader ) 
-						br.readLine(); //ignore header
+				//read head and first line to determine num columns
+				if( first ) {
+					if ( _props.hasHeader() ) 
+						reader.next(key, value); //ignore header
+					reader.next(key, value);
+					ncol = StringUtils.countMatches(value.toString(), _props.getDelim()) + 1;
+					nrow = 1; first = false;
 				}
 				
-				while ( br.readLine() != null ) {
+				//count remaining number of rows
+				while ( reader.next(key, value) )
 					nrow++;
-				}
 			}
 			finally {
-				IOUtilFunctions.closeSilently(br);
+				IOUtilFunctions.closeSilently(reader);
 			}
 		}
 		
-		//create new frame block
-		FrameBlock frameBlock = new FrameBlock(schema, names);
-		frameBlock.ensureAllocatedColumns(nrow);
-		return frameBlock;
+		return new Pair<Integer,Integer>(nrow, ncol);
 	}
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java
new file mode 100644
index 0000000..b998fce
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.Pair;
+
+/**
+ * Multi-threaded frame text csv reader.
+ * 
+ */
+public class FrameReaderTextCSVParallel extends FrameReaderTextCSV
+{
+	public FrameReaderTextCSVParallel(CSVFileFormatProperties props) {
+		super(props);
+	}
+
+	/**
+	 * 
+	 * @param path
+	 * @param job
+	 * @param fs
+	 * @param dest
+	 * @param schema
+	 * @param names
+	 * @param rlen
+	 * @param clen
+	 * @return
+	 * @throws IOException 
+	 */
+	@Override
+	protected void readCSVFrameFromHDFS( Path path, JobConf job, FileSystem fs, 
+			FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen) 
+		throws IOException
+	{
+		int numThreads = OptimizerUtils.getParallelTextReadParallelism();
+		
+		FileInputFormat.addInputPath(job, path);
+		TextInputFormat informat = new TextInputFormat();
+		informat.configure(job);
+		InputSplit[] splits = informat.getSplits(job, numThreads);
+		splits = IOUtilFunctions.sortInputSplits(splits);
+
+		try 
+		{
+			ExecutorService pool = Executors.newFixedThreadPool(numThreads);
+			
+			//compute num rows per split
+			ArrayList<CountRowsTask> tasks = new ArrayList<CountRowsTask>();
+			for( int i=0; i<splits.length; i++ )
+				tasks.add(new CountRowsTask(splits[i], informat, job, _props.hasHeader(), i==0));
+			List<Future<Long>> cret = pool.invokeAll(tasks);
+
+			//compute row offset per split via cumsum on row counts
+			long offset = 0;
+			List<Long> offsets = new ArrayList<Long>();
+			for( Future<Long> count : cret ) {
+				offsets.add(offset);
+				offset += count.get();
+			}
+			
+			//read individial splits
+			ArrayList<ReadRowsTask> tasks2 = new ArrayList<ReadRowsTask>();
+			for( int i=0; i<splits.length; i++ )
+				tasks2.add( new ReadRowsTask(splits[i], informat, job, dest, offsets.get(i).intValue(), i==0));
+			List<Future<Object>> rret = pool.invokeAll(tasks2);
+			pool.shutdown();
+			
+			//error handling
+			for( Future<Object> read : rret )
+				read.get();
+		} 
+		catch (Exception e) {
+			throw new IOException("Failed parallel read of text csv input.", e);
+		}
+	}
+
+	@Override
+	protected Pair<Integer,Integer> computeCSVSize( Path path, JobConf job, FileSystem fs) 
+		throws IOException 
+	{		
+		int numThreads = OptimizerUtils.getParallelTextReadParallelism();
+		
+		FileInputFormat.addInputPath(job, path);
+		TextInputFormat informat = new TextInputFormat();
+		informat.configure(job);
+		InputSplit[] splits = informat.getSplits(job, numThreads);
+		
+		//compute number of columns
+		RecordReader<LongWritable, Text> reader = informat.getRecordReader(splits[0], job, Reporter.NULL);
+		LongWritable key = new LongWritable();
+		Text value = new Text();
+		reader.next(key, value);
+		int ncol = StringUtils.countMatches(value.toString(), _props.getDelim()) + 1;
+		reader.close();
+		
+		//compute number of rows
+		ExecutorService pool = Executors.newFixedThreadPool(numThreads);
+		
+		//compute num rows per split
+		int nrow = 0;
+		try {
+			ArrayList<CountRowsTask> tasks = new ArrayList<CountRowsTask>();
+			for( int i=0; i<splits.length; i++ )
+				tasks.add(new CountRowsTask(splits[i], informat, job, _props.hasHeader(), i==0));
+			List<Future<Long>> cret = pool.invokeAll(tasks);
+			for( Future<Long> count : cret ) 
+				nrow += count.get().intValue();
+		}
+		catch (Exception e) {
+			throw new IOException("Failed parallel read of text csv input.", e);
+		}
+		
+		return new Pair<Integer,Integer>(nrow, ncol);
+	}
+	
+	/**
+	 * 
+	 * 
+	 */
+	private static class CountRowsTask implements Callable<Long> 
+	{
+		private InputSplit _split = null;
+		private TextInputFormat _informat = null;
+		private JobConf _job = null;
+		private boolean _hasHeader = false;
+		private boolean _firstSplit = false;
+
+		public CountRowsTask(InputSplit split, TextInputFormat informat, JobConf job, boolean hasHeader, boolean first) {
+			_split = split;
+			_informat = informat;
+			_job = job;
+			_hasHeader = hasHeader;
+			_firstSplit = first;
+		}
+
+		@Override
+		public Long call() 
+			throws Exception 
+		{
+			
+			RecordReader<LongWritable, Text> reader = _informat.getRecordReader(_split, _job, Reporter.NULL);
+			LongWritable key = new LongWritable();
+			Text value = new Text();
+			long nrows = 0;
+			
+			// count rows from the first non-header row
+			try {
+				if ( _firstSplit && _hasHeader )
+					reader.next(key, value);
+				while (reader.next(key, value))
+					nrows++;
+			} 
+			finally {
+				IOUtilFunctions.closeSilently(reader);
+			}
+
+			return nrows;
+		}
+	}
+
+	/**
+	 * 
+	 * 
+	 */
+	private class ReadRowsTask implements Callable<Object> 
+	{
+		private InputSplit _split = null;
+		private TextInputFormat _informat = null;
+		private JobConf _job = null;
+		private FrameBlock _dest = null;
+		private int _offset = -1;
+		private boolean _isFirstSplit = false;
+		
+		
+		public ReadRowsTask(InputSplit split, TextInputFormat informat, JobConf job, 
+				FrameBlock dest, int offset, boolean first) 
+		{
+			_split = split;
+			_informat = informat;
+			_job = job;
+			_dest = dest;
+			_offset = offset;
+			_isFirstSplit = first;
+		}
+
+		@Override
+		public Object call() 
+			throws Exception 
+		{
+			readCSVFrameFrameFromInputSplit(_split, _informat, _job, _dest, _dest.getSchema(), 
+					_dest.getColumnNames(), _dest.getNumRows(), _dest.getNumColumns(), _offset, _isFirstSplit);
+			return null;
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
index 2f42a64..8345b47 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
@@ -42,9 +42,12 @@ import org.apache.sysml.runtime.matrix.data.FrameBlock;
 import org.apache.sysml.runtime.util.FastStringTokenizer;
 import org.apache.sysml.runtime.util.UtilFunctions;
 
+/**
+ * Single-threaded frame textcell reader.
+ * 
+ */
 public class FrameReaderTextCell extends FrameReader
 {
-
 	/**
 	 * 
 	 * @param fname
@@ -57,12 +60,13 @@ public class FrameReaderTextCell extends FrameReader
 	 * @throws IOException 
 	 */
 	@Override
-	public FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names,
-			long rlen, long clen)
-			throws IOException, DMLRuntimeException
+	public final FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names, long rlen, long clen)
+		throws IOException, DMLRuntimeException
 	{
 		//allocate output frame block
-		FrameBlock ret = createOutputFrameBlock(schema, names, rlen);
+		List<ValueType> lschema = createOutputSchema(schema, clen);
+		List<String> lnames = createOutputNames(names, clen);
+		FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
 		
 		//prepare file access
 		JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());	
@@ -72,11 +76,8 @@ public class FrameReaderTextCell extends FrameReader
 		//check existence and non-empty file
 		checkValidInputFile(fs, path); 
 	
-		//core read 
-		if( fs.isDirectory(path) )
-			readTextCellFrameFromHDFS(path, job, ret, schema, names, rlen, clen);
-		else
-			readRawTextCellFrameFromHDFS(path, job, fs, ret, schema, names, rlen, clen);
+		//core read (sequential/parallel)
+		readTextCellFrameFromHDFS(path, job, fs, ret, lschema, lnames, rlen, clen);
 		
 		return ret;
 	}
@@ -90,7 +91,7 @@ public class FrameReaderTextCell extends FrameReader
 	 * @throws IOException
 	 * @throws DMLRuntimeException
 	 */
-	public FrameBlock readFrameFromInputStream(InputStream is, long rlen, long clen) 
+	public final FrameBlock readFrameFromInputStream(InputStream is, long rlen, long clen) 
 		throws IOException, DMLRuntimeException {
 		return readFrameFromInputStream(is, getDefSchema(clen), getDefColNames(clen), rlen, clen);
 	}
@@ -106,83 +107,98 @@ public class FrameReaderTextCell extends FrameReader
 	 * @throws DMLRuntimeException 
 	 * @throws IOException 
 	 */
-	public FrameBlock readFrameFromInputStream(InputStream is, List<ValueType> schema, List<String> names, long rlen, long clen) 
+	public final FrameBlock readFrameFromInputStream(InputStream is, List<ValueType> schema, List<String> names, long rlen, long clen) 
 		throws IOException, DMLRuntimeException 
 	{
 		//allocate output frame block
-		FrameBlock ret = createOutputFrameBlock(schema, names, rlen);
+		List<ValueType> lschema = createOutputSchema(schema, clen);
+		List<String> lnames = createOutputNames(names, clen);
+		FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
 	
 		//core read 
-		readRawTextCellFrameFromInputStream(is, ret, schema, names, rlen, clen);
+		readRawTextCellFrameFromInputStream(is, ret, lschema, lnames, rlen, clen);
 		
 		return ret;
 	}
 	
-
 	/**
 	 * 
 	 * @param path
 	 * @param job
+	 * @param fs
 	 * @param dest
 	 * @param schema
 	 * @param names
 	 * @param rlen
 	 * @param clen
-	 * @return
 	 * @throws IOException
 	 */
-	private void readTextCellFrameFromHDFS( Path path, JobConf job, FrameBlock dest, 
+	protected void readTextCellFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest, 
 			List<ValueType> schema, List<String> names, long rlen, long clen)
 		throws IOException
 	{
-		FileInputFormat.addInputPath(job, path);
-		TextInputFormat informat = new TextInputFormat();
-		informat.configure(job);
-		InputSplit[] splits = informat.getSplits(job, 1);
+		if( fs.isDirectory(path) ) {
+			FileInputFormat.addInputPath(job, path);
+			TextInputFormat informat = new TextInputFormat();
+			informat.configure(job);
+			InputSplit[] splits = informat.getSplits(job, 1);
+			for(InputSplit split: splits)
+				readTextCellFrameFromInputSplit(split, informat, job, dest);
+		}
+		else {
+			readRawTextCellFrameFromHDFS(path, job, fs, dest, schema, names, rlen, clen);
+		}
+	}
+
+	/**
+	 * 
+	 * @param split
+	 * @param dest
+	 * @param schema
+	 * @param names
+	 * @param rlen
+	 * @param clen
+	 * @throws IOException
+	 */
+	protected final void readTextCellFrameFromInputSplit( InputSplit split, TextInputFormat informat, JobConf job, FrameBlock dest)
+		throws IOException
+	{
+		List<ValueType> schema = dest.getSchema();
+		int rlen = dest.getNumRows();
+		int clen = dest.getNumColumns();
+		
+		//create record reader
+		RecordReader<LongWritable,Text> reader = informat.getRecordReader(split, job, Reporter.NULL);
 		
 		LongWritable key = new LongWritable();
 		Text value = new Text();
+		FastStringTokenizer st = new FastStringTokenizer(' ');
 		int row = -1;
 		int col = -1;
 		
 		try
 		{
-			FastStringTokenizer st = new FastStringTokenizer(' ');
-			
-			for(InputSplit split: splits)
-			{
-				RecordReader<LongWritable,Text> reader = informat.getRecordReader(split, job, Reporter.NULL);
-			
-				try
-				{
-					while( reader.next(key, value) )
-					{
-						st.reset( value.toString() ); //reinit tokenizer
-						row = st.nextInt()-1;
-						col = st.nextInt()-1;
-						dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), st.nextToken()));
-					}
-				}
-				finally
-				{
-					if( reader != null )
-						reader.close();
-				}
+			while( reader.next(key, value) ) {
+				st.reset( value.toString() ); //reinit tokenizer
+				row = st.nextInt()-1;
+				col = st.nextInt()-1;
+				dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), st.nextToken()));
 			}
 		}
-		catch(Exception ex)
+		catch(Exception ex) 
 		{
 			//post-mortem error handling and bounds checking
-			if( row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen )
-			{
+			if( row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen ) {
 				throw new IOException("Frame cell ["+(row+1)+","+(col+1)+"] " +
 									  "out of overall frame range [1:"+rlen+",1:"+clen+"].");
 			}
-			else
-			{
+			else {
 				throw new IOException( "Unable to read frame in text cell format.", ex );
 			}
 		}
+		finally {
+			IOUtilFunctions.closeSilently(reader);
+		}		
 	}
 
 	
@@ -199,7 +215,7 @@ public class FrameReaderTextCell extends FrameReader
 	 * @return
 	 * @throws IOException
 	 */
-	private void readRawTextCellFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest, 
+	protected final void readRawTextCellFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest, 
 			List<ValueType> schema, List<String> names, long rlen, long clen)
 		throws IOException
 	{
@@ -221,21 +237,20 @@ public class FrameReaderTextCell extends FrameReader
 	 * @return
 	 * @throws IOException
 	 */
-	private void readRawTextCellFrameFromInputStream( InputStream is, FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen)
-			throws IOException
+	protected final void readRawTextCellFrameFromInputStream( InputStream is, FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen)
+		throws IOException
 	{
+		//create buffered reader
 		BufferedReader br = new BufferedReader(new InputStreamReader( is ));	
 		
 		String value = null;
+		FastStringTokenizer st = new FastStringTokenizer(' ');
 		int row = -1;
 		int col = -1;
 		
 		try
 		{			
-			FastStringTokenizer st = new FastStringTokenizer(' ');
-			
-			while( (value=br.readLine())!=null )
-			{
+			while( (value=br.readLine())!=null ) {
 				st.reset( value ); //reinit tokenizer
 				row = st.nextInt()-1;
 				col = st.nextInt()-1;	
@@ -245,18 +260,15 @@ public class FrameReaderTextCell extends FrameReader
 		catch(Exception ex)
 		{
 			//post-mortem error handling and bounds checking
-			if( row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen ) 
-			{
+			if( row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen ) {
 				throw new IOException("Frame cell ["+(row+1)+","+(col+1)+"] " +
 									  "out of overall frame range [1:"+rlen+",1:"+clen+"].", ex);
 			}
-			else
-			{
+			else {
 				throw new IOException( "Unable to read frame in raw text cell format.", ex );
 			}
 		}
-		finally
-		{
+		finally {
 			IOUtilFunctions.closeSilently(br);
 		}
 	}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java
new file mode 100644
index 0000000..ce3993f
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+
+/**
+ * Multi-threaded frame textcell reader.
+ * 
+ */
+public class FrameReaderTextCellParallel extends FrameReaderTextCell
+{	
+	/**
+	 * 
+	 * @param path
+	 * @param job
+	 * @param fs
+	 * @param dest
+	 * @param schema
+	 * @param names
+	 * @param rlen
+	 * @param clen
+	 * @throws IOException
+	 */
+	@Override
+	protected void readTextCellFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest, 
+			List<ValueType> schema, List<String> names, long rlen, long clen)
+		throws IOException
+	{
+		int numThreads = OptimizerUtils.getParallelTextReadParallelism();
+		
+		FileInputFormat.addInputPath(job, path);
+		TextInputFormat informat = new TextInputFormat();
+		informat.configure(job);
+		
+		try 
+		{
+			//create read tasks for all splits
+			ExecutorService pool = Executors.newFixedThreadPool(numThreads);
+			InputSplit[] splits = informat.getSplits(job, numThreads);
+			ArrayList<ReadTask> tasks = new ArrayList<ReadTask>();
+			for( InputSplit split : splits )
+				tasks.add(new ReadTask(split, informat, job, dest));
+			
+			//wait until all tasks have been executed
+			List<Future<Object>> rt = pool.invokeAll(tasks);	
+			pool.shutdown();
+				
+			//check for exceptions
+			for( Future<Object> task : rt )
+				task.get();
+		} 
+		catch (Exception e) {
+			throw new IOException("Failed parallel read of text cell input.", e);
+		}
+	}
+	
+	/**
+	 * 
+	 */
+	public class ReadTask implements Callable<Object> 
+	{
+		private InputSplit _split = null;
+		private TextInputFormat _informat = null;
+		private JobConf _job = null;
+		private FrameBlock _dest = null;
+		
+		public ReadTask( InputSplit split, TextInputFormat informat, JobConf job, FrameBlock dest ) {
+			_split = split;
+			_informat = informat;
+			_job = job;
+			_dest = dest;
+		}
+
+		@Override
+		public Object call() throws Exception {
+			readTextCellFrameFromInputSplit(_split, _informat, _job, _dest);
+			return null;
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
index 69efec0..93a8818 100644
--- a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
+++ b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
@@ -24,10 +24,15 @@ import java.io.ByteArrayOutputStream;
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Arrays;
+import java.util.Comparator;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.RecordReader;
 import org.apache.sysml.runtime.util.LocalFileUtils;
 import org.apache.sysml.runtime.util.UtilFunctions;
@@ -156,4 +161,27 @@ public class IOUtilFunctions
 		input.close();		
 		return bos.toString("UTF-8");
 	}
+
+	/**
+	 * 
+	 * @param splits
+	 * @return
+	 */
+	public static InputSplit[] sortInputSplits(InputSplit[] splits) {
+		if (splits[0] instanceof FileSplit) {
+			// The splits do not always arrive in order by file name.
+			// Sort the splits lexicographically by path so that the header will
+			// be in the first split.
+			// Note that we're assuming that the splits come in order by offset
+			Arrays.sort(splits, new Comparator<InputSplit>() {
+				@Override
+				public int compare(InputSplit o1, InputSplit o2) {
+					Path p1 = ((FileSplit) o1).getPath();
+					Path p2 = ((FileSplit) o2).getPath();
+					return p1.toString().compareTo(p2.toString());
+				}
+			});
+		}		
+		return splits;
+	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java
index 64c055c..8041ba7 100644
--- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java
@@ -21,8 +21,6 @@ package org.apache.sysml.runtime.io;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -33,7 +31,6 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileSplit;
 import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.RecordReader;
@@ -84,21 +81,7 @@ public class ReaderTextCSVParallel extends MatrixReader
 		informat.configure(job);
 
 		InputSplit[] splits = informat.getSplits(job, _numThreads);
-
-		if (splits[0] instanceof FileSplit) {
-			// The splits do not always arrive in order by file name.
-			// Sort the splits lexicographically by path so that the header will
-			// be in the first split.
-			// Note that we're assuming that the splits come in order by offset
-			Arrays.sort(splits, new Comparator<InputSplit>() {
-				@Override
-				public int compare(InputSplit o1, InputSplit o2) {
-					Path p1 = ((FileSplit) o1).getPath();
-					Path p2 = ((FileSplit) o2).getPath();
-					return p1.toString().compareTo(p2.toString());
-				}
-			});
-		}
+		splits = IOUtilFunctions.sortInputSplits(splits);
 
 		// check existence and non-empty file
 		checkValidInputFile(fs, path);

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
index b07d09e..7d45ebc 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
@@ -62,35 +62,94 @@ public class FrameReadWriteTest extends AutomatedTestBase
 	}
 
 	@Test
-	public void testFrameStringsStrings()  {
-		runFrameReadWriteTest(schemaStrings, schemaStrings, false);
+	public void testFrameStringsStringsBinary()  {
+		runFrameReadWriteTest(OutputInfo.BinaryBlockOutputInfo, schemaStrings, schemaStrings, false);
 	}
 	
 	@Test
-	public void testFrameStringsStringsParallel()  { 
-		runFrameReadWriteTest(schemaStrings, schemaStrings, true);
+	public void testFrameStringsStringsBinaryParallel()  { 
+		runFrameReadWriteTest(OutputInfo.BinaryBlockOutputInfo, schemaStrings, schemaStrings, true);
 	}
 	
 	@Test
-	public void testFrameMixedStrings()  {
-		runFrameReadWriteTest(schemaMixed, schemaStrings, false);
+	public void testFrameMixedStringsBinary()  {
+		runFrameReadWriteTest(OutputInfo.BinaryBlockOutputInfo, schemaMixed, schemaStrings, false);
 	}
 	
 	@Test
-	public void testFrameStringsMixedParallel()  {
-		runFrameReadWriteTest(schemaStrings, schemaMixed, true);
+	public void testFrameStringsMixedBinaryParallel()  {
+		runFrameReadWriteTest(OutputInfo.BinaryBlockOutputInfo, schemaStrings, schemaMixed, true);
 	}
 	
 	@Test
-	public void testFrameMixedMixed()  {
-		runFrameReadWriteTest(schemaMixed, schemaMixed, false);
+	public void testFrameMixedMixedBinary()  {
+		runFrameReadWriteTest(OutputInfo.BinaryBlockOutputInfo, schemaMixed, schemaMixed, false);
 	}
 	
 	@Test
-	public void testFrameMixedMixedParallel()  {
-		runFrameReadWriteTest(schemaMixed, schemaMixed, true);
+	public void testFrameMixedMixedBinaryParallel()  {
+		runFrameReadWriteTest(OutputInfo.BinaryBlockOutputInfo, schemaMixed, schemaMixed, true);
 	}
 
+	@Test
+	public void testFrameStringsStringsTextCell()  {
+		runFrameReadWriteTest(OutputInfo.TextCellOutputInfo, schemaStrings, schemaStrings, false);
+	}
+	
+	@Test
+	public void testFrameStringsStringsTextCellParallel()  { 
+		runFrameReadWriteTest(OutputInfo.TextCellOutputInfo, schemaStrings, schemaStrings, true);
+	}
+	
+	@Test
+	public void testFrameMixedStringsTextCell()  {
+		runFrameReadWriteTest(OutputInfo.TextCellOutputInfo, schemaMixed, schemaStrings, false);
+	}
+	
+	@Test
+	public void testFrameStringsMixedTextCellParallel()  {
+		runFrameReadWriteTest(OutputInfo.TextCellOutputInfo, schemaStrings, schemaMixed, true);
+	}
+	
+	@Test
+	public void testFrameMixedMixedTextCell()  {
+		runFrameReadWriteTest(OutputInfo.TextCellOutputInfo, schemaMixed, schemaMixed, false);
+	}
+	
+	@Test
+	public void testFrameMixedMixedTextCellParallel()  {
+		runFrameReadWriteTest(OutputInfo.TextCellOutputInfo, schemaMixed, schemaMixed, true);
+	}
+
+	@Test
+	public void testFrameStringsStringsTextCSV()  {
+		runFrameReadWriteTest(OutputInfo.CSVOutputInfo, schemaStrings, schemaStrings, false);
+	}
+	
+	@Test
+	public void testFrameStringsStringsTextCSVParallel()  { 
+		runFrameReadWriteTest(OutputInfo.CSVOutputInfo, schemaStrings, schemaStrings, true);
+	}
+	
+	@Test
+	public void testFrameMixedStringsTextCSV()  {
+		runFrameReadWriteTest(OutputInfo.CSVOutputInfo, schemaMixed, schemaStrings, false);
+	}
+	
+	@Test
+	public void testFrameStringsMixedTextCSVParallel()  {
+		runFrameReadWriteTest(OutputInfo.CSVOutputInfo, schemaStrings, schemaMixed, true);
+	}
+	
+	@Test
+	public void testFrameMixedMixedTextCSV()  {
+		runFrameReadWriteTest(OutputInfo.CSVOutputInfo, schemaMixed, schemaMixed, false);
+	}
+	
+	@Test
+	public void testFrameMixedMixedTextCSVParallel()  {
+		runFrameReadWriteTest(OutputInfo.CSVOutputInfo, schemaMixed, schemaMixed, true);
+	}
 	
 	/**
 	 * 
@@ -98,7 +157,7 @@ public class FrameReadWriteTest extends AutomatedTestBase
 	 * @param sparseM2
 	 * @param instType
 	 */
-	private void runFrameReadWriteTest( ValueType[] schema1, ValueType[] schema2, boolean parallel)
+	private void runFrameReadWriteTest( OutputInfo oinfo, ValueType[] schema1, ValueType[] schema2, boolean parallel)
 	{
 		boolean oldParText = CompilerConfig.FLAG_PARREADWRITE_TEXT;
 		boolean oldParBin = CompilerConfig.FLAG_PARREADWRITE_BINARY;
@@ -129,9 +188,7 @@ public class FrameReadWriteTest extends AutomatedTestBase
 			fprop.setDelim(DELIMITER);
 			fprop.setHeader(HEADER);
 			
-			writeAndVerifyData(OutputInfo.TextCellOutputInfo, frame1, frame2, fprop);
-			writeAndVerifyData(OutputInfo.CSVOutputInfo, frame1, frame2, fprop);
-			writeAndVerifyData(OutputInfo.BinaryBlockOutputInfo, frame1, frame2, fprop);
+			writeAndVerifyData(oinfo, frame1, frame2, fprop);
 		}
 		catch(Exception ex) {
 			ex.printStackTrace();



[3/4] incubator-systemml git commit: [SYSTEMML-561] Performance double frame-matrix casting (columns, cache)

Posted by mb...@apache.org.
[SYSTEMML-561] Performance double frame-matrix casting (columns, cache)

Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/288438b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/288438b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/288438b0

Branch: refs/heads/master
Commit: 288438b0417dbb78977caeee5cbbc421fae0c5e9
Parents: 2163bfb
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Tue Jun 7 19:03:19 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Tue Jun 7 19:03:19 2016 -0700

----------------------------------------------------------------------
 .../sysml/runtime/matrix/data/FrameBlock.java   | 15 ++++++++
 .../sysml/runtime/util/DataConverter.java       | 39 +++++++++++++++++---
 2 files changed, 48 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/288438b0/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
index aef736b..53d2001 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
@@ -342,6 +342,21 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
 	}
 	
 	/**
+	 * 
+	 * @param c
+	 * @return
+	 */
+	public Object getColumn(int c) {
+		switch(_schema.get(c)) {
+			case STRING:  return ((StringArray)_coldata.get(c))._data; 
+			case BOOLEAN: return ((BooleanArray)_coldata.get(c))._data;
+			case INT:     return ((LongArray)_coldata.get(c))._data;
+			case DOUBLE:  return ((DoubleArray)_coldata.get(c))._data;
+			default:      return null;
+	 	}
+	}
+	
+	/**
 	 * Get a row iterator over the frame where all fields are encoded
 	 * as strings independent of their value types.  
 	 * 

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/288438b0/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
index eddd2e3..6b46c6c 100644
--- a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
@@ -624,14 +624,41 @@ public class DataConverter
 	public static MatrixBlock convertToMatrixBlock(FrameBlock frame) 
 		throws DMLRuntimeException
 	{
-		MatrixBlock mb = new MatrixBlock(frame.getNumRows(), frame.getNumColumns(), false);
+		int m = frame.getNumRows();
+		int n = frame.getNumColumns();
+		MatrixBlock mb = new MatrixBlock(m, n, false);
+		mb.allocateDenseBlock();
 		
 		List<ValueType> schema = frame.getSchema();
-		for( int i=0; i<frame.getNumRows(); i++ ) 
-			for( int j=0; j<frame.getNumColumns(); j++ ) {
-				mb.appendValue(i, j, UtilFunctions.objectToDouble(
-						schema.get(j), frame.get(i, j)));
-			}
+		int dFreq = Collections.frequency(schema, ValueType.DOUBLE);
+		
+		if( dFreq == schema.size() ) {
+			// special case double schema (without cell-object creation, 
+			// cache-friendly row-column copy)
+			double[][] a = new double[n][];
+			double[] c = mb.getDenseBlock();
+			for( int j=0; j<n; j++ )
+				a[j] = (double[])frame.getColumn(j);			
+			int blocksizeIJ = 16; //blocks of a+overhead/c in L1 cache
+			for( int bi=0; bi<m; bi+=blocksizeIJ )
+				for( int bj=0; bj<n; bj+=blocksizeIJ ) {
+					int bimin = Math.min(bi+blocksizeIJ, m);
+					int bjmin = Math.min(bj+blocksizeIJ, n);
+					for( int i=bi, aix=bi*n; i<bimin; i++, aix+=n )
+						for( int j=bj; j<bjmin; j++ )
+							c[aix+j] = a[j][i];
+				}
+		}
+		else { 
+			//general case
+			for( int i=0; i<frame.getNumRows(); i++ ) 
+				for( int j=0; j<frame.getNumColumns(); j++ ) {
+					mb.appendValue(i, j, UtilFunctions.objectToDouble(
+							schema.get(j), frame.get(i, j)));
+				}
+		}
+		
+		//post-processing
 		mb.examSparsity();
 		
 		return mb;


[2/4] incubator-systemml git commit: [SYSTEMML-630] New parallel binary block frame reader, cleanup

Posted by mb...@apache.org.
[SYSTEMML-630] New parallel binary block frame reader, cleanup

Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/2163bfb4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/2163bfb4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/2163bfb4

Branch: refs/heads/master
Commit: 2163bfb44ff26915adf18dfa4def3af8dcc6822e
Parents: fdf5518
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Tue Jun 7 17:59:01 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Tue Jun 7 17:59:01 2016 -0700

----------------------------------------------------------------------
 .../apache/sysml/runtime/io/FrameReader.java    |  25 +++++
 .../runtime/io/FrameReaderBinaryBlock.java      | 100 +++++++++--------
 .../io/FrameReaderBinaryBlockParallel.java      | 108 +++++++++++++++++++
 .../sysml/runtime/io/FrameReaderFactory.java    |  55 +++-------
 .../sysml/runtime/matrix/data/FrameBlock.java   |   4 +-
 5 files changed, 207 insertions(+), 85 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/2163bfb4/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReader.java b/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
index a39d129..6a4b469 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
@@ -22,6 +22,7 @@ package org.apache.sysml.runtime.io;
 import java.io.EOFException;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 
@@ -165,6 +166,30 @@ public abstract class FrameReader
 	
 	/**
 	 * 
+	 * @param schema
+	 * @param ncol
+	 * @return
+	 */
+	protected static List<ValueType> createOutputSchema(List<ValueType> schema, long ncol) {
+		if( schema.size()==1 && ncol > 1 )
+			return Collections.nCopies((int)ncol, schema.get(0));
+		return schema;
+	}
+	
+	/**
+	 * 
+	 * @param names
+	 * @param ncol
+	 * @return
+	 */
+	protected static List<String> createOutputNames(List<String> names, long ncol) {
+		if( names.size()==1 && ncol > 1 )
+			return FrameBlock.createColNames((int)ncol);
+		return names;
+	}
+	
+	/**
+	 * 
 	 * @param fs
 	 * @param path
 	 * @throws IOException 

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/2163bfb4/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
index 534decf..f2a4a8a 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
@@ -33,7 +33,10 @@ import org.apache.sysml.runtime.DMLRuntimeException;
 import org.apache.sysml.runtime.matrix.data.FrameBlock;
 
 
-
+/**
+ * Single-threaded frame binary block reader.
+ * 
+ */
 public class FrameReaderBinaryBlock extends FrameReader
 {
 	/**
@@ -48,12 +51,13 @@ public class FrameReaderBinaryBlock extends FrameReader
 	 * @throws IOException 
 	 */
 	@Override
-	public FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names,
-			long rlen, long clen) 
-			throws IOException, DMLRuntimeException 
+	public final FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names, long rlen, long clen) 
+		throws IOException, DMLRuntimeException 
 	{
 		//allocate output frame block
-		FrameBlock ret = createOutputFrameBlock(schema, names, rlen);
+		List<ValueType> lschema = createOutputSchema(schema, clen);
+		List<String> lnames = createOutputNames(names, clen);
+		FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
 		
 		//prepare file access
 		JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());	
@@ -63,21 +67,13 @@ public class FrameReaderBinaryBlock extends FrameReader
 		//check existence and non-empty file
 		checkValidInputFile(fs, path); 
 	
-		//core read 
+		//core read (sequential/parallel)
 		readBinaryBlockFrameFromHDFS(path, job, fs, ret, rlen, clen);
 		
 		return ret;
 	}
 	
 	/**
-	 * Note: For efficiency, we directly use SequenceFile.Reader instead of SequenceFileInputFormat-
-	 * InputSplits-RecordReader (SequenceFileRecordReader). First, this has no drawbacks since the
-	 * SequenceFileRecordReader internally uses SequenceFile.Reader as well. Second, it is 
-	 * advantageous if the actual sequence files are larger than the file splits created by   
-	 * informat.getSplits (which is usually aligned to the HDFS block size) because then there is 
-	 * overhead for finding the actual split between our 1k-1k blocks. This case happens
-	 * if the read frame was create by CP or when jobs directly write to large output files 
-	 * (e.g., parfor matrix partitioning).
 	 * 
 	 * @param path
 	 * @param job
@@ -89,46 +85,62 @@ public class FrameReaderBinaryBlock extends FrameReader
 	 * @throws IOException
 	 * @throws DMLRuntimeException 
 	 */
-	@SuppressWarnings("deprecation")
-	private static void readBinaryBlockFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest, long rlen, long clen )
+	protected void readBinaryBlockFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest, long rlen, long clen )
+		throws IOException, DMLRuntimeException
+	{
+		//sequential read from sequence files
+		for( Path lpath : getSequenceFilePaths(fs, path) ) //1..N files 
+			readBinaryBlockFrameFromSequenceFile(lpath, job, fs, dest);
+	}
+	
+	/**
+	 * 
+	 * @param path
+	 * @param job
+	 * @param fs
+	 * @param dest
+	 * @throws IOException
+	 * @throws DMLRuntimeException
+	 */
+	@SuppressWarnings({ "deprecation", "resource" })
+	protected final void readBinaryBlockFrameFromSequenceFile( Path path, JobConf job, FileSystem fs, FrameBlock dest )
 		throws IOException, DMLRuntimeException
 	{
+		int rlen = dest.getNumRows();
+		int clen = dest.getNumColumns();
+		
+		//directly read from sequence files (individual partfiles)
+		SequenceFile.Reader reader = new SequenceFile.Reader(fs,path,job);
 		LongWritable key = new LongWritable(-1L);
 		FrameBlock value = new FrameBlock();
 		
-		for( Path lpath : getSequenceFilePaths(fs, path) ) //1..N files 
+		try
 		{
-			//directly read from sequence files (individual partfiles)
-			SequenceFile.Reader reader = new SequenceFile.Reader(fs,lpath,job);
-			
-			try
-			{
-				//note: next(key, value) does not yet exploit the given serialization classes, record reader does but is generally slower.
-				while( reader.next(key, value) ) {	
-					int row_offset = (int)(key.get()-1);
-					
-					int rows = value.getNumRows();
-					int cols = value.getNumColumns();
+			//note: next(key, value) does not yet exploit the given serialization classes, record reader does but is generally slower.
+			while( reader.next(key, value) ) {	
+				int row_offset = (int)(key.get()-1);
+				
+				int rows = value.getNumRows();
+				int cols = value.getNumColumns();
 
-					if(rows == 0 || cols == 0)	//Empty block, ignore it.
-						continue;
-					
-					//bound check per block
-					if( row_offset + rows < 0 || row_offset + rows > rlen ) {
-						throw new IOException("Frame block ["+(row_offset+1)+":"+(row_offset+rows)+","+":"+"] " +
-								              "out of overall frame range [1:"+rlen+",1:"+clen+"].");
-					}
-			
-					dest.copy( row_offset, row_offset+rows-1, 
-							0, cols-1, value);
+				if(rows == 0 || cols == 0)	//Empty block, ignore it.
+					continue;
+				
+				//bound check per block
+				if( row_offset + rows < 0 || row_offset + rows > rlen ) {
+					throw new IOException("Frame block ["+(row_offset+1)+":"+(row_offset+rows)+","+":"+"] " +
+							              "out of overall frame range [1:"+rlen+",1:"+clen+"].");
 				}
-			}
-			finally {
-				IOUtilFunctions.closeSilently(reader);
+		
+				dest.copy( row_offset, row_offset+rows-1, 
+						0, cols-1, value);
 			}
 		}
-	}
-
+		finally {
+			IOUtilFunctions.closeSilently(reader);
+		}
+	}	
+	
 	/**
 	 * Specific functionality of FrameReaderBinaryBlock, mostly used for testing.
 	 * 

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/2163bfb4/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java
new file mode 100644
index 0000000..d684af0
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+
+
+/**
+ * Multi-threaded frame binary block reader.
+ * 
+ */
+public class FrameReaderBinaryBlockParallel extends FrameReaderBinaryBlock
+{
+	/**
+	 * 
+	 * @param path
+	 * @param job
+	 * @param fs 
+	 * @param dest
+	 * @param rlen
+	 * @param clen
+	 * 
+	 * @throws IOException
+	 * @throws DMLRuntimeException 
+	 */
+	protected void readBinaryBlockFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest, long rlen, long clen )
+		throws IOException, DMLRuntimeException
+	{
+		int numThreads = OptimizerUtils.getParallelBinaryReadParallelism();
+		
+		try 
+		{
+			//create read tasks for all files
+			ExecutorService pool = Executors.newFixedThreadPool(numThreads);
+			ArrayList<ReadFileTask> tasks = new ArrayList<ReadFileTask>();
+			for( Path lpath : getSequenceFilePaths(fs, path) ){
+				ReadFileTask t = new ReadFileTask(lpath, job, fs, dest);
+				tasks.add(t);
+			}
+
+			//wait until all tasks have been executed
+			List<Future<Object>> rt = pool.invokeAll(tasks);	
+			pool.shutdown();
+			
+			//check for exceptions and aggregate nnz
+			for( Future<Object> task : rt )
+				task.get();
+		} 
+		catch (Exception e) {
+			throw new IOException("Failed parallel read of binary block input.", e);
+		}
+	}
+	
+	/**
+	 * 
+	 */
+	private class ReadFileTask implements Callable<Object> 
+	{
+		private Path _path = null;
+		private JobConf _job = null;
+		private FileSystem _fs = null;
+		private FrameBlock _dest = null;
+		
+		public ReadFileTask(Path path, JobConf job, FileSystem fs, FrameBlock dest) {
+			_path = path;
+			_fs = fs;
+			_job = job;
+			_dest = dest;
+		}
+
+		@Override
+		public Object call() throws Exception 
+		{
+			readBinaryBlockFrameFromSequenceFile(_path, _job, _fs, _dest);
+			return null;
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/2163bfb4/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
index 35ec5f7..51606df 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
@@ -19,6 +19,8 @@
 
 package org.apache.sysml.runtime.io;
 
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
 import org.apache.sysml.runtime.DMLRuntimeException;
 import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
 import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
@@ -39,25 +41,10 @@ public class FrameReaderFactory
 	public static FrameReader createFrameReader( InputInfo iinfo ) 
 		throws DMLRuntimeException
 	{
-		FrameReader reader = null;
+		FileFormatProperties props = (iinfo==InputInfo.CSVInputInfo) ?
+			new CSVFileFormatProperties() : null;		
 		
-		if( iinfo == InputInfo.TextCellInputInfo )
-		{
-			reader = new FrameReaderTextCell();	
-		}
-		else if( iinfo == InputInfo.CSVInputInfo )
-		{
-			reader = new FrameReaderTextCSV(new CSVFileFormatProperties());
-		}
-		else if( iinfo == InputInfo.BinaryBlockInputInfo ) {
-			reader = new FrameReaderBinaryBlock();
-		}
-		else {
-			throw new DMLRuntimeException("Failed to create frame reader for unknown input info: "
-		                                   + InputInfo.inputInfoToString(iinfo));
-		}
-		
-		return reader;
+		return createFrameReader(iinfo, props);
 	}
 	
 	/**
@@ -66,31 +53,18 @@ public class FrameReaderFactory
 	 * @return
 	 * @throws DMLRuntimeException
 	 */
-	public static FrameReader createFrameReader( ReadProperties props ) 
+	public static FrameReader createFrameReader( ReadProperties rprops ) 
 		throws DMLRuntimeException
 	{
 		//check valid read properties
-		if( props == null )
+		if( rprops == null )
 			throw new DMLRuntimeException("Failed to create frame reader with empty properties.");
 		
-		FrameReader reader = null;
-		InputInfo iinfo = props.inputInfo;
-
-		if( iinfo == InputInfo.TextCellInputInfo ) {
-			reader = new FrameReaderTextCell();
-		}
-		else if( iinfo == InputInfo.CSVInputInfo ) {
-			reader = new FrameReaderTextCSV( props.formatProperties!=null ? (CSVFileFormatProperties)props.formatProperties : new CSVFileFormatProperties());
-		}
-		else if( iinfo == InputInfo.BinaryBlockInputInfo ) {
-			reader = new FrameReaderBinaryBlock();
-		}
-		else {
-			throw new DMLRuntimeException("Failed to create frame reader for unknown input info: "
-		                                   + InputInfo.inputInfoToString(iinfo));
-		}
-		
-		return reader;
+		InputInfo iinfo = rprops.inputInfo;
+		FileFormatProperties props = (iinfo==InputInfo.CSVInputInfo) ? ((rprops.formatProperties!=null) ? 
+			(CSVFileFormatProperties)rprops.formatProperties : new CSVFileFormatProperties()) : null;		
+			
+		return createFrameReader(iinfo, props);
 	}
 
 	
@@ -114,7 +88,10 @@ public class FrameReaderFactory
 			reader = new FrameReaderTextCSV( (CSVFileFormatProperties)props);
 		}
 		else if( iinfo == InputInfo.BinaryBlockInputInfo ) {
-			reader = new FrameReaderBinaryBlock();
+			if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_BINARYFORMATS) )
+				reader = new FrameReaderBinaryBlockParallel();
+			else
+				reader = new FrameReaderBinaryBlock();
 		}
 		else {
 			throw new DMLRuntimeException("Failed to create frame reader for unknown input info: "

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/2163bfb4/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
index b851704..aef736b 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
@@ -198,7 +198,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
 	 * @param size
 	 * @return
 	 */
-	private static List<String> createColNames(int size) {
+	public static List<String> createColNames(int size) {
 		ArrayList<String> ret = new ArrayList<String>(size);
 		for( int i=1; i<=size; i++ )
 			ret.add(createColName(i));
@@ -210,7 +210,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
 	 * @param i
 	 * @return
 	 */
-	private static String createColName(int i) {
+	public static String createColName(int i) {
 		return "C" + i;
 	}