You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/06/08 19:17:44 UTC
[1/4] incubator-systemml git commit: [SYSTEMML-630] New parallel
csv/textcell frame writers, cleanup
Repository: incubator-systemml
Updated Branches:
refs/heads/master 6bbae619a -> dea42de1f
[SYSTEMML-630] New parallel csv/textcell frame writers, cleanup
This patch adds, similar to the parallel binary block frame writer,
parallel frame writers for csv and textcell. On a 1Mx1k (8GB binary,
~25-30GB in csv/text) scenario, multi-threaded write achieved the
following improvements:
* Textcell: 588s (single-threaded) -> 59s (multi-threaded)
* CSV: 518s (single-threaded) -> 47s (multi-threaded)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/fdf55181
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/fdf55181
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/fdf55181
Branch: refs/heads/master
Commit: fdf551813572d6fd4f0d8076a525d4b1b3ab1b53
Parents: 6bbae61
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Tue Jun 7 13:31:15 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Tue Jun 7 13:31:15 2016 -0700
----------------------------------------------------------------------
.../io/FrameWriterBinaryBlockParallel.java | 7 +-
.../sysml/runtime/io/FrameWriterFactory.java | 10 +-
.../sysml/runtime/io/FrameWriterTextCSV.java | 62 ++++++---
.../runtime/io/FrameWriterTextCSVParallel.java | 133 +++++++++++++++++++
.../sysml/runtime/io/FrameWriterTextCell.java | 68 ++++++----
.../runtime/io/FrameWriterTextCellParallel.java | 128 ++++++++++++++++++
.../sysml/runtime/matrix/data/FrameBlock.java | 44 +++++-
7 files changed, 396 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/fdf55181/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlockParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlockParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlockParallel.java
index f0b5bd1..2ef7692 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlockParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlockParallel.java
@@ -56,13 +56,12 @@ public class FrameWriterBinaryBlockParallel extends FrameWriterBinaryBlock
* @throws DMLRuntimeException
*/
protected void writeBinaryBlockFrameToHDFS( Path path, JobConf job, FrameBlock src, long rlen, long clen )
- throws IOException, DMLRuntimeException
+ throws IOException, DMLRuntimeException
{
//estimate output size and number of output blocks (min 1)
int blen = ConfigurationManager.getBlocksize();
- int numPartFiles = (int)(OptimizerUtils.estimatePartitionedSizeExactSparsity(rlen, clen, blen, blen, rlen*clen)
- / InfrastructureAnalyzer.getHDFSBlockSize());
- numPartFiles = Math.max(numPartFiles, 1);
+ int numPartFiles = Math.max((int)(OptimizerUtils.estimatePartitionedSizeExactSparsity(rlen, clen, blen, blen, rlen*clen)
+ / InfrastructureAnalyzer.getHDFSBlockSize()), 1);
//determine degree of parallelism
int numThreads = OptimizerUtils.getParallelBinaryWriteParallelism();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/fdf55181/src/main/java/org/apache/sysml/runtime/io/FrameWriterFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterFactory.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterFactory.java
index a7f132f..c1021e6 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterFactory.java
@@ -57,12 +57,18 @@ public class FrameWriterFactory
FrameWriter writer = null;
if( oinfo == OutputInfo.TextCellOutputInfo ) {
- writer = new FrameWriterTextCell();
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS) )
+ writer = new FrameWriterTextCellParallel();
+ else
+ writer = new FrameWriterTextCell();
}
else if( oinfo == OutputInfo.CSVOutputInfo ) {
if( props!=null && !(props instanceof CSVFileFormatProperties) )
throw new DMLRuntimeException("Wrong type of file format properties for CSV writer.");
- writer = new FrameWriterTextCSV((CSVFileFormatProperties)props);
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS) )
+ writer = new FrameWriterTextCSVParallel((CSVFileFormatProperties)props);
+ else
+ writer = new FrameWriterTextCSV((CSVFileFormatProperties)props);
}
else if( oinfo == OutputInfo.BinaryBlockOutputInfo ) {
if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_WRITE_BINARYFORMATS) )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/fdf55181/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
index 455f44d..addf798 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
@@ -34,6 +34,7 @@ import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.util.MapReduceTool;
/**
+ * Single-threaded frame text csv writer.
*
*/
public class FrameWriterTextCSV extends FrameWriter
@@ -58,22 +59,23 @@ public class FrameWriterTextCSV extends FrameWriter
* @throws DMLRuntimeException
*/
@Override
- public void writeFrameToHDFS(FrameBlock src, String fname, long rlen, long clen)
+ public final void writeFrameToHDFS(FrameBlock src, String fname, long rlen, long clen)
throws IOException, DMLRuntimeException
{
- //validity check frame dimensions
- if( src.getNumRows() != rlen || src.getNumColumns() != clen ) {
- throw new IOException("Frame dimensions mismatch with metadata: "+src.getNumRows()+"x"+src.getNumColumns()+" vs "+rlen+"x"+clen+".");
- }
-
//prepare file access
JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
Path path = new Path( fname );
//if the file already exists on HDFS, remove it.
MapReduceTool.deleteFileIfExistOnHDFS( fname );
-
- //core write
+
+ //validity check frame dimensions
+ if( src.getNumRows() != rlen || src.getNumColumns() != clen ) {
+ throw new IOException("Frame dimensions mismatch with metadata: " +
+ src.getNumRows()+"x"+src.getNumColumns()+" vs "+rlen+"x"+clen+".");
+ }
+
+ //core write (sequential/parallel)
writeCSVFrameToHDFS(path, job, src, rlen, clen, _props);
}
@@ -84,16 +86,36 @@ public class FrameWriterTextCSV extends FrameWriter
* @param src
* @param rlen
* @param clen
+ * @param csvprops
+ * @throws IOException
+ */
+ protected void writeCSVFrameToHDFS( Path path, JobConf job, FrameBlock src, long rlen, long clen, CSVFileFormatProperties csvprops )
+ throws IOException
+ {
+ FileSystem fs = FileSystem.get(job);
+
+ //sequential write to single text file
+ writeCSVFrameToFile(path, job, fs, src, 0, (int)rlen, csvprops);
+ }
+
+ /**
+ *
+ * @param path
+ * @param job
+ * @param src
+ * @param rlen
+ * @param clen
* @param props
* @return
* @throws IOException
*/
- protected void writeCSVFrameToHDFS( Path path, JobConf job, FrameBlock src, long rlen, long clen, CSVFileFormatProperties props )
+ protected final void writeCSVFrameToFile( Path path, JobConf job, FileSystem fs, FrameBlock src, int rl, int ru, CSVFileFormatProperties props )
throws IOException
{
- FileSystem fs = FileSystem.get(job);
- BufferedWriter br=new BufferedWriter(new OutputStreamWriter(fs.create(path,true)));
-
+ //create buffered writer
+ BufferedWriter br=new BufferedWriter(new OutputStreamWriter(fs.create(path,true)));
+ int cols = src.getNumColumns();
+
try
{
//for obj reuse and preventing repeated buffer re-allocations
@@ -103,13 +125,13 @@ public class FrameWriterTextCSV extends FrameWriter
String delim = props.getDelim();
// Write header line, if needed
- if( props.hasHeader() )
+ if( props.hasHeader() && rl==0 )
{
//write row chunk-wise to prevent OOM on large number of columns
- for( int bj=0; bj<clen; bj+=BLOCKSIZE_J ) {
- for( int j=bj; j < Math.min(clen,bj+BLOCKSIZE_J); j++) {
+ for( int bj=0; bj<cols; bj+=BLOCKSIZE_J ) {
+ for( int j=bj; j < Math.min(cols,bj+BLOCKSIZE_J); j++) {
sb.append("C"+ (j+1));
- if ( j < clen-1 )
+ if ( j < cols-1 )
sb.append(delim);
}
br.write( sb.toString() );
@@ -121,15 +143,15 @@ public class FrameWriterTextCSV extends FrameWriter
}
// Write data lines
- Iterator<String[]> iter = src.getStringRowIterator();
+ Iterator<String[]> iter = src.getStringRowIterator(rl, ru);
while( iter.hasNext() ) {
//write row chunk-wise to prevent OOM on large number of columns
String[] row = iter.next();
- for( int bj=0; bj<clen; bj+=BLOCKSIZE_J ) {
- for( int j=bj; j<Math.min(clen,bj+BLOCKSIZE_J); j++ ) {
+ for( int bj=0; bj<cols; bj+=BLOCKSIZE_J ) {
+ for( int j=bj; j<Math.min(cols,bj+BLOCKSIZE_J); j++ ) {
if(row[j] != null)
sb.append(row[j]);
- if( j != clen-1 )
+ if( j != cols-1 )
sb.append(delim);
}
br.write( sb.toString() );
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/fdf55181/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSVParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSVParallel.java
new file mode 100644
index 0000000..de492e2
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSVParallel.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.MapReduceTool;
+
+/**
+ * Single-threaded frame text csv writer.
+ *
+ */
+public class FrameWriterTextCSVParallel extends FrameWriterTextCSV
+{
+ public FrameWriterTextCSVParallel( CSVFileFormatProperties props ) {
+ super(props);
+ }
+
+ /**
+ *
+ * @param path
+ * @param job
+ * @param src
+ * @param rlen
+ * @param clen
+ * @param csvprops
+ * @throws IOException
+ */
+ @Override
+ protected void writeCSVFrameToHDFS( Path path, JobConf job, FrameBlock src, long rlen, long clen, CSVFileFormatProperties csvprops )
+ throws IOException
+ {
+ //estimate output size and number of output blocks (min 1)
+ int numPartFiles = Math.max((int)(OptimizerUtils.estimateSizeTextOutput(rlen, clen, rlen*clen,
+ OutputInfo.CSVOutputInfo) / InfrastructureAnalyzer.getHDFSBlockSize()), 1);
+
+ //determine degree of parallelism
+ int numThreads = OptimizerUtils.getParallelTextWriteParallelism();
+ numThreads = Math.min(numThreads, numPartFiles);
+
+ //fall back to sequential write if dop is 1 (e.g., <128MB) in order to create single file
+ if( numThreads <= 1 ) {
+ super.writeCSVFrameToHDFS(path, job, src, rlen, clen, csvprops);
+ return;
+ }
+
+ //create directory for concurrent tasks
+ MapReduceTool.createDirIfNotExistOnHDFS(path.toString(), DMLConfig.DEFAULT_SHARED_DIR_PERMISSION);
+ FileSystem fs = FileSystem.get(job);
+
+ //create and execute tasks
+ try
+ {
+ ExecutorService pool = Executors.newFixedThreadPool(numThreads);
+ ArrayList<WriteFileTask> tasks = new ArrayList<WriteFileTask>();
+ int blklen = (int)Math.ceil((double)rlen / numThreads);
+ for(int i=0; i<numThreads & i*blklen<rlen; i++) {
+ Path newPath = new Path(path, String.format("0-m-%05d",i));
+ tasks.add(new WriteFileTask(newPath, job, fs, src, i*blklen, (int)Math.min((i+1)*blklen, rlen), csvprops));
+ }
+
+ //wait until all tasks have been executed
+ List<Future<Object>> rt = pool.invokeAll(tasks);
+ pool.shutdown();
+
+ //check for exceptions
+ for( Future<Object> task : rt )
+ task.get();
+ }
+ catch (Exception e) {
+ throw new IOException("Failed parallel write of csv output.", e);
+ }
+ }
+
+ private class WriteFileTask implements Callable<Object>
+ {
+ private Path _path = null;
+ private JobConf _job = null;
+ private FileSystem _fs = null;
+ private FrameBlock _src = null;
+ private int _rl = -1;
+ private int _ru = -1;
+ private CSVFileFormatProperties _csvprops = null;
+
+ public WriteFileTask(Path path, JobConf job, FileSystem fs, FrameBlock src, int rl, int ru, CSVFileFormatProperties csvprops) {
+ _path = path;
+ _fs = fs;
+ _job = job;
+ _src = src;
+ _rl = rl;
+ _ru = ru;
+ _csvprops = csvprops;
+ }
+
+ @Override
+ public Object call() throws Exception {
+ writeCSVFrameToFile(_path, _job, _fs, _src, _rl, _ru, _csvprops);
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/fdf55181/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
index 8064ce5..38348ad 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
@@ -32,6 +32,10 @@ import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.util.MapReduceTool;
+/**
+ * Single-threaded frame text cell writer.
+ *
+ */
public class FrameWriterTextCell extends FrameWriter
{
/**
@@ -42,22 +46,23 @@ public class FrameWriterTextCell extends FrameWriter
* @throws DMLRuntimeException
*/
@Override
- public void writeFrameToHDFS( FrameBlock src, String fname, long rlen, long clen )
+ public final void writeFrameToHDFS( FrameBlock src, String fname, long rlen, long clen )
throws IOException, DMLRuntimeException
{
- //validity check frame dimensions
- if( src.getNumRows() != rlen || src.getNumColumns() != clen ) {
- throw new IOException("Frame dimensions mismatch with metadata: "+src.getNumRows()+"x"+src.getNumColumns()+" vs "+rlen+"x"+clen+".");
- }
-
//prepare file access
JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
Path path = new Path( fname );
//if the file already exists on HDFS, remove it.
MapReduceTool.deleteFileIfExistOnHDFS( fname );
-
- //core write
+
+ //validity check frame dimensions
+ if( src.getNumRows() != rlen || src.getNumColumns() != clen ) {
+ throw new IOException("Frame dimensions mismatch with metadata: " +
+ src.getNumRows()+"x"+src.getNumColumns()+" vs "+rlen+"x"+clen+".");
+ }
+
+ //core write (sequential/parallel)
writeTextCellFrameToHDFS(path, job, src, src.getNumRows(), src.getNumColumns());
}
@@ -68,38 +73,51 @@ public class FrameWriterTextCell extends FrameWriter
* @param src
* @param rlen
* @param clen
- * @param brlen
- * @param bclen
* @throws IOException
*/
protected void writeTextCellFrameToHDFS( Path path, JobConf job, FrameBlock src, long rlen, long clen )
throws IOException
{
- boolean entriesWritten = false;
FileSystem fs = FileSystem.get(job);
- BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(path,true)));
- int rows = src.getNumRows();
- int cols = src.getNumColumns();
+ //sequential write to single text file
+ writeTextCellFrameToFile(path, job, fs, src, 0, (int)rlen);
+ }
+
+ /**
+ * Internal primitive to write a row range of a frame to a single text file,
+ * which is used for both single- and multi-threaded writers (for consistency).
+ *
+ * @param path
+ * @param job
+ * @param fs
+ * @param src
+ * @param rl
+ * @param ru
+ * @throws IOException
+ */
+ protected final void writeTextCellFrameToFile( Path path, JobConf job, FileSystem fs, FrameBlock src, int rl, int ru )
+ throws IOException
+ {
+ boolean entriesWritten = false;
+ int cols = src.getNumColumns();
+
+ //create buffered writer
+ BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(path,true)));
- //bound check per block
- if( rows > rlen || cols > clen ) {
- throw new IOException("Frame block [1:"+rows+",1:"+cols+"] " +
- "out of overall frame range [1:"+rlen+",1:"+clen+"].");
- }
-
try
{
//for obj reuse and preventing repeated buffer re-allocations
StringBuilder sb = new StringBuilder();
- Iterator<String[]> iter = src.getStringRowIterator();
- for( int i=0; iter.hasNext(); i++ ) { //for all rows
+ //write frame row range to output
+ Iterator<String[]> iter = src.getStringRowIterator(rl, ru);
+ for( int i=rl; iter.hasNext(); i++ ) { //for all rows
String rowIndex = Integer.toString(i+1);
String[] row = iter.next();
for( int j=0; j<cols; j++ ) {
if( row[j] != null ) {
- sb.append(rowIndex);
+ sb.append( rowIndex );
sb.append(' ');
sb.append( j+1 );
sb.append(' ');
@@ -119,6 +137,6 @@ public class FrameWriterTextCell extends FrameWriter
}
finally {
IOUtilFunctions.closeSilently(br);
- }
- }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/fdf55181/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCellParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCellParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCellParallel.java
new file mode 100644
index 0000000..82a139d
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCellParallel.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.OutputInfo;
+import org.apache.sysml.runtime.util.MapReduceTool;
+
+/**
+ * Multi-threaded frame text cell writer.
+ *
+ */
+public class FrameWriterTextCellParallel extends FrameWriterTextCell
+{
+ /**
+ *
+ * @param path
+ * @param job
+ * @param src
+ * @param rlen
+ * @param clen
+ * @throws IOException
+ */
+ @Override
+ protected void writeTextCellFrameToHDFS( Path path, JobConf job, FrameBlock src, long rlen, long clen )
+ throws IOException
+ {
+ //estimate output size and number of output blocks (min 1)
+ int numPartFiles = Math.max((int)(OptimizerUtils.estimateSizeTextOutput(rlen, clen, rlen*clen,
+ OutputInfo.TextCellOutputInfo) / InfrastructureAnalyzer.getHDFSBlockSize()), 1);
+
+ //determine degree of parallelism
+ int numThreads = OptimizerUtils.getParallelTextWriteParallelism();
+ numThreads = Math.min(numThreads, numPartFiles);
+
+ //fall back to sequential write if dop is 1 (e.g., <128MB) in order to create single file
+ if( numThreads <= 1 ) {
+ super.writeTextCellFrameToHDFS(path, job, src, rlen, clen);
+ return;
+ }
+
+ //create directory for concurrent tasks
+ MapReduceTool.createDirIfNotExistOnHDFS(path.toString(), DMLConfig.DEFAULT_SHARED_DIR_PERMISSION);
+ FileSystem fs = FileSystem.get(job);
+
+ //create and execute tasks
+ try
+ {
+ ExecutorService pool = Executors.newFixedThreadPool(numThreads);
+ ArrayList<WriteFileTask> tasks = new ArrayList<WriteFileTask>();
+ int blklen = (int)Math.ceil((double)rlen / numThreads);
+ for(int i=0; i<numThreads & i*blklen<rlen; i++) {
+ Path newPath = new Path(path, String.format("0-m-%05d",i));
+ tasks.add(new WriteFileTask(newPath, job, fs, src, i*blklen, (int)Math.min((i+1)*blklen, rlen)));
+ }
+
+ //wait until all tasks have been executed
+ List<Future<Object>> rt = pool.invokeAll(tasks);
+ pool.shutdown();
+
+ //check for exceptions
+ for( Future<Object> task : rt )
+ task.get();
+ }
+ catch (Exception e) {
+ throw new IOException("Failed parallel write of text output.", e);
+ }
+ }
+
+ /**
+ *
+ */
+ private class WriteFileTask implements Callable<Object>
+ {
+ private Path _path = null;
+ private JobConf _job = null;
+ private FileSystem _fs = null;
+ private FrameBlock _src = null;
+ private int _rl = -1;
+ private int _ru = -1;
+
+ public WriteFileTask(Path path, JobConf job, FileSystem fs, FrameBlock src, int rl, int ru) {
+ _path = path;
+ _fs = fs;
+ _job = job;
+ _src = src;
+ _rl = rl;
+ _ru = ru;
+ }
+
+ @Override
+ public Object call() throws Exception {
+ writeTextCellFrameToFile(_path, _job, _fs, _src, _rl, _ru);
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/fdf55181/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
index 268ba44..b851704 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
@@ -348,7 +348,19 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
* @return
*/
public Iterator<String[]> getStringRowIterator() {
- return new StringRowIterator();
+ return new StringRowIterator(0, _numRows);
+ }
+
+ /**
+ * Get a row iterator over the frame where all fields are encoded
+ * as strings independent of their value types.
+ *
+ * @param rl
+ * @param ru
+ * @return
+ */
+ public Iterator<String[]> getStringRowIterator(int rl, int ru) {
+ return new StringRowIterator(rl, ru);
}
/**
@@ -358,7 +370,19 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
* @return
*/
public Iterator<Object[]> getObjectRowIterator() {
- return new ObjectRowIterator();
+ return new ObjectRowIterator(0, _numRows);
+ }
+
+ /**
+ * Get a row iterator over the frame where all fields are encoded
+ * as boxed objects according to their value types.
+ *
+ * @param rl
+ * @param ru
+ * @return
+ */
+ public Iterator<Object[]> getObjectRowIterator(int rl, int ru) {
+ return new ObjectRowIterator(rl, ru);
}
///////
@@ -744,15 +768,17 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
private abstract class RowIterator<T> implements Iterator<T[]> {
protected T[] _curRow = null;
protected int _curPos = -1;
+ protected int _maxPos = -1;
- protected RowIterator() {
- _curPos = 0;
+ protected RowIterator(int rl, int ru) {
+ _curPos = rl;
+ _maxPos = ru;
_curRow = createRow(getNumColumns());
}
@Override
public boolean hasNext() {
- return (_curPos < _numRows);
+ return (_curPos < _maxPos);
}
@Override
@@ -767,6 +793,10 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
*
*/
private class StringRowIterator extends RowIterator<String> {
+ public StringRowIterator(int rl, int ru) {
+ super(rl, ru);
+ }
+
@Override
protected String[] createRow(int size) {
return new String[size];
@@ -787,6 +817,10 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
*
*/
private class ObjectRowIterator extends RowIterator<Object> {
+ public ObjectRowIterator(int rl, int ru) {
+ super(rl, ru);
+ }
+
@Override
protected Object[] createRow(int size) {
return new Object[size];
[4/4] incubator-systemml git commit: [SYSTEMML-630] New parallel
csv/textcell frame readers, cleanup/tests
Posted by mb...@apache.org.
[SYSTEMML-630] New parallel csv/textcell frame readers, cleanup/tests
This patch introduces parallel frame readers for csv and textcell. On a
1Mx1k (8GB binary,~25-30GB in csv/text) scenario, multi-threaded read
led to the following performance improvements:
* Textcell: 930s (single-threaded) -> 83s (multi-threaded)
* CSV: 538s (single-threaded) -> 51s (multi-threaded)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/dea42de1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/dea42de1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/dea42de1
Branch: refs/heads/master
Commit: dea42de1ff7e8ada41ad50e4e941f080f82d2346
Parents: 288438b
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Wed Jun 8 00:08:27 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Wed Jun 8 01:24:02 2016 -0700
----------------------------------------------------------------------
.../io/FrameReaderBinaryBlockParallel.java | 11 +-
.../sysml/runtime/io/FrameReaderFactory.java | 10 +-
.../sysml/runtime/io/FrameReaderTextCSV.java | 217 ++++++++++-------
.../runtime/io/FrameReaderTextCSVParallel.java | 235 +++++++++++++++++++
.../sysml/runtime/io/FrameReaderTextCell.java | 132 ++++++-----
.../runtime/io/FrameReaderTextCellParallel.java | 114 +++++++++
.../sysml/runtime/io/IOUtilFunctions.java | 28 +++
.../sysml/runtime/io/ReaderTextCSVParallel.java | 19 +-
.../functions/frame/FrameReadWriteTest.java | 89 +++++--
9 files changed, 662 insertions(+), 193 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java
index d684af0..3571c37 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java
@@ -63,16 +63,14 @@ public class FrameReaderBinaryBlockParallel extends FrameReaderBinaryBlock
//create read tasks for all files
ExecutorService pool = Executors.newFixedThreadPool(numThreads);
ArrayList<ReadFileTask> tasks = new ArrayList<ReadFileTask>();
- for( Path lpath : getSequenceFilePaths(fs, path) ){
- ReadFileTask t = new ReadFileTask(lpath, job, fs, dest);
- tasks.add(t);
- }
+ for( Path lpath : getSequenceFilePaths(fs, path) )
+ tasks.add(new ReadFileTask(lpath, job, fs, dest));
//wait until all tasks have been executed
List<Future<Object>> rt = pool.invokeAll(tasks);
pool.shutdown();
- //check for exceptions and aggregate nnz
+ //check for exceptions
for( Future<Object> task : rt )
task.get();
}
@@ -99,8 +97,7 @@ public class FrameReaderBinaryBlockParallel extends FrameReaderBinaryBlock
}
@Override
- public Object call() throws Exception
- {
+ public Object call() throws Exception {
readBinaryBlockFrameFromSequenceFile(_path, _job, _fs, _dest);
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
index 51606df..b768450 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
@@ -80,12 +80,18 @@ public class FrameReaderFactory
FrameReader reader = null;
if( iinfo == InputInfo.TextCellInputInfo ) {
- reader = new FrameReaderTextCell();
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_BINARYFORMATS) )
+ reader = new FrameReaderTextCellParallel();
+ else
+ reader = new FrameReaderTextCell();
}
else if( iinfo == InputInfo.CSVInputInfo ) {
if( props!=null && !(props instanceof CSVFileFormatProperties) )
throw new DMLRuntimeException("Wrong type of file format properties for CSV writer.");
- reader = new FrameReaderTextCSV( (CSVFileFormatProperties)props);
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_BINARYFORMATS) )
+ reader = new FrameReaderTextCSVParallel( (CSVFileFormatProperties)props );
+ else
+ reader = new FrameReaderTextCSV( (CSVFileFormatProperties)props );
}
else if( iinfo == InputInfo.BinaryBlockInputInfo ) {
if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_BINARYFORMATS) )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
index 2086ea3..d282aef 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java
@@ -19,32 +19,37 @@
package org.apache.sysml.runtime.io;
-import java.io.BufferedReader;
import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
-import org.apache.hadoop.fs.FileStatus;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.matrix.CSVReblockMR;
import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.Pair;
import org.apache.sysml.runtime.util.UtilFunctions;
+/**
+ * Single-threaded frame text csv reader.
+ *
+ */
public class FrameReaderTextCSV extends FrameReader
{
-
- private CSVFileFormatProperties _props = null;
+ protected CSVFileFormatProperties _props = null;
- public FrameReaderTextCSV(CSVFileFormatProperties props)
- {
+ public FrameReaderTextCSV(CSVFileFormatProperties props) {
_props = props;
}
@@ -61,15 +66,10 @@ public class FrameReaderTextCSV extends FrameReader
* @throws IOException
*/
@Override
- public FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names,
+ public final FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names,
long rlen, long clen)
throws IOException, DMLRuntimeException
{
- //allocate output frame block
- FrameBlock ret = null;
- if( rlen>0 && clen>0 ) //otherwise CSV reblock based on file size for frame w/ unknown dimensions
- ret = createOutputFrameBlock(schema, names, rlen);
-
//prepare file access
JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
FileSystem fs = FileSystem.get(job);
@@ -77,13 +77,52 @@ public class FrameReaderTextCSV extends FrameReader
//check existence and non-empty file
checkValidInputFile(fs, path);
+
+ //compute size if necessary
+ if( rlen <= 0 || clen <= 0 ) {
+ Pair<Integer,Integer> size = computeCSVSize(path, job, fs);
+ rlen = size.getKey();
+ clen = size.getValue();
+ }
+
+ //allocate output frame block
+ List<ValueType> lschema = createOutputSchema(schema, clen);
+ List<String> lnames = createOutputNames(names, clen);
+ FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
- //core read
- ret = readCSVFrameFromHDFS(path, job, fs, ret, schema, names, rlen, clen,
- _props.hasHeader(), _props.getDelim(), _props.isFill() );
+ //core read (sequential/parallel)
+ readCSVFrameFromHDFS(path, job, fs, ret, lschema, lnames, rlen, clen);
return ret;
}
+
+ /**
+ *
+ * @param path
+ * @param job
+ * @param fs
+ * @param dest
+ * @param schema
+ * @param names
+ * @param rlen
+ * @param clen
+ * @return
+ * @throws IOException
+ */
+ protected void readCSVFrameFromHDFS( Path path, JobConf job, FileSystem fs,
+ FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen)
+ throws IOException
+ {
+ FileInputFormat.addInputPath(job, path);
+ TextInputFormat informat = new TextInputFormat();
+ informat.configure(job);
+ InputSplit[] splits = informat.getSplits(job, 1);
+ splits = IOUtilFunctions.sortInputSplits(splits);
+ for( int i=0; i<splits.length; i++ )
+ readCSVFrameFrameFromInputSplit(splits[i], informat, job, dest, schema, names, rlen, clen, 0, i==0);
+ }
+
+
/**
*
@@ -99,72 +138,60 @@ public class FrameReaderTextCSV extends FrameReader
* @return
* @throws IOException
*/
- @SuppressWarnings("unchecked")
- private FrameBlock readCSVFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest,
- List<ValueType> schema, List<String> names, long rlen, long clen, boolean hasHeader, String delim, boolean fill)
+ protected final void readCSVFrameFrameFromInputSplit( InputSplit split, TextInputFormat informat, JobConf job,
+ FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen, int rl, boolean first)
throws IOException
{
- ArrayList<Path> files=new ArrayList<Path>();
- if(fs.isDirectory(path)) {
- for(FileStatus stat: fs.listStatus(path, CSVReblockMR.hiddenFileFilter))
- files.add(stat.getPath());
- Collections.sort(files);
- }
- else
- files.add(path);
+ boolean hasHeader = _props.hasHeader();
+ boolean isFill = _props.isFill();
+ double dfillValue = _props.getFillValue();
+ String sfillValue = String.valueOf(_props.getFillValue());
+ String delim = _props.getDelim();
- if ( dest == null ) {
- dest = computeCSVSize(files, fs, schema, names, hasHeader, delim);
- clen = dest.getNumColumns();
- }
-
- /////////////////////////////////////////
- String value = null;
- int row = 0;
+ //create record reader
+ RecordReader<LongWritable, Text> reader = informat.getRecordReader(split, job, Reporter.NULL);
+ LongWritable key = new LongWritable();
+ Text value = new Text();
+ int row = rl;
int col = -1;
- for(int fileNo=0; fileNo<files.size(); fileNo++)
- {
- BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(files.get(fileNo))));
- if(fileNo==0 && hasHeader )
- br.readLine(); //ignore header
+ //handle header if existing
+ if(first && hasHeader )
+ reader.next(key, value); //ignore header
- // Read the data
- boolean emptyValuesFound = false;
- try
+ // Read the data
+ boolean emptyValuesFound = false;
+ try
+ {
+ while( reader.next(key, value) ) //foreach line
{
- while( (value=br.readLine())!=null ) //foreach line
+ String cellStr = value.toString().trim();
+ emptyValuesFound = false; col = 0;
+ String[] parts = IOUtilFunctions.split(cellStr, delim);
+
+ for( String part : parts ) //foreach cell
{
- String cellStr = value.toString().trim();
- emptyValuesFound = false;
- String[] parts = IOUtilFunctions.split(cellStr, delim);
- col = 0;
-
- for( String part : parts ) //foreach cell
- {
- part = part.trim();
- if ( part.isEmpty() ) {
- //TODO: Do we need to handle empty cell condition?
- emptyValuesFound = true;
- }
- else {
- dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), part));
- }
- col++;
+ part = part.trim();
+ if ( part.isEmpty() ) {
+ if( isFill && dfillValue!=0 )
+ dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), sfillValue));
+ emptyValuesFound = true;
}
-
- //sanity checks for empty values and number of columns
- IOUtilFunctions.checkAndRaiseErrorCSVEmptyField(cellStr, fill, emptyValuesFound);
- IOUtilFunctions.checkAndRaiseErrorCSVNumColumns(path.toString(), cellStr, parts, clen);
- row++;
+ else {
+ dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), part));
+ }
+ col++;
}
- }
- finally {
- IOUtilFunctions.closeSilently(br);
+
+ //sanity checks for empty values and number of columns
+ IOUtilFunctions.checkAndRaiseErrorCSVEmptyField(cellStr, isFill, emptyValuesFound);
+ IOUtilFunctions.checkAndRaiseErrorCSVNumColumns("", cellStr, parts, clen);
+ row++;
}
}
-
- return dest;
+ finally {
+ IOUtilFunctions.closeSilently(reader);
+ }
}
/**
@@ -178,35 +205,45 @@ public class FrameReaderTextCSV extends FrameReader
* @return
* @throws IOException
*/
- private FrameBlock computeCSVSize ( List<Path> files, FileSystem fs, List<ValueType> schema, List<String> names, boolean hasHeader, String delim)
+ protected Pair<Integer,Integer> computeCSVSize( Path path, JobConf job, FileSystem fs)
throws IOException
{
- int nrow = 0;
- for(int fileNo=0; fileNo<files.size(); fileNo++)
+ FileInputFormat.addInputPath(job, path);
+ TextInputFormat informat = new TextInputFormat();
+ informat.configure(job);
+ InputSplit[] splits = informat.getSplits(job, 1);
+ splits = IOUtilFunctions.sortInputSplits(splits);
+
+ boolean first = true;
+ int ncol = -1;
+ int nrow = -1;
+
+ for( InputSplit split : splits )
{
- BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(files.get(fileNo))));
+ RecordReader<LongWritable, Text> reader = informat.getRecordReader(split, job, Reporter.NULL);
+ LongWritable key = new LongWritable();
+ Text value = new Text();
+
try
{
- // Read the header line, if there is one.
- if(fileNo==0)
- {
- if ( hasHeader )
- br.readLine(); //ignore header
+ //read head and first line to determine num columns
+ if( first ) {
+ if ( _props.hasHeader() )
+ reader.next(key, value); //ignore header
+ reader.next(key, value);
+ ncol = StringUtils.countMatches(value.toString(), _props.getDelim()) + 1;
+ nrow = 1; first = false;
}
- while ( br.readLine() != null ) {
+ //count remaining number of rows
+ while ( reader.next(key, value) )
nrow++;
- }
}
finally {
- IOUtilFunctions.closeSilently(br);
+ IOUtilFunctions.closeSilently(reader);
}
}
- //create new frame block
- FrameBlock frameBlock = new FrameBlock(schema, names);
- frameBlock.ensureAllocatedColumns(nrow);
- return frameBlock;
+ return new Pair<Integer,Integer>(nrow, ncol);
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java
new file mode 100644
index 0000000..b998fce
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.Pair;
+
+/**
+ * Multi-threaded frame text csv reader.
+ *
+ */
+public class FrameReaderTextCSVParallel extends FrameReaderTextCSV
+{
+ public FrameReaderTextCSVParallel(CSVFileFormatProperties props) {
+ super(props);
+ }
+
+ /**
+ *
+ * @param path
+ * @param job
+ * @param fs
+ * @param dest
+ * @param schema
+ * @param names
+ * @param rlen
+ * @param clen
+ * @return
+ * @throws IOException
+ */
+ @Override
+ protected void readCSVFrameFromHDFS( Path path, JobConf job, FileSystem fs,
+ FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen)
+ throws IOException
+ {
+ int numThreads = OptimizerUtils.getParallelTextReadParallelism();
+
+ FileInputFormat.addInputPath(job, path);
+ TextInputFormat informat = new TextInputFormat();
+ informat.configure(job);
+ InputSplit[] splits = informat.getSplits(job, numThreads);
+ splits = IOUtilFunctions.sortInputSplits(splits);
+
+ try
+ {
+ ExecutorService pool = Executors.newFixedThreadPool(numThreads);
+
+ //compute num rows per split
+ ArrayList<CountRowsTask> tasks = new ArrayList<CountRowsTask>();
+ for( int i=0; i<splits.length; i++ )
+ tasks.add(new CountRowsTask(splits[i], informat, job, _props.hasHeader(), i==0));
+ List<Future<Long>> cret = pool.invokeAll(tasks);
+
+ //compute row offset per split via cumsum on row counts
+ long offset = 0;
+ List<Long> offsets = new ArrayList<Long>();
+ for( Future<Long> count : cret ) {
+ offsets.add(offset);
+ offset += count.get();
+ }
+
+ //read individial splits
+ ArrayList<ReadRowsTask> tasks2 = new ArrayList<ReadRowsTask>();
+ for( int i=0; i<splits.length; i++ )
+ tasks2.add( new ReadRowsTask(splits[i], informat, job, dest, offsets.get(i).intValue(), i==0));
+ List<Future<Object>> rret = pool.invokeAll(tasks2);
+ pool.shutdown();
+
+ //error handling
+ for( Future<Object> read : rret )
+ read.get();
+ }
+ catch (Exception e) {
+ throw new IOException("Failed parallel read of text csv input.", e);
+ }
+ }
+
+ @Override
+ protected Pair<Integer,Integer> computeCSVSize( Path path, JobConf job, FileSystem fs)
+ throws IOException
+ {
+ int numThreads = OptimizerUtils.getParallelTextReadParallelism();
+
+ FileInputFormat.addInputPath(job, path);
+ TextInputFormat informat = new TextInputFormat();
+ informat.configure(job);
+ InputSplit[] splits = informat.getSplits(job, numThreads);
+
+ //compute number of columns
+ RecordReader<LongWritable, Text> reader = informat.getRecordReader(splits[0], job, Reporter.NULL);
+ LongWritable key = new LongWritable();
+ Text value = new Text();
+ reader.next(key, value);
+ int ncol = StringUtils.countMatches(value.toString(), _props.getDelim()) + 1;
+ reader.close();
+
+ //compute number of rows
+ ExecutorService pool = Executors.newFixedThreadPool(numThreads);
+
+ //compute num rows per split
+ int nrow = 0;
+ try {
+ ArrayList<CountRowsTask> tasks = new ArrayList<CountRowsTask>();
+ for( int i=0; i<splits.length; i++ )
+ tasks.add(new CountRowsTask(splits[i], informat, job, _props.hasHeader(), i==0));
+ List<Future<Long>> cret = pool.invokeAll(tasks);
+ for( Future<Long> count : cret )
+ nrow += count.get().intValue();
+ }
+ catch (Exception e) {
+ throw new IOException("Failed parallel read of text csv input.", e);
+ }
+
+ return new Pair<Integer,Integer>(nrow, ncol);
+ }
+
+ /**
+ *
+ *
+ */
+ private static class CountRowsTask implements Callable<Long>
+ {
+ private InputSplit _split = null;
+ private TextInputFormat _informat = null;
+ private JobConf _job = null;
+ private boolean _hasHeader = false;
+ private boolean _firstSplit = false;
+
+ public CountRowsTask(InputSplit split, TextInputFormat informat, JobConf job, boolean hasHeader, boolean first) {
+ _split = split;
+ _informat = informat;
+ _job = job;
+ _hasHeader = hasHeader;
+ _firstSplit = first;
+ }
+
+ @Override
+ public Long call()
+ throws Exception
+ {
+
+ RecordReader<LongWritable, Text> reader = _informat.getRecordReader(_split, _job, Reporter.NULL);
+ LongWritable key = new LongWritable();
+ Text value = new Text();
+ long nrows = 0;
+
+ // count rows from the first non-header row
+ try {
+ if ( _firstSplit && _hasHeader )
+ reader.next(key, value);
+ while (reader.next(key, value))
+ nrows++;
+ }
+ finally {
+ IOUtilFunctions.closeSilently(reader);
+ }
+
+ return nrows;
+ }
+ }
+
+ /**
+ *
+ *
+ */
+ private class ReadRowsTask implements Callable<Object>
+ {
+ private InputSplit _split = null;
+ private TextInputFormat _informat = null;
+ private JobConf _job = null;
+ private FrameBlock _dest = null;
+ private int _offset = -1;
+ private boolean _isFirstSplit = false;
+
+
+ public ReadRowsTask(InputSplit split, TextInputFormat informat, JobConf job,
+ FrameBlock dest, int offset, boolean first)
+ {
+ _split = split;
+ _informat = informat;
+ _job = job;
+ _dest = dest;
+ _offset = offset;
+ _isFirstSplit = first;
+ }
+
+ @Override
+ public Object call()
+ throws Exception
+ {
+ readCSVFrameFrameFromInputSplit(_split, _informat, _job, _dest, _dest.getSchema(),
+ _dest.getColumnNames(), _dest.getNumRows(), _dest.getNumColumns(), _offset, _isFirstSplit);
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
index 2f42a64..8345b47 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java
@@ -42,9 +42,12 @@ import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.util.FastStringTokenizer;
import org.apache.sysml.runtime.util.UtilFunctions;
+/**
+ * Single-threaded frame textcell reader.
+ *
+ */
public class FrameReaderTextCell extends FrameReader
{
-
/**
*
* @param fname
@@ -57,12 +60,13 @@ public class FrameReaderTextCell extends FrameReader
* @throws IOException
*/
@Override
- public FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names,
- long rlen, long clen)
- throws IOException, DMLRuntimeException
+ public final FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names, long rlen, long clen)
+ throws IOException, DMLRuntimeException
{
//allocate output frame block
- FrameBlock ret = createOutputFrameBlock(schema, names, rlen);
+ List<ValueType> lschema = createOutputSchema(schema, clen);
+ List<String> lnames = createOutputNames(names, clen);
+ FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
//prepare file access
JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
@@ -72,11 +76,8 @@ public class FrameReaderTextCell extends FrameReader
//check existence and non-empty file
checkValidInputFile(fs, path);
- //core read
- if( fs.isDirectory(path) )
- readTextCellFrameFromHDFS(path, job, ret, schema, names, rlen, clen);
- else
- readRawTextCellFrameFromHDFS(path, job, fs, ret, schema, names, rlen, clen);
+ //core read (sequential/parallel)
+ readTextCellFrameFromHDFS(path, job, fs, ret, lschema, lnames, rlen, clen);
return ret;
}
@@ -90,7 +91,7 @@ public class FrameReaderTextCell extends FrameReader
* @throws IOException
* @throws DMLRuntimeException
*/
- public FrameBlock readFrameFromInputStream(InputStream is, long rlen, long clen)
+ public final FrameBlock readFrameFromInputStream(InputStream is, long rlen, long clen)
throws IOException, DMLRuntimeException {
return readFrameFromInputStream(is, getDefSchema(clen), getDefColNames(clen), rlen, clen);
}
@@ -106,83 +107,98 @@ public class FrameReaderTextCell extends FrameReader
* @throws DMLRuntimeException
* @throws IOException
*/
- public FrameBlock readFrameFromInputStream(InputStream is, List<ValueType> schema, List<String> names, long rlen, long clen)
+ public final FrameBlock readFrameFromInputStream(InputStream is, List<ValueType> schema, List<String> names, long rlen, long clen)
throws IOException, DMLRuntimeException
{
//allocate output frame block
- FrameBlock ret = createOutputFrameBlock(schema, names, rlen);
+ List<ValueType> lschema = createOutputSchema(schema, clen);
+ List<String> lnames = createOutputNames(names, clen);
+ FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
//core read
- readRawTextCellFrameFromInputStream(is, ret, schema, names, rlen, clen);
+ readRawTextCellFrameFromInputStream(is, ret, lschema, lnames, rlen, clen);
return ret;
}
-
/**
*
* @param path
* @param job
+ * @param fs
* @param dest
* @param schema
* @param names
* @param rlen
* @param clen
- * @return
* @throws IOException
*/
- private void readTextCellFrameFromHDFS( Path path, JobConf job, FrameBlock dest,
+ protected void readTextCellFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest,
List<ValueType> schema, List<String> names, long rlen, long clen)
throws IOException
{
- FileInputFormat.addInputPath(job, path);
- TextInputFormat informat = new TextInputFormat();
- informat.configure(job);
- InputSplit[] splits = informat.getSplits(job, 1);
+ if( fs.isDirectory(path) ) {
+ FileInputFormat.addInputPath(job, path);
+ TextInputFormat informat = new TextInputFormat();
+ informat.configure(job);
+ InputSplit[] splits = informat.getSplits(job, 1);
+ for(InputSplit split: splits)
+ readTextCellFrameFromInputSplit(split, informat, job, dest);
+ }
+ else {
+ readRawTextCellFrameFromHDFS(path, job, fs, dest, schema, names, rlen, clen);
+ }
+ }
+
+ /**
+ *
+ * @param split
+ * @param dest
+ * @param schema
+ * @param names
+ * @param rlen
+ * @param clen
+ * @throws IOException
+ */
+ protected final void readTextCellFrameFromInputSplit( InputSplit split, TextInputFormat informat, JobConf job, FrameBlock dest)
+ throws IOException
+ {
+ List<ValueType> schema = dest.getSchema();
+ int rlen = dest.getNumRows();
+ int clen = dest.getNumColumns();
+
+ //create record reader
+ RecordReader<LongWritable,Text> reader = informat.getRecordReader(split, job, Reporter.NULL);
LongWritable key = new LongWritable();
Text value = new Text();
+ FastStringTokenizer st = new FastStringTokenizer(' ');
int row = -1;
int col = -1;
try
{
- FastStringTokenizer st = new FastStringTokenizer(' ');
-
- for(InputSplit split: splits)
- {
- RecordReader<LongWritable,Text> reader = informat.getRecordReader(split, job, Reporter.NULL);
-
- try
- {
- while( reader.next(key, value) )
- {
- st.reset( value.toString() ); //reinit tokenizer
- row = st.nextInt()-1;
- col = st.nextInt()-1;
- dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), st.nextToken()));
- }
- }
- finally
- {
- if( reader != null )
- reader.close();
- }
+ while( reader.next(key, value) ) {
+ st.reset( value.toString() ); //reinit tokenizer
+ row = st.nextInt()-1;
+ col = st.nextInt()-1;
+ dest.set(row, col, UtilFunctions.stringToObject(schema.get(col), st.nextToken()));
}
}
- catch(Exception ex)
+ catch(Exception ex)
{
//post-mortem error handling and bounds checking
- if( row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen )
- {
+ if( row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen ) {
throw new IOException("Frame cell ["+(row+1)+","+(col+1)+"] " +
"out of overall frame range [1:"+rlen+",1:"+clen+"].");
}
- else
- {
+ else {
throw new IOException( "Unable to read frame in text cell format.", ex );
}
}
+ finally {
+ IOUtilFunctions.closeSilently(reader);
+ }
}
@@ -199,7 +215,7 @@ public class FrameReaderTextCell extends FrameReader
* @return
* @throws IOException
*/
- private void readRawTextCellFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest,
+ protected final void readRawTextCellFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest,
List<ValueType> schema, List<String> names, long rlen, long clen)
throws IOException
{
@@ -221,21 +237,20 @@ public class FrameReaderTextCell extends FrameReader
* @return
* @throws IOException
*/
- private void readRawTextCellFrameFromInputStream( InputStream is, FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen)
- throws IOException
+ protected final void readRawTextCellFrameFromInputStream( InputStream is, FrameBlock dest, List<ValueType> schema, List<String> names, long rlen, long clen)
+ throws IOException
{
+ //create buffered reader
BufferedReader br = new BufferedReader(new InputStreamReader( is ));
String value = null;
+ FastStringTokenizer st = new FastStringTokenizer(' ');
int row = -1;
int col = -1;
try
{
- FastStringTokenizer st = new FastStringTokenizer(' ');
-
- while( (value=br.readLine())!=null )
- {
+ while( (value=br.readLine())!=null ) {
st.reset( value ); //reinit tokenizer
row = st.nextInt()-1;
col = st.nextInt()-1;
@@ -245,18 +260,15 @@ public class FrameReaderTextCell extends FrameReader
catch(Exception ex)
{
//post-mortem error handling and bounds checking
- if( row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen )
- {
+ if( row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen ) {
throw new IOException("Frame cell ["+(row+1)+","+(col+1)+"] " +
"out of overall frame range [1:"+rlen+",1:"+clen+"].", ex);
}
- else
- {
+ else {
throw new IOException( "Unable to read frame in raw text cell format.", ex );
}
}
- finally
- {
+ finally {
IOUtilFunctions.closeSilently(br);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java
new file mode 100644
index 0000000..ce3993f
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+
+/**
+ * Multi-threaded frame textcell reader.
+ *
+ */
+public class FrameReaderTextCellParallel extends FrameReaderTextCell
+{
+ /**
+ *
+ * @param path
+ * @param job
+ * @param fs
+ * @param dest
+ * @param schema
+ * @param names
+ * @param rlen
+ * @param clen
+ * @throws IOException
+ */
+ @Override
+ protected void readTextCellFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest,
+ List<ValueType> schema, List<String> names, long rlen, long clen)
+ throws IOException
+ {
+ int numThreads = OptimizerUtils.getParallelTextReadParallelism();
+
+ FileInputFormat.addInputPath(job, path);
+ TextInputFormat informat = new TextInputFormat();
+ informat.configure(job);
+
+ try
+ {
+ //create read tasks for all splits
+ ExecutorService pool = Executors.newFixedThreadPool(numThreads);
+ InputSplit[] splits = informat.getSplits(job, numThreads);
+ ArrayList<ReadTask> tasks = new ArrayList<ReadTask>();
+ for( InputSplit split : splits )
+ tasks.add(new ReadTask(split, informat, job, dest));
+
+ //wait until all tasks have been executed
+ List<Future<Object>> rt = pool.invokeAll(tasks);
+ pool.shutdown();
+
+ //check for exceptions
+ for( Future<Object> task : rt )
+ task.get();
+ }
+ catch (Exception e) {
+ throw new IOException("Failed parallel read of text cell input.", e);
+ }
+ }
+
+ /**
+ *
+ */
+ public class ReadTask implements Callable<Object>
+ {
+ private InputSplit _split = null;
+ private TextInputFormat _informat = null;
+ private JobConf _job = null;
+ private FrameBlock _dest = null;
+
+ public ReadTask( InputSplit split, TextInputFormat informat, JobConf job, FrameBlock dest ) {
+ _split = split;
+ _informat = informat;
+ _job = job;
+ _dest = dest;
+ }
+
+ @Override
+ public Object call() throws Exception {
+ readTextCellFrameFromInputSplit(_split, _informat, _job, _dest);
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
index 69efec0..93a8818 100644
--- a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
+++ b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
@@ -24,10 +24,15 @@ import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
+import java.util.Arrays;
+import java.util.Comparator;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.sysml.runtime.util.LocalFileUtils;
import org.apache.sysml.runtime.util.UtilFunctions;
@@ -156,4 +161,27 @@ public class IOUtilFunctions
input.close();
return bos.toString("UTF-8");
}
+
+ /**
+ *
+ * @param splits
+ * @return
+ */
+ public static InputSplit[] sortInputSplits(InputSplit[] splits) {
+ if (splits[0] instanceof FileSplit) {
+ // The splits do not always arrive in order by file name.
+ // Sort the splits lexicographically by path so that the header will
+ // be in the first split.
+ // Note that we're assuming that the splits come in order by offset
+ Arrays.sort(splits, new Comparator<InputSplit>() {
+ @Override
+ public int compare(InputSplit o1, InputSplit o2) {
+ Path p1 = ((FileSplit) o1).getPath();
+ Path p2 = ((FileSplit) o2).getPath();
+ return p1.toString().compareTo(p2.toString());
+ }
+ });
+ }
+ return splits;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java
index 64c055c..8041ba7 100644
--- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java
@@ -21,8 +21,6 @@ package org.apache.sysml.runtime.io;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -33,7 +31,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
@@ -84,21 +81,7 @@ public class ReaderTextCSVParallel extends MatrixReader
informat.configure(job);
InputSplit[] splits = informat.getSplits(job, _numThreads);
-
- if (splits[0] instanceof FileSplit) {
- // The splits do not always arrive in order by file name.
- // Sort the splits lexicographically by path so that the header will
- // be in the first split.
- // Note that we're assuming that the splits come in order by offset
- Arrays.sort(splits, new Comparator<InputSplit>() {
- @Override
- public int compare(InputSplit o1, InputSplit o2) {
- Path p1 = ((FileSplit) o1).getPath();
- Path p2 = ((FileSplit) o2).getPath();
- return p1.toString().compareTo(p2.toString());
- }
- });
- }
+ splits = IOUtilFunctions.sortInputSplits(splits);
// check existence and non-empty file
checkValidInputFile(fs, path);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/dea42de1/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
index b07d09e..7d45ebc 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java
@@ -62,35 +62,94 @@ public class FrameReadWriteTest extends AutomatedTestBase
}
@Test
- public void testFrameStringsStrings() {
- runFrameReadWriteTest(schemaStrings, schemaStrings, false);
+ public void testFrameStringsStringsBinary() {
+ runFrameReadWriteTest(OutputInfo.BinaryBlockOutputInfo, schemaStrings, schemaStrings, false);
}
@Test
- public void testFrameStringsStringsParallel() {
- runFrameReadWriteTest(schemaStrings, schemaStrings, true);
+ public void testFrameStringsStringsBinaryParallel() {
+ runFrameReadWriteTest(OutputInfo.BinaryBlockOutputInfo, schemaStrings, schemaStrings, true);
}
@Test
- public void testFrameMixedStrings() {
- runFrameReadWriteTest(schemaMixed, schemaStrings, false);
+ public void testFrameMixedStringsBinary() {
+ runFrameReadWriteTest(OutputInfo.BinaryBlockOutputInfo, schemaMixed, schemaStrings, false);
}
@Test
- public void testFrameStringsMixedParallel() {
- runFrameReadWriteTest(schemaStrings, schemaMixed, true);
+ public void testFrameStringsMixedBinaryParallel() {
+ runFrameReadWriteTest(OutputInfo.BinaryBlockOutputInfo, schemaStrings, schemaMixed, true);
}
@Test
- public void testFrameMixedMixed() {
- runFrameReadWriteTest(schemaMixed, schemaMixed, false);
+ public void testFrameMixedMixedBinary() {
+ runFrameReadWriteTest(OutputInfo.BinaryBlockOutputInfo, schemaMixed, schemaMixed, false);
}
@Test
- public void testFrameMixedMixedParallel() {
- runFrameReadWriteTest(schemaMixed, schemaMixed, true);
+ public void testFrameMixedMixedBinaryParallel() {
+ runFrameReadWriteTest(OutputInfo.BinaryBlockOutputInfo, schemaMixed, schemaMixed, true);
}
+ @Test
+ public void testFrameStringsStringsTextCell() {
+ runFrameReadWriteTest(OutputInfo.TextCellOutputInfo, schemaStrings, schemaStrings, false);
+ }
+
+ @Test
+ public void testFrameStringsStringsTextCellParallel() {
+ runFrameReadWriteTest(OutputInfo.TextCellOutputInfo, schemaStrings, schemaStrings, true);
+ }
+
+ @Test
+ public void testFrameMixedStringsTextCell() {
+ runFrameReadWriteTest(OutputInfo.TextCellOutputInfo, schemaMixed, schemaStrings, false);
+ }
+
+ @Test
+ public void testFrameStringsMixedTextCellParallel() {
+ runFrameReadWriteTest(OutputInfo.TextCellOutputInfo, schemaStrings, schemaMixed, true);
+ }
+
+ @Test
+ public void testFrameMixedMixedTextCell() {
+ runFrameReadWriteTest(OutputInfo.TextCellOutputInfo, schemaMixed, schemaMixed, false);
+ }
+
+ @Test
+ public void testFrameMixedMixedTextCellParallel() {
+ runFrameReadWriteTest(OutputInfo.TextCellOutputInfo, schemaMixed, schemaMixed, true);
+ }
+
+ @Test
+ public void testFrameStringsStringsTextCSV() {
+ runFrameReadWriteTest(OutputInfo.CSVOutputInfo, schemaStrings, schemaStrings, false);
+ }
+
+ @Test
+ public void testFrameStringsStringsTextCSVParallel() {
+ runFrameReadWriteTest(OutputInfo.CSVOutputInfo, schemaStrings, schemaStrings, true);
+ }
+
+ @Test
+ public void testFrameMixedStringsTextCSV() {
+ runFrameReadWriteTest(OutputInfo.CSVOutputInfo, schemaMixed, schemaStrings, false);
+ }
+
+ @Test
+ public void testFrameStringsMixedTextCSVParallel() {
+ runFrameReadWriteTest(OutputInfo.CSVOutputInfo, schemaStrings, schemaMixed, true);
+ }
+
+ @Test
+ public void testFrameMixedMixedTextCSV() {
+ runFrameReadWriteTest(OutputInfo.CSVOutputInfo, schemaMixed, schemaMixed, false);
+ }
+
+ @Test
+ public void testFrameMixedMixedTextCSVParallel() {
+ runFrameReadWriteTest(OutputInfo.CSVOutputInfo, schemaMixed, schemaMixed, true);
+ }
/**
*
@@ -98,7 +157,7 @@ public class FrameReadWriteTest extends AutomatedTestBase
* @param sparseM2
* @param instType
*/
- private void runFrameReadWriteTest( ValueType[] schema1, ValueType[] schema2, boolean parallel)
+ private void runFrameReadWriteTest( OutputInfo oinfo, ValueType[] schema1, ValueType[] schema2, boolean parallel)
{
boolean oldParText = CompilerConfig.FLAG_PARREADWRITE_TEXT;
boolean oldParBin = CompilerConfig.FLAG_PARREADWRITE_BINARY;
@@ -129,9 +188,7 @@ public class FrameReadWriteTest extends AutomatedTestBase
fprop.setDelim(DELIMITER);
fprop.setHeader(HEADER);
- writeAndVerifyData(OutputInfo.TextCellOutputInfo, frame1, frame2, fprop);
- writeAndVerifyData(OutputInfo.CSVOutputInfo, frame1, frame2, fprop);
- writeAndVerifyData(OutputInfo.BinaryBlockOutputInfo, frame1, frame2, fprop);
+ writeAndVerifyData(oinfo, frame1, frame2, fprop);
}
catch(Exception ex) {
ex.printStackTrace();
[3/4] incubator-systemml git commit: [SYSTEMML-561] Performance
double frame-matrix casting (columns, cache)
Posted by mb...@apache.org.
[SYSTEMML-561] Performance double frame-matrix casting (columns, cache)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/288438b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/288438b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/288438b0
Branch: refs/heads/master
Commit: 288438b0417dbb78977caeee5cbbc421fae0c5e9
Parents: 2163bfb
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Tue Jun 7 19:03:19 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Tue Jun 7 19:03:19 2016 -0700
----------------------------------------------------------------------
.../sysml/runtime/matrix/data/FrameBlock.java | 15 ++++++++
.../sysml/runtime/util/DataConverter.java | 39 +++++++++++++++++---
2 files changed, 48 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/288438b0/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
index aef736b..53d2001 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
@@ -342,6 +342,21 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
}
/**
+ *
+ * @param c
+ * @return
+ */
+ public Object getColumn(int c) {
+ switch(_schema.get(c)) {
+ case STRING: return ((StringArray)_coldata.get(c))._data;
+ case BOOLEAN: return ((BooleanArray)_coldata.get(c))._data;
+ case INT: return ((LongArray)_coldata.get(c))._data;
+ case DOUBLE: return ((DoubleArray)_coldata.get(c))._data;
+ default: return null;
+ }
+ }
+
+ /**
* Get a row iterator over the frame where all fields are encoded
* as strings independent of their value types.
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/288438b0/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
index eddd2e3..6b46c6c 100644
--- a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
@@ -624,14 +624,41 @@ public class DataConverter
public static MatrixBlock convertToMatrixBlock(FrameBlock frame)
throws DMLRuntimeException
{
- MatrixBlock mb = new MatrixBlock(frame.getNumRows(), frame.getNumColumns(), false);
+ int m = frame.getNumRows();
+ int n = frame.getNumColumns();
+ MatrixBlock mb = new MatrixBlock(m, n, false);
+ mb.allocateDenseBlock();
List<ValueType> schema = frame.getSchema();
- for( int i=0; i<frame.getNumRows(); i++ )
- for( int j=0; j<frame.getNumColumns(); j++ ) {
- mb.appendValue(i, j, UtilFunctions.objectToDouble(
- schema.get(j), frame.get(i, j)));
- }
+ int dFreq = Collections.frequency(schema, ValueType.DOUBLE);
+
+ if( dFreq == schema.size() ) {
+ // special case double schema (without cell-object creation,
+ // cache-friendly row-column copy)
+ double[][] a = new double[n][];
+ double[] c = mb.getDenseBlock();
+ for( int j=0; j<n; j++ )
+ a[j] = (double[])frame.getColumn(j);
+ int blocksizeIJ = 16; //blocks of a+overhead/c in L1 cache
+ for( int bi=0; bi<m; bi+=blocksizeIJ )
+ for( int bj=0; bj<n; bj+=blocksizeIJ ) {
+ int bimin = Math.min(bi+blocksizeIJ, m);
+ int bjmin = Math.min(bj+blocksizeIJ, n);
+ for( int i=bi, aix=bi*n; i<bimin; i++, aix+=n )
+ for( int j=bj; j<bjmin; j++ )
+ c[aix+j] = a[j][i];
+ }
+ }
+ else {
+ //general case
+ for( int i=0; i<frame.getNumRows(); i++ )
+ for( int j=0; j<frame.getNumColumns(); j++ ) {
+ mb.appendValue(i, j, UtilFunctions.objectToDouble(
+ schema.get(j), frame.get(i, j)));
+ }
+ }
+
+ //post-processing
mb.examSparsity();
return mb;
[2/4] incubator-systemml git commit: [SYSTEMML-630] New parallel
binary block frame reader, cleanup
Posted by mb...@apache.org.
[SYSTEMML-630] New parallel binary block frame reader, cleanup
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/2163bfb4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/2163bfb4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/2163bfb4
Branch: refs/heads/master
Commit: 2163bfb44ff26915adf18dfa4def3af8dcc6822e
Parents: fdf5518
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Tue Jun 7 17:59:01 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Tue Jun 7 17:59:01 2016 -0700
----------------------------------------------------------------------
.../apache/sysml/runtime/io/FrameReader.java | 25 +++++
.../runtime/io/FrameReaderBinaryBlock.java | 100 +++++++++--------
.../io/FrameReaderBinaryBlockParallel.java | 108 +++++++++++++++++++
.../sysml/runtime/io/FrameReaderFactory.java | 55 +++-------
.../sysml/runtime/matrix/data/FrameBlock.java | 4 +-
5 files changed, 207 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/2163bfb4/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReader.java b/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
index a39d129..6a4b469 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReader.java
@@ -22,6 +22,7 @@ package org.apache.sysml.runtime.io;
import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
@@ -165,6 +166,30 @@ public abstract class FrameReader
/**
*
+ * @param schema
+ * @param ncol
+ * @return
+ */
+ protected static List<ValueType> createOutputSchema(List<ValueType> schema, long ncol) {
+ if( schema.size()==1 && ncol > 1 )
+ return Collections.nCopies((int)ncol, schema.get(0));
+ return schema;
+ }
+
+ /**
+ *
+ * @param names
+ * @param ncol
+ * @return
+ */
+ protected static List<String> createOutputNames(List<String> names, long ncol) {
+ if( names.size()==1 && ncol > 1 )
+ return FrameBlock.createColNames((int)ncol);
+ return names;
+ }
+
+ /**
+ *
* @param fs
* @param path
* @throws IOException
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/2163bfb4/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
index 534decf..f2a4a8a 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java
@@ -33,7 +33,10 @@ import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
-
+/**
+ * Single-threaded frame binary block reader.
+ *
+ */
public class FrameReaderBinaryBlock extends FrameReader
{
/**
@@ -48,12 +51,13 @@ public class FrameReaderBinaryBlock extends FrameReader
* @throws IOException
*/
@Override
- public FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names,
- long rlen, long clen)
- throws IOException, DMLRuntimeException
+ public final FrameBlock readFrameFromHDFS(String fname, List<ValueType> schema, List<String> names, long rlen, long clen)
+ throws IOException, DMLRuntimeException
{
//allocate output frame block
- FrameBlock ret = createOutputFrameBlock(schema, names, rlen);
+ List<ValueType> lschema = createOutputSchema(schema, clen);
+ List<String> lnames = createOutputNames(names, clen);
+ FrameBlock ret = createOutputFrameBlock(lschema, lnames, rlen);
//prepare file access
JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
@@ -63,21 +67,13 @@ public class FrameReaderBinaryBlock extends FrameReader
//check existence and non-empty file
checkValidInputFile(fs, path);
- //core read
+ //core read (sequential/parallel)
readBinaryBlockFrameFromHDFS(path, job, fs, ret, rlen, clen);
return ret;
}
/**
- * Note: For efficiency, we directly use SequenceFile.Reader instead of SequenceFileInputFormat-
- * InputSplits-RecordReader (SequenceFileRecordReader). First, this has no drawbacks since the
- * SequenceFileRecordReader internally uses SequenceFile.Reader as well. Second, it is
- * advantageous if the actual sequence files are larger than the file splits created by
- * informat.getSplits (which is usually aligned to the HDFS block size) because then there is
- * overhead for finding the actual split between our 1k-1k blocks. This case happens
- * if the read frame was create by CP or when jobs directly write to large output files
- * (e.g., parfor matrix partitioning).
*
* @param path
* @param job
@@ -89,46 +85,62 @@ public class FrameReaderBinaryBlock extends FrameReader
* @throws IOException
* @throws DMLRuntimeException
*/
- @SuppressWarnings("deprecation")
- private static void readBinaryBlockFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest, long rlen, long clen )
+ protected void readBinaryBlockFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest, long rlen, long clen )
+ throws IOException, DMLRuntimeException
+ {
+ //sequential read from sequence files
+ for( Path lpath : getSequenceFilePaths(fs, path) ) //1..N files
+ readBinaryBlockFrameFromSequenceFile(lpath, job, fs, dest);
+ }
+
+ /**
+ *
+ * @param path
+ * @param job
+ * @param fs
+ * @param dest
+ * @throws IOException
+ * @throws DMLRuntimeException
+ */
+ @SuppressWarnings({ "deprecation", "resource" })
+ protected final void readBinaryBlockFrameFromSequenceFile( Path path, JobConf job, FileSystem fs, FrameBlock dest )
throws IOException, DMLRuntimeException
{
+ int rlen = dest.getNumRows();
+ int clen = dest.getNumColumns();
+
+ //directly read from sequence files (individual partfiles)
+ SequenceFile.Reader reader = new SequenceFile.Reader(fs,path,job);
LongWritable key = new LongWritable(-1L);
FrameBlock value = new FrameBlock();
- for( Path lpath : getSequenceFilePaths(fs, path) ) //1..N files
+ try
{
- //directly read from sequence files (individual partfiles)
- SequenceFile.Reader reader = new SequenceFile.Reader(fs,lpath,job);
-
- try
- {
- //note: next(key, value) does not yet exploit the given serialization classes, record reader does but is generally slower.
- while( reader.next(key, value) ) {
- int row_offset = (int)(key.get()-1);
-
- int rows = value.getNumRows();
- int cols = value.getNumColumns();
+ //note: next(key, value) does not yet exploit the given serialization classes, record reader does but is generally slower.
+ while( reader.next(key, value) ) {
+ int row_offset = (int)(key.get()-1);
+
+ int rows = value.getNumRows();
+ int cols = value.getNumColumns();
- if(rows == 0 || cols == 0) //Empty block, ignore it.
- continue;
-
- //bound check per block
- if( row_offset + rows < 0 || row_offset + rows > rlen ) {
- throw new IOException("Frame block ["+(row_offset+1)+":"+(row_offset+rows)+","+":"+"] " +
- "out of overall frame range [1:"+rlen+",1:"+clen+"].");
- }
-
- dest.copy( row_offset, row_offset+rows-1,
- 0, cols-1, value);
+ if(rows == 0 || cols == 0) //Empty block, ignore it.
+ continue;
+
+ //bound check per block
+ if( row_offset + rows < 0 || row_offset + rows > rlen ) {
+ throw new IOException("Frame block ["+(row_offset+1)+":"+(row_offset+rows)+","+":"+"] " +
+ "out of overall frame range [1:"+rlen+",1:"+clen+"].");
}
- }
- finally {
- IOUtilFunctions.closeSilently(reader);
+
+ dest.copy( row_offset, row_offset+rows-1,
+ 0, cols-1, value);
}
}
- }
-
+ finally {
+ IOUtilFunctions.closeSilently(reader);
+ }
+ }
+
/**
* Specific functionality of FrameReaderBinaryBlock, mostly used for testing.
*
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/2163bfb4/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java
new file mode 100644
index 0000000..d684af0
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.runtime.io;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+
+
+/**
+ * Multi-threaded frame binary block reader.
+ *
+ */
+public class FrameReaderBinaryBlockParallel extends FrameReaderBinaryBlock
+{
+ /**
+ *
+ * @param path
+ * @param job
+ * @param fs
+ * @param dest
+ * @param rlen
+ * @param clen
+ *
+ * @throws IOException
+ * @throws DMLRuntimeException
+ */
+ protected void readBinaryBlockFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest, long rlen, long clen )
+ throws IOException, DMLRuntimeException
+ {
+ int numThreads = OptimizerUtils.getParallelBinaryReadParallelism();
+
+ try
+ {
+ //create read tasks for all files
+ ExecutorService pool = Executors.newFixedThreadPool(numThreads);
+ ArrayList<ReadFileTask> tasks = new ArrayList<ReadFileTask>();
+ for( Path lpath : getSequenceFilePaths(fs, path) ){
+ ReadFileTask t = new ReadFileTask(lpath, job, fs, dest);
+ tasks.add(t);
+ }
+
+ //wait until all tasks have been executed
+ List<Future<Object>> rt = pool.invokeAll(tasks);
+ pool.shutdown();
+
+ //check for exceptions and aggregate nnz
+ for( Future<Object> task : rt )
+ task.get();
+ }
+ catch (Exception e) {
+ throw new IOException("Failed parallel read of binary block input.", e);
+ }
+ }
+
+ /**
+ *
+ */
+ private class ReadFileTask implements Callable<Object>
+ {
+ private Path _path = null;
+ private JobConf _job = null;
+ private FileSystem _fs = null;
+ private FrameBlock _dest = null;
+
+ public ReadFileTask(Path path, JobConf job, FileSystem fs, FrameBlock dest) {
+ _path = path;
+ _fs = fs;
+ _job = job;
+ _dest = dest;
+ }
+
+ @Override
+ public Object call() throws Exception
+ {
+ readBinaryBlockFrameFromSequenceFile(_path, _job, _fs, _dest);
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/2163bfb4/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
index 35ec5f7..51606df 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderFactory.java
@@ -19,6 +19,8 @@
package org.apache.sysml.runtime.io;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
@@ -39,25 +41,10 @@ public class FrameReaderFactory
public static FrameReader createFrameReader( InputInfo iinfo )
throws DMLRuntimeException
{
- FrameReader reader = null;
+ FileFormatProperties props = (iinfo==InputInfo.CSVInputInfo) ?
+ new CSVFileFormatProperties() : null;
- if( iinfo == InputInfo.TextCellInputInfo )
- {
- reader = new FrameReaderTextCell();
- }
- else if( iinfo == InputInfo.CSVInputInfo )
- {
- reader = new FrameReaderTextCSV(new CSVFileFormatProperties());
- }
- else if( iinfo == InputInfo.BinaryBlockInputInfo ) {
- reader = new FrameReaderBinaryBlock();
- }
- else {
- throw new DMLRuntimeException("Failed to create frame reader for unknown input info: "
- + InputInfo.inputInfoToString(iinfo));
- }
-
- return reader;
+ return createFrameReader(iinfo, props);
}
/**
@@ -66,31 +53,18 @@ public class FrameReaderFactory
* @return
* @throws DMLRuntimeException
*/
- public static FrameReader createFrameReader( ReadProperties props )
+ public static FrameReader createFrameReader( ReadProperties rprops )
throws DMLRuntimeException
{
//check valid read properties
- if( props == null )
+ if( rprops == null )
throw new DMLRuntimeException("Failed to create frame reader with empty properties.");
- FrameReader reader = null;
- InputInfo iinfo = props.inputInfo;
-
- if( iinfo == InputInfo.TextCellInputInfo ) {
- reader = new FrameReaderTextCell();
- }
- else if( iinfo == InputInfo.CSVInputInfo ) {
- reader = new FrameReaderTextCSV( props.formatProperties!=null ? (CSVFileFormatProperties)props.formatProperties : new CSVFileFormatProperties());
- }
- else if( iinfo == InputInfo.BinaryBlockInputInfo ) {
- reader = new FrameReaderBinaryBlock();
- }
- else {
- throw new DMLRuntimeException("Failed to create frame reader for unknown input info: "
- + InputInfo.inputInfoToString(iinfo));
- }
-
- return reader;
+ InputInfo iinfo = rprops.inputInfo;
+ FileFormatProperties props = (iinfo==InputInfo.CSVInputInfo) ? ((rprops.formatProperties!=null) ?
+ (CSVFileFormatProperties)rprops.formatProperties : new CSVFileFormatProperties()) : null;
+
+ return createFrameReader(iinfo, props);
}
@@ -114,7 +88,10 @@ public class FrameReaderFactory
reader = new FrameReaderTextCSV( (CSVFileFormatProperties)props);
}
else if( iinfo == InputInfo.BinaryBlockInputInfo ) {
- reader = new FrameReaderBinaryBlock();
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_BINARYFORMATS) )
+ reader = new FrameReaderBinaryBlockParallel();
+ else
+ reader = new FrameReaderBinaryBlock();
}
else {
throw new DMLRuntimeException("Failed to create frame reader for unknown input info: "
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/2163bfb4/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
index b851704..aef736b 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
@@ -198,7 +198,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
* @param size
* @return
*/
- private static List<String> createColNames(int size) {
+ public static List<String> createColNames(int size) {
ArrayList<String> ret = new ArrayList<String>(size);
for( int i=1; i<=size; i++ )
ret.add(createColName(i));
@@ -210,7 +210,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
* @param i
* @return
*/
- private static String createColName(int i) {
+ public static String createColName(int i) {
return "C" + i;
}