You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/06/10 01:01:08 UTC
incubator-systemml git commit: [SYSTEMML-562] Spark frame write
instruction (binary/text/csv), tests
Repository: incubator-systemml
Updated Branches:
refs/heads/master cf4e5ab6e -> c76b01a75
[SYSTEMML-562] Spark frame write instruction (binary/text/csv), tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/c76b01a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/c76b01a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/c76b01a7
Branch: refs/heads/master
Commit: c76b01a753837150c590c79557acdccb9d756a7e
Parents: cf4e5ab
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Wed Jun 8 23:24:18 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Thu Jun 9 18:00:50 2016 -0700
----------------------------------------------------------------------
src/main/java/org/apache/sysml/hops/DataOp.java | 5 +-
.../instructions/spark/WriteSPInstruction.java | 257 ++++++++++++-------
.../functions/ConvertFrameBlockToIJVLines.java | 43 ++--
.../spark/utils/FrameRDDConverterUtils.java | 65 +++--
.../functions/frame/FrameConverterTest.java | 7 +-
.../functions/frame/FrameMatrixWriteTest.java | 193 ++++++++++++++
.../functions/frame/FrameMatrixWrite.dml | 26 ++
7 files changed, 454 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c76b01a7/src/main/java/org/apache/sysml/hops/DataOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/DataOp.java b/src/main/java/org/apache/sysml/hops/DataOp.java
index d8e5519..b4dbae4 100644
--- a/src/main/java/org/apache/sysml/hops/DataOp.java
+++ b/src/main/java/org/apache/sysml/hops/DataOp.java
@@ -37,7 +37,6 @@ import org.apache.sysml.runtime.util.LocalFileUtils;
public class DataOp extends Hop
{
-
private DataOpTypes _dataop;
private String _fileName = null;
@@ -436,8 +435,8 @@ public class DataOp extends Hop
{
checkAndSetForcedPlatform();
- //additional check for write only (TODO: remove frame here once support for distributed)
- if( getDataType()==DataType.SCALAR || getDataType()==DataType.FRAME )
+ //additional check for write only
+ if( getDataType()==DataType.SCALAR || (getDataType()==DataType.FRAME && REMOTE==ExecType.MR) )
_etypeForced = ExecType.CP;
if( _etypeForced != null )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c76b01a7/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
index 3ea03ef..682e0b7 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java
@@ -23,11 +23,12 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Random;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.spark.Accumulator;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
-
+import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
@@ -36,11 +37,15 @@ import org.apache.sysml.runtime.instructions.InstructionUtils;
import org.apache.sysml.runtime.instructions.cp.CPOperand;
import org.apache.sysml.runtime.instructions.spark.functions.ComputeBinaryBlockNnzFunction;
import org.apache.sysml.runtime.instructions.spark.functions.ConvertMatrixBlockToIJVLines;
+import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils;
+import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils.LongFrameBlockToLongWritableFrameBlock;
import org.apache.sysml.runtime.instructions.spark.utils.RDDConverterUtils;
import org.apache.sysml.runtime.instructions.spark.utils.SparkUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
@@ -54,6 +59,7 @@ public class WriteSPInstruction extends SPInstruction
private FileFormatProperties formatProperties;
//scalars might occur for transform
+ // TODO remove once transform over frames supported
private boolean isInputMatrixBlock = true;
public WriteSPInstruction(String opcode, String istr) {
@@ -87,10 +93,9 @@ public class WriteSPInstruction extends SPInstruction
//SPARK�write�_mVar2�MATRIX�DOUBLE�./src/test/scripts/functions/data/out/B�SCALAR�STRING�true�matrixmarket�SCALAR�STRING�true
// _mVar2�MATRIX�DOUBLE
- CPOperand in1=null, in2=null, in3=null;
- in1 = new CPOperand(parts[1]);
- in2 = new CPOperand(parts[2]);
- in3 = new CPOperand(parts[3]);
+ CPOperand in1 = new CPOperand(parts[1]);
+ CPOperand in2 = new CPOperand(parts[2]);
+ CPOperand in3 = new CPOperand(parts[3]);
WriteSPInstruction inst = new WriteSPInstruction(in1, in2, in3, opcode, str);
@@ -141,110 +146,172 @@ public class WriteSPInstruction extends SPInstruction
//prepare output info according to meta data
String outFmt = input3.getName();
OutputInfo oi = OutputInfo.stringToOutputInfo(outFmt);
-
- //get input rdd
- JavaPairRDD<MatrixIndexes,MatrixBlock> in1 = sec.getBinaryBlockRDDHandleForVariable( input1.getName() );
- MatrixCharacteristics mc = sec.getMatrixCharacteristics(input1.getName());
- if( oi == OutputInfo.MatrixMarketOutputInfo
- || oi == OutputInfo.TextCellOutputInfo )
- {
- //recompute nnz if necessary (required for header if matrix market)
- if ( isInputMatrixBlock && !mc.nnzKnown() )
- mc.setNonZeros( SparkUtils.computeNNZFromBlocks(in1) );
-
- JavaRDD<String> header = null;
- if(outFmt.equalsIgnoreCase("matrixmarket")) {
- ArrayList<String> headerContainer = new ArrayList<String>(1);
- // First output MM header
- String headerStr = "%%MatrixMarket matrix coordinate real general\n" +
- // output number of rows, number of columns and number of nnz
- mc.getRows() + " " + mc.getCols() + " " + mc.getNonZeros();
- headerContainer.add(headerStr);
- header = sec.getSparkContext().parallelize(headerContainer);
- }
-
- JavaRDD<String> ijv = in1.flatMap(new ConvertMatrixBlockToIJVLines(mc.getRowsPerBlock(), mc.getColsPerBlock()));
- if(header != null)
- customSaveTextFile(header.union(ijv), fname, true);
- else
- customSaveTextFile(ijv, fname, false);
- }
- else if( oi == OutputInfo.CSVOutputInfo )
- {
- JavaRDD<String> out = null;
- Accumulator<Double> aNnz = null;
-
- if ( isInputMatrixBlock ) {
- //piggyback nnz computation on actual write
- if( !mc.nnzKnown() ) {
- aNnz = sec.getSparkContext().accumulator(0L);
- in1 = in1.mapValues(new ComputeBinaryBlockNnzFunction(aNnz));
- }
-
- out = RDDConverterUtils.binaryBlockToCsv(in1, mc,
- (CSVFileFormatProperties) formatProperties, true);
- }
- else
- {
- // This case is applicable when the CSV output from transform() is written out
- @SuppressWarnings("unchecked")
- JavaPairRDD<Long,String> rdd = (JavaPairRDD<Long, String>) (sec.getMatrixObject(input1.getName())).getRDDHandle().getRDD();
- out = rdd.values();
-
- String sep = ",";
- boolean hasHeader = false;
- if(formatProperties != null) {
- sep = ((CSVFileFormatProperties) formatProperties).getDelim();
- hasHeader = ((CSVFileFormatProperties) formatProperties).hasHeader();
- }
-
- if(hasHeader) {
- StringBuffer buf = new StringBuffer();
- for(int j = 1; j < mc.getCols(); j++) {
- if(j != 1) {
- buf.append(sep);
- }
- buf.append("C" + j);
- }
- ArrayList<String> headerContainer = new ArrayList<String>(1);
- headerContainer.add(0, buf.toString());
- JavaRDD<String> header = sec.getSparkContext().parallelize(headerContainer);
- out = header.union(out);
- }
- }
-
- customSaveTextFile(out, fname, false);
-
- if( isInputMatrixBlock && !mc.nnzKnown() )
- mc.setNonZeros((long)aNnz.value().longValue());
+ //core matrix/frame write
+ if( input1.getDataType()==DataType.MATRIX )
+ processMatrixWriteInstruction(sec, fname, oi);
+ else
+ processFrameWriteInstruction(sec, fname, oi);
+ }
+ catch(IOException ex)
+ {
+ throw new DMLRuntimeException("Failed to process write instruction", ex);
+ }
+ }
+
+ /**
+ *
+ * @param sec
+ * @param fname
+ * @param oi
+ * @throws DMLRuntimeException
+ * @throws IOException
+ */
+ protected void processMatrixWriteInstruction(SparkExecutionContext sec, String fname, OutputInfo oi)
+ throws DMLRuntimeException, IOException
+ {
+ //get input rdd
+ JavaPairRDD<MatrixIndexes,MatrixBlock> in1 = sec.getBinaryBlockRDDHandleForVariable( input1.getName() );
+ MatrixCharacteristics mc = sec.getMatrixCharacteristics(input1.getName());
+
+ if( oi == OutputInfo.MatrixMarketOutputInfo
+ || oi == OutputInfo.TextCellOutputInfo )
+ {
+ //recompute nnz if necessary (required for header if matrix market)
+ if ( isInputMatrixBlock && !mc.nnzKnown() )
+ mc.setNonZeros( SparkUtils.computeNNZFromBlocks(in1) );
+
+ JavaRDD<String> header = null;
+ if( oi == OutputInfo.MatrixMarketOutputInfo ) {
+ ArrayList<String> headerContainer = new ArrayList<String>(1);
+ // First output MM header
+ String headerStr = "%%MatrixMarket matrix coordinate real general\n" +
+ // output number of rows, number of columns and number of nnz
+ mc.getRows() + " " + mc.getCols() + " " + mc.getNonZeros();
+ headerContainer.add(headerStr);
+ header = sec.getSparkContext().parallelize(headerContainer);
}
- else if( oi == OutputInfo.BinaryBlockOutputInfo ) {
+
+ JavaRDD<String> ijv = in1.flatMap(new ConvertMatrixBlockToIJVLines(mc.getRowsPerBlock(), mc.getColsPerBlock()));
+ if(header != null)
+ customSaveTextFile(header.union(ijv), fname, true);
+ else
+ customSaveTextFile(ijv, fname, false);
+ }
+ else if( oi == OutputInfo.CSVOutputInfo )
+ {
+ JavaRDD<String> out = null;
+ Accumulator<Double> aNnz = null;
+
+ if ( isInputMatrixBlock ) {
//piggyback nnz computation on actual write
- Accumulator<Double> aNnz = null;
if( !mc.nnzKnown() ) {
aNnz = sec.getSparkContext().accumulator(0L);
in1 = in1.mapValues(new ComputeBinaryBlockNnzFunction(aNnz));
- }
+ }
- //save binary block rdd on hdfs
- in1.saveAsHadoopFile(fname, MatrixIndexes.class, MatrixBlock.class, SequenceFileOutputFormat.class);
+ out = RDDConverterUtils.binaryBlockToCsv(in1, mc,
+ (CSVFileFormatProperties) formatProperties, true);
+ }
+ else
+ {
+ // This case is applicable when the CSV output from transform() is written out
+ // TODO remove once transform over frames supported
+ @SuppressWarnings("unchecked")
+ JavaPairRDD<Long,String> rdd = (JavaPairRDD<Long, String>) (sec.getMatrixObject(input1.getName())).getRDDHandle().getRDD();
+ out = rdd.values();
+
+ String sep = ",";
+ boolean hasHeader = false;
+ if(formatProperties != null) {
+ sep = ((CSVFileFormatProperties) formatProperties).getDelim();
+ hasHeader = ((CSVFileFormatProperties) formatProperties).hasHeader();
+ }
- if( !mc.nnzKnown() )
- mc.setNonZeros((long)aNnz.value().longValue());
+ if(hasHeader) {
+ StringBuffer buf = new StringBuffer();
+ for(int j = 1; j < mc.getCols(); j++) {
+ if(j != 1) {
+ buf.append(sep);
+ }
+ buf.append("C" + j);
+ }
+ ArrayList<String> headerContainer = new ArrayList<String>(1);
+ headerContainer.add(0, buf.toString());
+ JavaRDD<String> header = sec.getSparkContext().parallelize(headerContainer);
+ out = header.union(out);
+ }
}
- else {
- //unsupported formats: binarycell (not externalized)
- throw new DMLRuntimeException("Unexpected data format: " + outFmt);
+
+ customSaveTextFile(out, fname, false);
+
+ if( isInputMatrixBlock && !mc.nnzKnown() )
+ mc.setNonZeros((long)aNnz.value().longValue());
+ }
+ else if( oi == OutputInfo.BinaryBlockOutputInfo ) {
+ //piggyback nnz computation on actual write
+ Accumulator<Double> aNnz = null;
+ if( !mc.nnzKnown() ) {
+ aNnz = sec.getSparkContext().accumulator(0L);
+ in1 = in1.mapValues(new ComputeBinaryBlockNnzFunction(aNnz));
}
- // write meta data file
- MapReduceTool.writeMetaDataFile (fname + ".mtd", ValueType.DOUBLE, mc, oi, formatProperties);
+ //save binary block rdd on hdfs
+ in1.saveAsHadoopFile(fname, MatrixIndexes.class, MatrixBlock.class, SequenceFileOutputFormat.class);
+
+ if( !mc.nnzKnown() )
+ mc.setNonZeros((long)aNnz.value().longValue());
}
- catch(IOException ex)
+ else {
+ //unsupported formats: binarycell (not externalized)
+ throw new DMLRuntimeException("Unexpected data format: " + OutputInfo.outputInfoToString(oi));
+ }
+
+ // write meta data file
+ MapReduceTool.writeMetaDataFile (fname + ".mtd", ValueType.DOUBLE, mc, oi, formatProperties);
+ }
+
+ /**
+ *
+ * @param sec
+ * @param fname
+ * @param oi
+ * @throws DMLRuntimeException
+ * @throws IOException
+ */
+ @SuppressWarnings("unchecked")
+ protected void processFrameWriteInstruction(SparkExecutionContext sec, String fname, OutputInfo oi)
+ throws DMLRuntimeException, IOException
+ {
+ //get input rdd
+ JavaPairRDD<Long,FrameBlock> in1 = (JavaPairRDD<Long,FrameBlock>)sec
+ .getRDDHandleForVariable( input1.getName(), InputInfo.BinaryBlockInputInfo );
+ MatrixCharacteristics mc = sec.getMatrixCharacteristics(input1.getName());
+
+ if( oi == OutputInfo.TextCellOutputInfo )
{
- throw new DMLRuntimeException("Failed to process write instruction", ex);
+ JavaRDD<String> out = FrameRDDConverterUtils.binaryBlockToTextCell(in1, mc);
+ customSaveTextFile(out, fname, false);
+ }
+ else if( oi == OutputInfo.CSVOutputInfo )
+ {
+ CSVFileFormatProperties props = (formatProperties!=null) ?
+ (CSVFileFormatProperties) formatProperties : null;
+ JavaRDD<String> out = FrameRDDConverterUtils.binaryBlockToCsv(in1, mc, props, true);
+ customSaveTextFile(out, fname, false);
+ }
+ else if( oi == OutputInfo.BinaryBlockOutputInfo )
+ {
+ JavaPairRDD<LongWritable,FrameBlock> out = in1.mapToPair(new LongFrameBlockToLongWritableFrameBlock());
+ out.saveAsHadoopFile(fname, LongWritable.class, FrameBlock.class, SequenceFileOutputFormat.class);
+ }
+ else {
+ //unsupported formats: binarycell (not externalized)
+ throw new DMLRuntimeException("Unexpected data format: " + OutputInfo.outputInfoToString(oi));
}
+
+ // write meta data file
+ MapReduceTool.writeMetaDataFile(fname + ".mtd", input1.getValueType(), DataType.FRAME, mc, oi, formatProperties);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c76b01a7/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertFrameBlockToIJVLines.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertFrameBlockToIJVLines.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertFrameBlockToIJVLines.java
index b8f8e11..3c50668 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertFrameBlockToIJVLines.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertFrameBlockToIJVLines.java
@@ -19,37 +19,46 @@
package org.apache.sysml.runtime.instructions.spark.functions;
import java.util.ArrayList;
-import org.apache.hadoop.io.LongWritable;
+import java.util.Iterator;
+
import org.apache.spark.api.java.function.FlatMapFunction;
import scala.Tuple2;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
-public class ConvertFrameBlockToIJVLines implements FlatMapFunction<Tuple2<LongWritable,FrameBlock>, String> {
-
+public class ConvertFrameBlockToIJVLines implements FlatMapFunction<Tuple2<Long,FrameBlock>, String>
+{
private static final long serialVersionUID = 1803516615963340115L;
- int brlen; int bclen;
- public ConvertFrameBlockToIJVLines(int brlen, int bclen) {
- this.brlen = brlen;
- this.bclen = bclen;
- }
-
@Override
- public Iterable<String> call(Tuple2<LongWritable, FrameBlock> kv) throws Exception {
-
- long lRowIndex = kv._1.get();
+ public Iterable<String> call(Tuple2<Long, FrameBlock> kv)
+ throws Exception
+ {
+ long rowoffset = kv._1;
FrameBlock block = kv._2;
ArrayList<String> cells = new ArrayList<String>();
- for (int i=0; i<block.getNumRows(); ++i)
- for (int j=0; j<block.getNumColumns(); ++j) {
- Object obj = block.get(i, j);
- if(obj != null)
- cells.add(lRowIndex+i+" "+(j+1)+" "+obj.toString());
+ //convert frame block to list of ijv cell triples
+ StringBuilder sb = new StringBuilder();
+ Iterator<String[]> iter = block.getStringRowIterator();
+ for( int i=0; iter.hasNext(); i++ ) { //for all rows
+ String rowIndex = Long.toString(rowoffset + i);
+ String[] row = iter.next();
+ for( int j=0; j<row.length; j++ ) {
+ if( row[j] != null ) {
+ sb.append( rowIndex );
+ sb.append(' ');
+ sb.append( j+1 );
+ sb.append(' ');
+ sb.append( row[j] );
+ cells.add( sb.toString() );
+ sb.setLength(0);
+ }
}
+ }
+
return cells;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c76b01a7/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
index 3805164..003b016 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java
@@ -132,11 +132,9 @@ public class FrameRDDConverterUtils
* @param strict
* @return
*/
- public static JavaRDD<String> binaryBlockToCsv(JavaPairRDD<LongWritable,FrameBlock> in, MatrixCharacteristics mcIn, CSVFileFormatProperties props, boolean strict)
+ public static JavaRDD<String> binaryBlockToCsv(JavaPairRDD<Long,FrameBlock> in, MatrixCharacteristics mcIn, CSVFileFormatProperties props, boolean strict)
{
- //convert input rdd to serializable long/frame block
- JavaPairRDD<LongWritable,FrameBlock> input =
- in.mapToPair(new LongWritableToSerFunction());
+ JavaPairRDD<Long,FrameBlock> input = in;
//sort if required (on blocks/rows)
if( strict ) {
@@ -152,7 +150,8 @@ public class FrameRDDConverterUtils
//=====================================
- // cellText <--> Binary block
+ // Text cell <--> Binary block
+
/**
*
* @param sc
@@ -206,16 +205,19 @@ public class FrameRDDConverterUtils
return out;
}
-
- // Useful for printing, testing binary blocked RDD and also for external use.
- public static JavaRDD<String> binaryBlockToStringRDD(JavaPairRDD<LongWritable, FrameBlock> input, MatrixCharacteristics mcIn, String format) throws DMLRuntimeException {
- if(format.equals("text")) {
- JavaRDD<String> ijv = input.flatMap(new ConvertFrameBlockToIJVLines(mcIn.getRowsPerBlock(), mcIn.getColsPerBlock()));
- return ijv;
- }
- else {
- throw new DMLRuntimeException("The output format:" + format + " is not implemented yet.");
- }
+ /**
+ *
+ * @param input
+ * @param mcIn
+ * @param format
+ * @return
+ * @throws DMLRuntimeException
+ */
+ public static JavaRDD<String> binaryBlockToTextCell(JavaPairRDD<Long, FrameBlock> input, MatrixCharacteristics mcIn)
+ throws DMLRuntimeException
+ {
+ //convert frame blocks to ijv string triples
+ return input.flatMap(new ConvertFrameBlockToIJVLines());
}
//=====================================
@@ -339,6 +341,19 @@ public class FrameRDDConverterUtils
}
}
+ /**
+ *
+ */
+ public static class LongFrameBlockToLongWritableFrameBlock implements PairFunction<Tuple2<Long,FrameBlock>,LongWritable,FrameBlock>
+ {
+ private static final long serialVersionUID = 3201887196237766424L;
+
+ @Override
+ public Tuple2<LongWritable, FrameBlock> call(Tuple2<Long, FrameBlock> arg0) throws Exception {
+ return new Tuple2<LongWritable,FrameBlock>(new LongWritable(arg0._1), arg0._2);
+ }
+ }
+
/**
@@ -465,7 +480,7 @@ public class FrameRDDConverterUtils
/**
*
*/
- private static class BinaryBlockToCSVFunction implements FlatMapFunction<Tuple2<LongWritable,FrameBlock>,String>
+ private static class BinaryBlockToCSVFunction implements FlatMapFunction<Tuple2<Long,FrameBlock>,String>
{
private static final long serialVersionUID = 8020608184930291069L;
@@ -476,16 +491,16 @@ public class FrameRDDConverterUtils
}
@Override
- public Iterable<String> call(Tuple2<LongWritable, FrameBlock> arg0)
+ public Iterable<String> call(Tuple2<Long, FrameBlock> arg0)
throws Exception
{
- LongWritable ix = arg0._1();
+ Long ix = arg0._1();
FrameBlock blk = arg0._2();
ArrayList<String> ret = new ArrayList<String>();
//handle header information
- if(_props.hasHeader() && ix.get()==1 ) {
+ if(_props.hasHeader() && ix==1 ) {
StringBuilder sb = new StringBuilder();
for(int j = 1; j <= blk.getNumColumns(); j++) {
if(j != 1)
@@ -497,14 +512,14 @@ public class FrameRDDConverterUtils
//handle Frame block data
StringBuilder sb = new StringBuilder();
- for(int i=0; i<blk.getNumRows(); i++) {
- for(int j=0; j<blk.getNumColumns(); j++) {
+ Iterator<String[]> iter = blk.getStringRowIterator();
+ while( iter.hasNext() ) {
+ String[] row = iter.next();
+ for(int j=0; j<row.length; j++) {
if(j != 0)
sb.append(_props.getDelim());
- Object val = blk.get(i, j);
-
- if(val != null)
- sb.append(val);
+ if(row[j] != null)
+ sb.append(row[j]);
}
ret.add(sb.toString());
sb.setLength(0); //reset
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c76b01a7/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
index a0056dc..97ac27c 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java
@@ -37,6 +37,7 @@ import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysml.runtime.instructions.spark.functions.CopyFrameBlockPairFunction;
import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils;
import org.apache.sysml.runtime.io.FrameReader;
import org.apache.sysml.runtime.io.FrameReaderFactory;
@@ -449,8 +450,9 @@ public class FrameConverterTest extends AutomatedTestBase
case BIN2CSV: {
InputInfo iinfo = InputInfo.BinaryBlockInputInfo;
JavaPairRDD<LongWritable, FrameBlock> rddIn = sc.hadoopFile(fnameIn, iinfo.inputFormatClass, LongWritable.class, FrameBlock.class);
+ JavaPairRDD<Long, FrameBlock> rddIn2 = rddIn.mapToPair(new CopyFrameBlockPairFunction(false));
CSVFileFormatProperties fprop = new CSVFileFormatProperties();
- JavaRDD<String> rddOut = FrameRDDConverterUtils.binaryBlockToCsv(rddIn, mc, fprop, true);
+ JavaRDD<String> rddOut = FrameRDDConverterUtils.binaryBlockToCsv(rddIn2, mc, fprop, true);
rddOut.saveAsTextFile(fnameOut);
break;
}
@@ -466,7 +468,8 @@ public class FrameConverterTest extends AutomatedTestBase
case BIN2TXTCELL: {
InputInfo iinfo = InputInfo.BinaryBlockInputInfo;
JavaPairRDD<LongWritable, FrameBlock> rddIn = sc.hadoopFile(fnameIn, iinfo.inputFormatClass, LongWritable.class, FrameBlock.class);
- JavaRDD<String> rddOut = FrameRDDConverterUtils.binaryBlockToStringRDD(rddIn, mc, "text");
+ JavaPairRDD<Long, FrameBlock> rddIn2 = rddIn.mapToPair(new CopyFrameBlockPairFunction(false));
+ JavaRDD<String> rddOut = FrameRDDConverterUtils.binaryBlockToTextCell(rddIn2, mc);
rddOut.saveAsTextFile(fnameOut);
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c76b01a7/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameMatrixWriteTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameMatrixWriteTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameMatrixWriteTest.java
new file mode 100644
index 0000000..7e28640
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameMatrixWriteTest.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.frame;
+
+import java.io.IOException;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.lops.LopProperties.ExecType;
+import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.io.FrameReader;
+import org.apache.sysml.runtime.io.FrameReaderFactory;
+import org.apache.sysml.runtime.matrix.data.FrameBlock;
+import org.apache.sysml.runtime.matrix.data.InputInfo;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class FrameMatrixWriteTest extends AutomatedTestBase
+{
+ private final static String TEST_DIR = "functions/frame/";
+ private final static String TEST_NAME1 = "FrameMatrixWrite";
+ private final static String TEST_CLASS_DIR = TEST_DIR + FrameMatrixWriteTest.class.getSimpleName() + "/";
+
+ private final static int rows = 2593;
+ private final static int cols1 = 372;
+ private final static int cols2 = 1102;
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"B"}));
+ }
+
+ @Test
+ public void testFrameWriteSingleBinaryCP() {
+ runFrameWriteTest(TEST_NAME1, false, "binary", ExecType.CP);
+ }
+
+ @Test
+ public void testFrameWriteSingleTextcellCP() {
+ runFrameWriteTest(TEST_NAME1, false, "text", ExecType.CP);
+ }
+
+ @Test
+ public void testFrameWriteSingleCsvCP() {
+ runFrameWriteTest(TEST_NAME1, false, "csv", ExecType.CP);
+ }
+
+ @Test
+ public void testFrameWriteMultipleBinaryCP() {
+ runFrameWriteTest(TEST_NAME1, true, "binary", ExecType.CP);
+ }
+
+ @Test
+ public void testFrameWriteMultipleTextcellCP() {
+ runFrameWriteTest(TEST_NAME1, true, "text", ExecType.CP);
+ }
+
+ @Test
+ public void testFrameWriteMultipleCsvCP() {
+ runFrameWriteTest(TEST_NAME1, true, "csv", ExecType.CP);
+ }
+
+ @Test
+ public void testFrameWriteSingleBinarySpark() {
+ runFrameWriteTest(TEST_NAME1, false, "binary", ExecType.SPARK);
+ }
+
+ @Test
+ public void testFrameWriteSingleTextcellSpark() {
+ runFrameWriteTest(TEST_NAME1, false, "text", ExecType.SPARK);
+ }
+
+ @Test
+ public void testFrameWriteSingleCsvSpark() {
+ runFrameWriteTest(TEST_NAME1, false, "csv", ExecType.SPARK);
+ }
+
+ @Test
+ public void testFrameWriteMultipleBinarySpark() {
+ runFrameWriteTest(TEST_NAME1, true, "binary", ExecType.SPARK);
+ }
+
+ @Test
+ public void testFrameWriteMultipleTextcellSpark() {
+ runFrameWriteTest(TEST_NAME1, true, "text", ExecType.SPARK);
+ }
+
+ @Test
+ public void testFrameWriteMultipleCsvSpark() {
+ runFrameWriteTest(TEST_NAME1, true, "csv", ExecType.SPARK);
+ }
+
+ /**
+ *
+ * @param testname
+ * @param multColBlks
+ * @param ofmt
+ * @param et
+ */
+ private void runFrameWriteTest( String testname, boolean multColBlks, String ofmt, ExecType et)
+ {
+ //rtplatform for MR
+ RUNTIME_PLATFORM platformOld = rtplatform;
+ switch( et ){
+ case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break;
+ case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break;
+ default: rtplatform = RUNTIME_PLATFORM.HYBRID; break;
+ }
+
+ boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+ if( rtplatform == RUNTIME_PLATFORM.SPARK )
+ DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+
+ try
+ {
+ int cols = multColBlks ? cols2 : cols1;
+
+ TestConfiguration config = getTestConfiguration(testname);
+ loadTestConfiguration(config);
+
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + testname + ".dml";
+ programArgs = new String[]{"-explain","-args", String.valueOf(rows),
+ String.valueOf(cols), output("B"), ofmt };
+
+ //run testcase
+ runTest(true, false, null, -1);
+
+ //generate compare data
+ double[][] A = new double[rows][cols];
+ for( int i=0; i<rows; i++ )
+ for( int j=0; j<cols; j++ )
+ A[i][j] = (i+1)+(j+1);
+
+ //compare matrices
+ double[][] B = readFrameInput(output("B"), ofmt, rows, cols);
+ TestUtils.compareMatrices(A, B, rows, cols, 0);
+ }
+ catch(Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ finally {
+ rtplatform = platformOld;
+ DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+ }
+ }
+
+ /**
+ *
+ * @param fname
+ * @param ofmt
+ * @param rows
+ * @param cols
+ * @return
+ * @throws DMLRuntimeException
+ * @throws IOException
+ */
+ private double[][] readFrameInput(String fname, String ofmt, int rows, int cols)
+ throws DMLRuntimeException, IOException
+ {
+ //read input data
+ FrameReader reader = FrameReaderFactory.createFrameReader(InputInfo.stringExternalToInputInfo(ofmt));
+ FrameBlock fb = reader.readFrameFromHDFS(fname, rows, cols);
+ MatrixBlock ret = DataConverter.convertToMatrixBlock(fb);
+
+ return DataConverter.convertToDoubleMatrix(ret);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c76b01a7/src/test/scripts/functions/frame/FrameMatrixWrite.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/frame/FrameMatrixWrite.dml b/src/test/scripts/functions/frame/FrameMatrixWrite.dml
new file mode 100644
index 0000000..3cd8f7c
--- /dev/null
+++ b/src/test/scripts/functions/frame/FrameMatrixWrite.dml
@@ -0,0 +1,26 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+A = matrix(1, rows=$1, cols=$2);
+A = A * seq(1,nrow(A)) + t(seq(1,ncol(A)));
+
+B = as.frame(A);
+write(B, $3, format=$4);