You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/07/14 05:54:25 UTC
systemml git commit: [SYSTEMML-2434] Support for matrix market fields
real/integer/pattern
Repository: systemml
Updated Branches:
refs/heads/master 11d119877 -> 682da9cdc
[SYSTEMML-2434] Support for matrix market fields real/integer/pattern
This patch introduces support for matrix market field types beyond real
(i.e., floating point values) in CP and Spark execution modes.
Specifically, we now also support integer and pattern types, where the
latter only represents the non-zero structure (i.e., boolean values)
which is very common for sparse graph datasets.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/682da9cd
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/682da9cd
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/682da9cd
Branch: refs/heads/master
Commit: 682da9cdcaf30505e0cea6f418271661766c2022
Parents: 11d1198
Author: Matthias Boehm <mb...@gmail.com>
Authored: Fri Jul 13 22:55:30 2018 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Jul 13 22:55:30 2018 -0700
----------------------------------------------------------------------
.../spark/utils/RDDConverterUtils.java | 7 +-
.../runtime/io/FileFormatPropertiesMM.java | 8 +
.../sysml/runtime/io/IOUtilFunctions.java | 3 +-
.../apache/sysml/runtime/io/ReaderTextCell.java | 88 +++++-----
.../runtime/io/ReaderTextCellParallel.java | 79 ++++-----
.../apache/sysml/runtime/matrix/data/IJV.java | 3 +-
.../runtime/matrix/mapred/ReblockBuffer.java | 3 +-
.../sysml/runtime/util/FastStringTokenizer.java | 171 ++++++-------------
.../functions/data/MatrixMarketFormatTest.java | 64 +------
9 files changed, 143 insertions(+), 283 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java
index 94dfb55..4f9e5f9 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java
@@ -500,7 +500,6 @@ public class RDDConverterUtils
{
private static final long serialVersionUID = 4907483236186747224L;
- @SuppressWarnings("unused")
private final FileFormatPropertiesMM _mmProps;
protected TextToBinaryBlockFunction(MatrixCharacteristics mc, FileFormatPropertiesMM mmProps) {
@@ -531,11 +530,11 @@ public class RDDConverterUtils
}
//parse input ijv triple
- st.reset( strVal );
+ st.reset( strVal.toString() ); //reinit tokenizer
long row = st.nextLong();
long col = st.nextLong();
- if( row == 0 || col == 0 ) continue;
- double val = st.nextDouble();
+ double val = (_mmProps == null) ? st.nextDouble() :
+ _mmProps.isPatternField() ? 1 : _mmProps.isIntField() ? st.nextLong() : st.nextDouble();
//flush buffer if necessary
if( rbuff.getSize() >= rbuff.getCapacity() )
http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/main/java/org/apache/sysml/runtime/io/FileFormatPropertiesMM.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FileFormatPropertiesMM.java b/src/main/java/org/apache/sysml/runtime/io/FileFormatPropertiesMM.java
index 0cdbe7c..dd806d4 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FileFormatPropertiesMM.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FileFormatPropertiesMM.java
@@ -87,6 +87,14 @@ public class FileFormatPropertiesMM extends FileFormatProperties implements Seri
public MMSymmetry getSymmetry() {
return _symmetry;
}
+
+ public boolean isIntField() {
+ return _field == MMField.INTEGER;
+ }
+
+ public boolean isPatternField() {
+ return _field == MMField.PATTERN;
+ }
public static FileFormatPropertiesMM parse(String header) {
//example: %%MatrixMarket matrix coordinate real general
http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
index e158f61..94941a1 100644
--- a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
+++ b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java
@@ -131,8 +131,7 @@ public class IOUtilFunctions
}
}
- public static double parseDoubleParallel( String str )
- {
+ public static double parseDoubleParallel( String str ) {
//return FloatingDecimal.parseDouble(str);
return Double.parseDouble(str);
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java
index 321aec4..45f117f 100644
--- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.matrix.data.DenseBlock;
+import org.apache.sysml.runtime.matrix.data.IJV;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.util.FastStringTokenizer;
@@ -47,7 +48,7 @@ public class ReaderTextCell extends MatrixReader
{
protected final boolean _allowRawRead;
protected final boolean _isMMFile;
- protected FileFormatProperties _mmProps = null;
+ protected FileFormatPropertiesMM _mmProps = null;
public ReaderTextCell(InputInfo info) {
this(info, true);
@@ -122,7 +123,7 @@ public class ReaderTextCell extends MatrixReader
LongWritable key = new LongWritable();
Text value = new Text();
- int row = -1, col = -1;
+ IJV cell = new IJV();
long nnz = 0;
try
@@ -132,30 +133,22 @@ public class ReaderTextCell extends MatrixReader
for(InputSplit split: splits) {
RecordReader<LongWritable,Text> reader = informat.getRecordReader(split, job, Reporter.NULL);
try {
- if( sparse ) //SPARSE<-value
- {
+ if( sparse ) { //SPARSE<-value
while( reader.next(key, value) ) {
- st.reset( value.toString() ); //reinit tokenizer
- row = st.nextInt() - 1;
- col = st.nextInt() - 1;
- if(row == -1 || col == -1) continue;
- double lvalue = st.nextDouble();
- dest.appendValue(row, col, lvalue);
+ cell = parseCell(value.toString(), st, cell, _mmProps);
+ if( cell.getV() != 0 )
+ dest.appendValue(cell.getI(), cell.getJ(), cell.getV());
}
-
dest.sortSparseRows();
}
- else //DENSE<-value
- {
+ else { //DENSE<-value
DenseBlock a = dest.getDenseBlock();
while( reader.next(key, value) ) {
- st.reset( value.toString() ); //reinit tokenizer
- row = st.nextInt()-1;
- col = st.nextInt()-1;
- if(row == -1 || col == -1) continue;
- double lvalue = st.nextDouble();
- a.set( row, col, lvalue );
- nnz += (lvalue != 0) ? 1 : 0;
+ cell = parseCell(value.toString(), st, cell, _mmProps);
+ if( cell.getV() != 0 ) {
+ a.set( cell.getI(), cell.getJ(), cell.getV() );
+ nnz ++;
+ }
}
}
}
@@ -169,13 +162,22 @@ public class ReaderTextCell extends MatrixReader
}
catch(Exception ex) {
//post-mortem error handling and bounds checking
- if( row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen )
- throw new IOException("Matrix cell ["+(row+1)+","+(col+1)+"] " +
- "out of overall matrix range [1:"+rlen+",1:"+clen+"].");
+ if( cell.getI() < 0 || cell.getI() + 1 > rlen || cell.getJ() < 0 || cell.getJ() + 1 > clen )
+ throw new IOException("Matrix cell ["+(cell.getI()+1)+","+(cell.getJ()+1)+"] "
+ + "out of overall matrix range [1:"+rlen+",1:"+clen+"].");
else
throw new IOException( "Unable to read matrix in text cell format.", ex );
}
}
+
+ protected static IJV parseCell(String line, FastStringTokenizer st, IJV cell, FileFormatPropertiesMM mmProps) {
+ st.reset( line.toString() ); //reinit tokenizer
+ int row = st.nextInt() - 1;
+ int col = st.nextInt() - 1;
+ double value = (mmProps == null) ? st.nextDouble() :
+ mmProps.isPatternField() ? 1 : mmProps.isIntField() ? st.nextLong() : st.nextDouble();
+ return cell.set(row, col, value);
+ }
private static void readRawTextCellMatrixFromHDFS( Path path, JobConf job, FileSystem fs, MatrixBlock dest, long rlen, long clen, int brlen, int bclen, boolean matrixMarket )
throws IOException
@@ -191,12 +193,11 @@ public class ReaderTextCell extends MatrixReader
throws IOException
{
BufferedReader br = new BufferedReader(new InputStreamReader( is ));
- @SuppressWarnings("unused")
FileFormatPropertiesMM mmProps = null;
boolean sparse = dest.isInSparseFormat();
String value = null;
- int row = -1, col = -1;
+ IJV cell = new IJV();
long nnz = 0;
// Read the header lines, if reading from a matrixMarket file
@@ -226,40 +227,31 @@ public class ReaderTextCell extends MatrixReader
{
FastStringTokenizer st = new FastStringTokenizer(' ');
- if( sparse ) //SPARSE<-value
- {
- while( (value=br.readLine())!=null )
- {
- st.reset( value ); //reinit tokenizer
- row = st.nextInt()-1;
- col = st.nextInt()-1;
- if(row == -1 || col == -1) continue;
- double lvalue = st.nextDouble();
- dest.appendValue(row, col, lvalue);
+ if( sparse ) { //SPARSE<-value
+ while( (value=br.readLine())!=null ) {
+ cell = parseCell(value.toString(), st, cell, mmProps);
+ if( cell.getV() != 0 )
+ dest.appendValue(cell.getI(), cell.getJ(), cell.getV());
}
-
dest.sortSparseRows();
}
- else //DENSE<-value
- {
+ else { //DENSE<-value
DenseBlock a = dest.getDenseBlock();
while( (value=br.readLine())!=null ) {
- st.reset( value ); //reinit tokenizer
- row = st.nextInt()-1;
- col = st.nextInt()-1;
- if(row == -1 || col == -1) continue;
- double lvalue = st.nextDouble();
- a.set( row, col, lvalue );
- nnz += (lvalue != 0) ? 1 : 0;
+ cell = parseCell(value.toString(), st, cell, mmProps);
+ if( cell.getV() != 0 ) {
+ a.set( cell.getI(), cell.getJ(), cell.getV() );
+ nnz ++;
+ }
}
dest.setNonZeros(nnz);
}
}
catch(Exception ex) {
//post-mortem error handling and bounds checking
- if( row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen )
- throw new IOException("Matrix cell ["+(row+1)+","+(col+1)+"] " +
- "out of overall matrix range [1:"+rlen+",1:"+clen+"].", ex);
+ if( cell.getI() < 0 || cell.getI() + 1 > rlen || cell.getJ() < 0 || cell.getJ() + 1 > clen )
+ throw new IOException("Matrix cell ["+(cell.getI()+1)+","+(cell.getJ()+1)+"] "
+ + "out of overall matrix range [1:"+rlen+",1:"+clen+"].", ex);
else
throw new IOException( "Unable to read matrix in raw text cell format.", ex );
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java
index 3a67e56..1ce0164 100644
--- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.matrix.data.DenseBlock;
+import org.apache.sysml.runtime.matrix.data.IJV;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.util.CommonThreadPool;
@@ -95,7 +96,7 @@ public class ReaderTextCellParallel extends ReaderTextCell
InputSplit[] splits = informat.getSplits(job, par);
ArrayList<ReadTask> tasks = new ArrayList<>();
for( InputSplit split : splits ){
- ReadTask t = new ReadTask(split, informat, job, dest, rlen, clen, _isMMFile);
+ ReadTask t = new ReadTask(split, informat, job, dest, rlen, clen, _isMMFile, _mmProps);
tasks.add(t);
}
@@ -106,7 +107,7 @@ public class ReaderTextCellParallel extends ReaderTextCell
long lnnz = 0;
for( Future<Long> task : rt )
lnnz += task.get();
-
+
//post-processing
dest.setNonZeros( lnnz );
if( dest.isInSparseFormat() )
@@ -121,17 +122,17 @@ public class ReaderTextCellParallel extends ReaderTextCell
public static class ReadTask implements Callable<Long>
{
- private InputSplit _split = null;
- private boolean _sparse = false;
- private TextInputFormat _informat = null;
- private JobConf _job = null;
- private MatrixBlock _dest = null;
- private long _rlen = -1;
- private long _clen = -1;
- private boolean _matrixMarket = false;
+ private final InputSplit _split;
+ private final boolean _sparse;
+ private final TextInputFormat _informat;
+ private final JobConf _job;
+ private final MatrixBlock _dest;
+ private final long _rlen;
+ private final long _clen;
+ private final boolean _matrixMarket;
+ private final FileFormatPropertiesMM _mmProps;
- public ReadTask( InputSplit split, TextInputFormat informat, JobConf job, MatrixBlock dest, long rlen, long clen, boolean matrixMarket )
- {
+ public ReadTask( InputSplit split, TextInputFormat informat, JobConf job, MatrixBlock dest, long rlen, long clen, boolean mm, FileFormatPropertiesMM mmProps ) {
_split = split;
_sparse = dest.isInSparseFormat();
_informat = informat;
@@ -139,7 +140,8 @@ public class ReaderTextCellParallel extends ReaderTextCell
_dest = dest;
_rlen = rlen;
_clen = clen;
- _matrixMarket = matrixMarket;
+ _matrixMarket = mm;
+ _mmProps = mmProps;
}
@Override
@@ -150,10 +152,7 @@ public class ReaderTextCellParallel extends ReaderTextCell
//writables for reuse during read
LongWritable key = new LongWritable();
Text value = new Text();
-
- //required for error handling
- int row = -1;
- int col = -1;
+ IJV cell = new IJV();
FastStringTokenizer st = new FastStringTokenizer(' ');
RecordReader<LongWritable,Text> reader = _informat.getRecordReader(_split, _job, Reporter.NULL);
@@ -171,63 +170,47 @@ public class ReaderTextCellParallel extends ReaderTextCell
//process current value (otherwise ignore following meta data)
if( !foundComment ) {
- st.reset( value.toString() ); //reinit tokenizer
- row = st.nextInt()-1;
- col = st.nextInt()-1;
- if(row != -1 || col != -1) {
- double lvalue = st.nextDoubleForParallel();
+ cell = parseCell(value.toString(), st, cell, _mmProps);
+ if( cell.getV() != 0 ) {
synchronized( _dest ){ //sparse requires lock
- _dest.appendValue(row, col, lvalue);
+ _dest.appendValue(cell.getI(), cell.getJ(), cell.getV());
lnnz++;
}
}
}
}
- if( _sparse ) //SPARSE<-value
- {
+ if( _sparse ) { //SPARSE<-value
CellBuffer buff = new CellBuffer();
-
while( reader.next(key, value) ) {
- st.reset( value.toString() ); //reinit tokenizer
- row = st.nextInt() - 1;
- col = st.nextInt() - 1;
- if(row == -1 || col == -1) continue;
- double lvalue = st.nextDoubleForParallel();
-
- buff.addCell(row, col, lvalue);
- //capacity buffer flush on demand
+ buff.addCell(parseCell(value.toString(), st, cell, _mmProps));
if( buff.size()>=CellBuffer.CAPACITY )
synchronized( _dest ){ //sparse requires lock
lnnz += buff.size();
buff.flushCellBufferToMatrixBlock(_dest);
}
}
-
//final buffer flush
synchronized( _dest ){ //sparse requires lock
lnnz += buff.size();
buff.flushCellBufferToMatrixBlock(_dest);
}
}
- else //DENSE<-value
- {
+ else { //DENSE<-value
DenseBlock a = _dest.getDenseBlock();
while( reader.next(key, value) ) {
- st.reset( value.toString() ); //reinit tokenizer
- row = st.nextInt()-1;
- col = st.nextInt()-1;
- if(row == -1 || col == -1) continue;
- double lvalue = st.nextDoubleForParallel();
- a.set( row, col, lvalue );
- lnnz += (lvalue!=0) ? 1 : 0;
+ cell = parseCell(value.toString(), st, cell, _mmProps);
+ if( cell.getV() != 0 ) {
+ a.set( cell.getI(), cell.getJ(), cell.getV() );
+ lnnz ++;
+ }
}
}
}
catch(Exception ex) {
//post-mortem error handling and bounds checking
- if( row < 0 || row + 1 > _rlen || col < 0 || col + 1 > _clen )
- throw new RuntimeException("Matrix cell ["+(row+1)+","+(col+1)+"] " +
+ if( cell.getI() < 0 || cell.getI() + 1 > _rlen || cell.getJ() < 0 || cell.getJ() + 1 > _clen )
+ throw new RuntimeException("Matrix cell ["+(cell.getI()+1)+","+(cell.getJ()+1)+"] " +
"out of overall matrix range [1:"+_rlen+",1:"+_clen+"]. ", ex);
else
throw new RuntimeException("Unable to read matrix in text cell format. ", ex);
@@ -261,6 +244,10 @@ public class ReaderTextCellParallel extends ReaderTextCell
_pos = -1;
}
+ public void addCell(IJV cell) {
+ addCell(cell.getI(), cell.getJ(), cell.getV());
+ }
+
public void addCell(int rlen, int clen, double val) {
if( val==0 ) return;
_pos++;
http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java b/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java
index f4c6c7a..86c25e3 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java
@@ -33,10 +33,11 @@ public class IJV
//do nothing
}
- public void set(int i, int j, double v) {
+ public IJV set(int i, int j, double v) {
_i = i;
_j = j;
_v = v;
+ return this;
}
public int getI() {
http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java
index 8d6f2e6..7fc24ed 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java
@@ -71,8 +71,7 @@ public class ReblockBuffer
_bclen = bclen;
}
- public void appendCell( long r, long c, double v )
- {
+ public void appendCell( long r, long c, double v ) {
long tmp = Double.doubleToRawLongBits(v);
_buff[_count][0] = r;
_buff[_count][1] = c;
http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/main/java/org/apache/sysml/runtime/util/FastStringTokenizer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/FastStringTokenizer.java b/src/main/java/org/apache/sysml/runtime/util/FastStringTokenizer.java
index 2320fa1..fbc380e 100644
--- a/src/main/java/org/apache/sysml/runtime/util/FastStringTokenizer.java
+++ b/src/main/java/org/apache/sysml/runtime/util/FastStringTokenizer.java
@@ -32,125 +32,60 @@ public class FastStringTokenizer implements Serializable
{
private static final long serialVersionUID = 4051436015710778611L;
private String _string = null;
- private char _del = 0;
- private int _pos = -1;
+ private char _del = 0;
+ private int _pos = -1;
- /**
- * Constructs a new StringTokenizer for string using the specified
- * delimiter
- *
- * @param delimiter
- * the delimiter to use
- */
- public FastStringTokenizer(char delimiter)
- {
- _del = delimiter;
- reset( null );
- }
+ /**
+ * Constructs a new StringTokenizer for string using the specified
+ * delimiter
+ *
+ * @param delimiter
+ * the delimiter to use
+ */
+ public FastStringTokenizer(char delimiter) {
+ _del = delimiter;
+ reset( null );
+ }
- public void reset( String string )
- {
- _string = string;
- _pos = 0;
- }
-
- /**
- * Returns the next token in the string as a String.
- *
- * @return next token in the string as a String
- */
- public String nextToken()
- {
- int len = _string.length();
- int start = _pos;
-
- //find start (skip over leading delimiters)
- while(start < len && _del == _string.charAt(start) )
- start++;
-
- //find end (next delimiter) and return
- if(start < len) {
- _pos = _string.indexOf(_del, start);
- if( start < _pos && _pos < len )
- return _string.substring(start, _pos);
- else
- return _string.substring(start);
- }
-
- //no next token
+ public void reset( String string ) {
+ _string = string;
+ _pos = 0;
+ }
+
+ /**
+ * Returns the next token in the string as a String.
+ *
+ * @return next token in the string as a String
+ */
+ public String nextToken() {
+ int len = _string.length();
+ int start = _pos;
+
+ //find start (skip over leading delimiters)
+ while(start < len && _del == _string.charAt(start) )
+ start++;
+
+ //find end (next delimiter) and return
+ if(start < len) {
+ _pos = _string.indexOf(_del, start);
+ if( start < _pos && _pos < len )
+ return _string.substring(start, _pos);
+ else
+ return _string.substring(start);
+ }
+ //no next token
throw new NoSuchElementException();
- }
-
- ////////////////////////////////////////
- // Custom parsing methods for textcell
- ////////////////////////////////////////
-
- public int nextInt()
- {
- return Integer.parseInt( nextToken() );
- }
-
- public long nextLong()
- {
- return Long.parseLong( nextToken() );
- }
-
- public double nextDouble()
- {
- return Double.parseDouble( nextToken() );
-
- //see nextDoubleForParallel, we use the same double parsing
- //for sequential and parallel parsing because (1) it is faster (~10%)
- //and (2) for consistency between sequential and parallel readers
-
- //return FloatingDecimal.parseDouble(nextToken());
- }
-
- public double nextDoubleForParallel()
- {
- //JDK 8 floating decimal, which removes a severe scalability bottleneck
- //(synchronized static cache) in JDK7
- //return FloatingDecimal.parseDouble(nextToken());
- return Double.parseDouble( nextToken() );
-
- /*
- //return Double.parseDouble( nextToken() );
-
- //NOTE: Depending on the platform string-2-double conversions were
- //the main bottleneck in reading text data. Furthermore, we observed
- //severe contention on multi-threaded parsing on Linux JDK.
- // ---
- //This is a known issue and has been fixed in JDK8.
- //JDK-7032154 : Performance tuning of sun.misc.FloatingDecimal/FormattedFloatingDecimal
-
- // Simple workaround without JDK8 code, however, this does NOT guarantee exactly
- // the same result due to potential for round off errors.
-
- String val = nextToken();
- double ret = 0;
-
- if( UtilFunctions.isSimpleDoubleNumber(val) )
- {
- int ix = val.indexOf('.');
- if( ix > 0 ) //DOUBLE parsing
- {
- String s1 = val.substring(0, ix);
- String s2 = val.substring(ix+1);
- long tmp1 = Long.parseLong(s1);
- long tmp2 = Long.parseLong(s2);
- ret = (double)tmp2 / Math.pow(10, s2.length()) + tmp1;
- }
- else //LONG parsing and cast to double
- ret = (double)Long.parseLong(val);
- }
- else
- {
- //fall-back to slow default impl if special characters
- //e.g., ...E-0X, NAN, +-INFINITY, etc
- ret = Double.parseDouble( val );
- }
-
- return ret;
- */
- }
+ }
+
+ public int nextInt() {
+ return Integer.parseInt( nextToken() );
+ }
+
+ public long nextLong() {
+ return Long.parseLong( nextToken() );
+ }
+
+ public double nextDouble() {
+ return Double.parseDouble( nextToken() );
+ }
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/682da9cd/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java
index 3ec542e..59e3aa5 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java
@@ -84,11 +84,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase
}
@Test
- public void testMMCooRealSymmetricMR() {
- runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.REAL, MMSymmetry.SYMMETRIC, ExecType.MR);
- }
-
- @Test
public void testMMCooRealSkewSymmetricCP() {
runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.REAL, MMSymmetry.SKEW_SYMMETRIC, ExecType.CP);
}
@@ -97,11 +92,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase
public void testMMCooRealSkewSymmetricSp() {
runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.REAL, MMSymmetry.SKEW_SYMMETRIC, ExecType.SPARK);
}
-
- @Test
- public void testMMCooRealSkewSymmetricMR() {
- runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.REAL, MMSymmetry.SKEW_SYMMETRIC, ExecType.MR);
- }
@Test
public void testMMCooIntegerGeneralCP() {
@@ -129,11 +119,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase
}
@Test
- public void testMMCooIntegerSymmetricMR() {
- runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.INTEGER, MMSymmetry.SYMMETRIC, ExecType.MR);
- }
-
- @Test
public void testMMCooIntegerSkewSymmetricCP() {
runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.INTEGER, MMSymmetry.SKEW_SYMMETRIC, ExecType.CP);
}
@@ -142,11 +127,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase
public void testMMCooIntegerSkewSymmetricSp() {
runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.INTEGER, MMSymmetry.SKEW_SYMMETRIC, ExecType.SPARK);
}
-
- @Test
- public void testMMCooIntegerSkewSymmetricMR() {
- runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.INTEGER, MMSymmetry.SKEW_SYMMETRIC, ExecType.MR);
- }
@Test
public void testMMCooPatternGeneralCP() {
@@ -157,11 +137,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase
public void testMMCooPatternGeneralSp() {
runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.PATTERN, MMSymmetry.GENERAL, ExecType.SPARK);
}
-
- @Test
- public void testMMCooPatternGeneralMR() {
- runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.PATTERN, MMSymmetry.GENERAL, ExecType.MR);
- }
@Test
public void testMMCooPatternSymmetricCP() {
@@ -174,11 +149,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase
}
@Test
- public void testMMCooPatternSymmetricMR() {
- runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.PATTERN, MMSymmetry.SYMMETRIC, ExecType.MR);
- }
-
- @Test
public void testMMArrRealGeneralCP() {
runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.GENERAL, ExecType.CP);
}
@@ -187,11 +157,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase
public void testMMArrRealGeneralSp() {
runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.GENERAL, ExecType.SPARK);
}
-
- @Test
- public void testMMArrRealGeneralMR() {
- runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.GENERAL, ExecType.MR);
- }
@Test
public void testMMArrRealSymmetricCP() {
@@ -202,11 +167,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase
public void testMMArrRealSymmetricSp() {
runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.SYMMETRIC, ExecType.SPARK);
}
-
- @Test
- public void testMMArrRealSymmetricMR() {
- runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.SYMMETRIC, ExecType.MR);
- }
@Test
public void testMMArrRealSkewSymmetricCP() {
@@ -217,11 +177,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase
public void testMMArrRealSkewSymmetricSp() {
runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.SKEW_SYMMETRIC, ExecType.SPARK);
}
-
- @Test
- public void testMMArrRealSkewSymmetricMR() {
- runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.SKEW_SYMMETRIC, ExecType.MR);
- }
@Test
public void testMMArrIntegerGeneralCP() {
@@ -232,11 +187,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase
public void testMMArrIntegerGeneralSp() {
runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.GENERAL, ExecType.SPARK);
}
-
- @Test
- public void testMMArrIntegerGeneralMR() {
- runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.GENERAL, ExecType.MR);
- }
@Test
public void testMMArrIntegerSymmetricCP() {
@@ -247,11 +197,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase
public void testMMArrIntegerSymmetricSp() {
runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.SYMMETRIC, ExecType.SPARK);
}
-
- @Test
- public void testMMArrIntegerSymmetricMR() {
- runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.SYMMETRIC, ExecType.MR);
- }
@Test
public void testMMArrIntegerSkewSymmetricCP() {
@@ -263,11 +208,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase
runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.SKEW_SYMMETRIC, ExecType.SPARK);
}
- @Test
- public void testMMArrIntegerSkewSymmetricMR() {
- runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.SKEW_SYMMETRIC, ExecType.MR);
- }
-
private void runMatrixMarketFormatTest(MMFormat fmt, MMField field, MMSymmetry symmetry, ExecType et)
{
//rtplatform for MR
@@ -360,8 +300,8 @@ public class MatrixMarketFormatTest extends AutomatedTestBase
sb.append(cell.getJ()+1);
if( field != MMField.PATTERN ) {
sb.append(' ');
- sb.append(String.valueOf((field == MMField.INTEGER) ?
- (int) cell.getV() : cell.getV()));
+ sb.append((field == MMField.INTEGER) ?
+ String.valueOf((int) cell.getV()) : String.valueOf(cell.getV()));
}
sb.append('\n');
br.write( sb.toString() ); //same as append