You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/03/27 22:41:25 UTC
[4/4] incubator-systemml git commit: [SYSTEMML-573] Various code
cleanups (frame writers, utils, costs estim)
[SYSTEMML-573] Various code cleanups (frame writers, utils, costs estim)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/42e47107
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/42e47107
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/42e47107
Branch: refs/heads/master
Commit: 42e471079b296529c2e54a72961e11d28ab70dda
Parents: 5e9782f
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Mar 27 13:08:55 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sun Mar 27 13:08:55 2016 -0700
----------------------------------------------------------------------
.../apache/sysml/hops/cost/CostEstimator.java | 4 +-
.../spark/MatrixIndexingSPInstruction.java | 32 +++++------
.../spark/QuantilePickSPInstruction.java | 16 +++---
.../spark/TernarySPInstruction.java | 2 +-
.../ConvertTextLineToBinaryCellFunction.java | 8 +--
.../instructions/spark/utils/RDDSortUtils.java | 16 +++---
.../instructions/spark/utils/SparkUtils.java | 2 +-
.../sysml/runtime/io/FrameWriterTextCSV.java | 29 ++++------
.../sysml/runtime/io/FrameWriterTextCell.java | 26 ++++-----
.../apache/sysml/runtime/matrix/CombineMR.java | 8 +--
.../data/BinaryBlockToBinaryCellConverter.java | 4 +-
.../data/BinaryBlockToRowBlockConverter.java | 2 +-
.../data/BinaryBlockToTextCellConverter.java | 4 +-
.../sysml/runtime/matrix/data/LibMatrixAgg.java | 2 +-
.../sysml/runtime/matrix/data/MatrixBlock.java | 2 +-
.../matrix/data/OperationsOnMatrixValues.java | 32 +++++------
.../runtime/matrix/mapred/CSVReblockMapper.java | 4 +-
.../sysml/runtime/util/UtilFunctions.java | 58 ++++----------------
18 files changed, 104 insertions(+), 147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
index 00790a4..0e2a353 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
@@ -503,8 +503,8 @@ public abstract class CostEstimator
String ret = inst;
while( ret.contains(Lop.VARIABLE_NAME_PLACEHOLDER) ) {
int index1 = ret.indexOf(Lop.VARIABLE_NAME_PLACEHOLDER);
- int index2 = ret.indexOf(Lop.VARIABLE_NAME_PLACEHOLDER, index1+3);
- String replace = ret.substring(index1,index2+2);
+ int index2 = ret.indexOf(Lop.VARIABLE_NAME_PLACEHOLDER, index1+1);
+ String replace = ret.substring(index1,index2+1);
ret = ret.replaceAll(replace, "1");
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
index 89992b8..3bf4d98 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
@@ -294,10 +294,10 @@ public class MatrixIndexingSPInstruction extends UnarySPInstruction
long end_lhs_globalRowIndex = start_lhs_globalRowIndex + rightKV._2.getNumRows() - 1;
long end_lhs_globalColIndex = start_lhs_globalColIndex + rightKV._2.getNumColumns() - 1;
- long start_lhs_rowIndex = UtilFunctions.blockIndexCalculation(start_lhs_globalRowIndex, brlen);
- long end_lhs_rowIndex = UtilFunctions.blockIndexCalculation(end_lhs_globalRowIndex, brlen);
- long start_lhs_colIndex = UtilFunctions.blockIndexCalculation(start_lhs_globalColIndex, bclen);
- long end_lhs_colIndex = UtilFunctions.blockIndexCalculation(end_lhs_globalColIndex, bclen);
+ long start_lhs_rowIndex = UtilFunctions.computeBlockIndex(start_lhs_globalRowIndex, brlen);
+ long end_lhs_rowIndex = UtilFunctions.computeBlockIndex(end_lhs_globalRowIndex, brlen);
+ long start_lhs_colIndex = UtilFunctions.computeBlockIndex(start_lhs_globalColIndex, bclen);
+ long end_lhs_colIndex = UtilFunctions.computeBlockIndex(end_lhs_globalColIndex, bclen);
for(long leftRowIndex = start_lhs_rowIndex; leftRowIndex <= end_lhs_rowIndex; leftRowIndex++) {
for(long leftColIndex = start_lhs_colIndex; leftColIndex <= end_lhs_colIndex; leftColIndex++) {
@@ -308,20 +308,20 @@ public class MatrixIndexingSPInstruction extends UnarySPInstruction
long lhs_cl = Math.max((leftColIndex-1)*bclen+1, start_lhs_globalColIndex);
long lhs_cu = Math.min(leftColIndex*bclen, end_lhs_globalColIndex);
- int lhs_lrl = UtilFunctions.cellInBlockCalculation(lhs_rl, brlen);
- int lhs_lru = UtilFunctions.cellInBlockCalculation(lhs_ru, brlen);
- int lhs_lcl = UtilFunctions.cellInBlockCalculation(lhs_cl, bclen);
- int lhs_lcu = UtilFunctions.cellInBlockCalculation(lhs_cu, bclen);
+ int lhs_lrl = UtilFunctions.computeCellInBlock(lhs_rl, brlen);
+ int lhs_lru = UtilFunctions.computeCellInBlock(lhs_ru, brlen);
+ int lhs_lcl = UtilFunctions.computeCellInBlock(lhs_cl, bclen);
+ int lhs_lcu = UtilFunctions.computeCellInBlock(lhs_cu, bclen);
long rhs_rl = lhs_rl - rl + 1;
long rhs_ru = rhs_rl + (lhs_ru - lhs_rl);
long rhs_cl = lhs_cl - cl + 1;
long rhs_cu = rhs_cl + (lhs_cu - lhs_cl);
- int rhs_lrl = UtilFunctions.cellInBlockCalculation(rhs_rl, brlen);
- int rhs_lru = UtilFunctions.cellInBlockCalculation(rhs_ru, brlen);
- int rhs_lcl = UtilFunctions.cellInBlockCalculation(rhs_cl, bclen);
- int rhs_lcu = UtilFunctions.cellInBlockCalculation(rhs_cu, bclen);
+ int rhs_lrl = UtilFunctions.computeCellInBlock(rhs_rl, brlen);
+ int rhs_lru = UtilFunctions.computeCellInBlock(rhs_ru, brlen);
+ int rhs_lcl = UtilFunctions.computeCellInBlock(rhs_cl, bclen);
+ int rhs_lcu = UtilFunctions.computeCellInBlock(rhs_cu, bclen);
MatrixBlock slicedRHSBlk = rightKV._2.sliceOperations(rhs_lrl, rhs_lru, rhs_lcl, rhs_lcu, new MatrixBlock());
@@ -440,10 +440,10 @@ public class MatrixIndexingSPInstruction extends UnarySPInstruction
MatrixBlock slicedRHSMatBlock = _binput.sliceOperations(rhs_rl, rhs_ru, rhs_cl, rhs_cu, new MatrixBlock());
// Provide local zero-based index to leftIndexingOperations
- int lhs_lrl = UtilFunctions.cellInBlockCalculation(lhs_rl, _brlen);
- int lhs_lru = UtilFunctions.cellInBlockCalculation(lhs_ru, _brlen);
- int lhs_lcl = UtilFunctions.cellInBlockCalculation(lhs_cl, _bclen);
- int lhs_lcu = UtilFunctions.cellInBlockCalculation(lhs_cu, _bclen);
+ int lhs_lrl = UtilFunctions.computeCellInBlock(lhs_rl, _brlen);
+ int lhs_lru = UtilFunctions.computeCellInBlock(lhs_ru, _brlen);
+ int lhs_lcl = UtilFunctions.computeCellInBlock(lhs_cl, _bclen);
+ int lhs_lcu = UtilFunctions.computeCellInBlock(lhs_cu, _bclen);
MatrixBlock ret = arg._2.leftIndexingOperations(slicedRHSMatBlock, lhs_lrl, lhs_lru, lhs_lcl, lhs_lcu, new MatrixBlock(), false);
return new Tuple2<MatrixIndexes, MatrixBlock>(arg._1, ret);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java
index 5cea24f..55b9584 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java
@@ -177,8 +177,8 @@ public class QuantilePickSPInstruction extends BinarySPInstruction
*/
private double lookupKey(JavaPairRDD<MatrixIndexes,MatrixBlock> in, long key, int brlen)
{
- long rix = UtilFunctions.blockIndexCalculation(key, brlen);
- long pos = UtilFunctions.cellInBlockCalculation(key, brlen);
+ long rix = UtilFunctions.computeBlockIndex(key, brlen);
+ long pos = UtilFunctions.computeCellInBlock(key, brlen);
List<MatrixBlock> val = in.lookup(new MatrixIndexes(rix,1));
return val.get(0).quickGetValue((int)pos, 0);
@@ -211,8 +211,8 @@ public class QuantilePickSPInstruction extends BinarySPInstruction
public FilterFunction(long key25, long key75, int brlen)
{
- _minRowIndex = UtilFunctions.blockIndexCalculation(key25, brlen);
- _maxRowIndex = UtilFunctions.blockIndexCalculation(key75, brlen);
+ _minRowIndex = UtilFunctions.computeBlockIndex(key25, brlen);
+ _maxRowIndex = UtilFunctions.computeBlockIndex(key75, brlen);
}
@Override
@@ -239,10 +239,10 @@ public class QuantilePickSPInstruction extends BinarySPInstruction
public ExtractAndSumFunction(long key25, long key75, int brlen)
{
- _minRowIndex = UtilFunctions.blockIndexCalculation(key25, brlen);
- _maxRowIndex = UtilFunctions.blockIndexCalculation(key75, brlen);
- _minPos = UtilFunctions.cellInBlockCalculation(key25, brlen);
- _maxPos = UtilFunctions.cellInBlockCalculation(key75, brlen);
+ _minRowIndex = UtilFunctions.computeBlockIndex(key25, brlen);
+ _maxRowIndex = UtilFunctions.computeBlockIndex(key75, brlen);
+ _minPos = UtilFunctions.computeCellInBlock(key25, brlen);
+ _maxPos = UtilFunctions.computeCellInBlock(key75, brlen);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java
index ec2d4b0..fa00eb0 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java
@@ -280,7 +280,7 @@ public class TernarySPInstruction extends ComputationSPInstruction
for( int i=0; i<mb.getNumRows(); i++ )
{
//compute global target indexes (via ctable obj for error handling consistency)
- long row = UtilFunctions.cellIndexCalculation(ix.getRowIndex(), _brlen, i);
+ long row = UtilFunctions.computeCellIndex(ix.getRowIndex(), _brlen, i);
double v2 = mb.quickGetValue(i, 0);
Pair<MatrixIndexes,Double> p = ctab.execute(row, v2, 1.0);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java
index cde5d4e..b135497 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java
@@ -69,10 +69,10 @@ public class ConvertTextLineToBinaryCellFunction implements PairFunction<String,
// Get appropriate indexes for blockIndexes and cell
// For line: 1020 704 2.362153706180234 (assuming default block size: 1000 X 1000),
// blockRowIndex = 2, blockColIndex = 1, rowIndexInBlock = 19, colIndexInBlock = 703 ... TODO: double check this !!!
- long blockRowIndex = UtilFunctions.blockIndexCalculation(retVal.getKey().getRowIndex(), (int) brlen);
- long blockColIndex = UtilFunctions.blockIndexCalculation(retVal.getKey().getColumnIndex(), (int) bclen);
- long rowIndexInBlock = UtilFunctions.cellInBlockCalculation(retVal.getKey().getRowIndex(), brlen);
- long colIndexInBlock = UtilFunctions.cellInBlockCalculation(retVal.getKey().getColumnIndex(), bclen);
+ long blockRowIndex = UtilFunctions.computeBlockIndex(retVal.getKey().getRowIndex(), (int) brlen);
+ long blockColIndex = UtilFunctions.computeBlockIndex(retVal.getKey().getColumnIndex(), (int) bclen);
+ long rowIndexInBlock = UtilFunctions.computeCellInBlock(retVal.getKey().getRowIndex(), brlen);
+ long colIndexInBlock = UtilFunctions.computeCellInBlock(retVal.getKey().getColumnIndex(), bclen);
// Perform sanity check
if(blockRowIndex <= 0 || blockColIndex <= 0 || rowIndexInBlock < 0 || colIndexInBlock < 0) {
throw new Exception("Error computing indexes for the line:" + line);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java
index e3c3779..34c1358 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java
@@ -432,8 +432,8 @@ public class RDDSortUtils
{
Tuple2<Double,Long> val = arg0.next();
long valix = val._2 + 1;
- long rix = UtilFunctions.blockIndexCalculation(valix, _brlen);
- int pos = UtilFunctions.cellInBlockCalculation(valix, _brlen);
+ long rix = UtilFunctions.computeBlockIndex(valix, _brlen);
+ int pos = UtilFunctions.computeCellInBlock(valix, _brlen);
if( ix == null || ix.getRowIndex() != rix )
{
@@ -483,8 +483,8 @@ public class RDDSortUtils
{
Tuple2<DoublePair,Long> val = arg0.next();
long valix = val._2 + 1;
- long rix = UtilFunctions.blockIndexCalculation(valix, _brlen);
- int pos = UtilFunctions.cellInBlockCalculation(valix, _brlen);
+ long rix = UtilFunctions.computeBlockIndex(valix, _brlen);
+ int pos = UtilFunctions.computeCellInBlock(valix, _brlen);
if( ix == null || ix.getRowIndex() != rix )
{
@@ -535,8 +535,8 @@ public class RDDSortUtils
{
Tuple2<ValueIndexPair,Long> val = arg0.next();
long valix = val._2 + 1;
- long rix = UtilFunctions.blockIndexCalculation(valix, _brlen);
- int pos = UtilFunctions.cellInBlockCalculation(valix, _brlen);
+ long rix = UtilFunctions.computeBlockIndex(valix, _brlen);
+ int pos = UtilFunctions.computeCellInBlock(valix, _brlen);
if( ix == null || ix.getRowIndex() != rix )
{
@@ -586,8 +586,8 @@ public class RDDSortUtils
{
Tuple2<Long,Long> val = arg0.next();
long valix = val._1;
- long rix = UtilFunctions.blockIndexCalculation(valix, _brlen);
- int pos = UtilFunctions.cellInBlockCalculation(valix, _brlen);
+ long rix = UtilFunctions.computeBlockIndex(valix, _brlen);
+ int pos = UtilFunctions.computeCellInBlock(valix, _brlen);
if( ix == null || ix.getRowIndex() != rix )
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
index f3232f7..6b9037a 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
@@ -167,7 +167,7 @@ public class SparkUtils
// len = {clen or rlen}, blen = {brlen or bclen}
public static long getStartGlobalIndex(long blockIndex, int blen, long len) {
- return UtilFunctions.cellIndexCalculation(blockIndex, blen, 0);
+ return UtilFunctions.computeCellIndex(blockIndex, blen, 0);
}
public static JavaPairRDD<MatrixIndexes, MatrixBlock> getRDDWithEmptyBlocks(JavaSparkContext sc,
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
index 650778d..10d957f 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
@@ -22,6 +22,7 @@ package org.apache.sysml.runtime.io;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
+import java.util.Iterator;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -107,10 +108,8 @@ public class FrameWriterTextCSV extends FrameWriter
if( props.hasHeader() )
{
//write row chunk-wise to prevent OOM on large number of columns
- for( int bj=0; bj<clen; bj+=BLOCKSIZE_J )
- {
- for( int j=bj; j < Math.min(clen,bj+BLOCKSIZE_J); j++)
- {
+ for( int bj=0; bj<clen; bj+=BLOCKSIZE_J ) {
+ for( int j=bj; j < Math.min(clen,bj+BLOCKSIZE_J); j++) {
sb.append("C"+ (j+1));
if ( j < clen-1 )
sb.append(delim);
@@ -124,16 +123,14 @@ public class FrameWriterTextCSV extends FrameWriter
}
// Write data lines
- for( int i=0; i<rlen; i++ )
- {
+ Iterator<String[]> iter = src.getStringRowIterator();
+ while( iter.hasNext() ) {
//write row chunk-wise to prevent OOM on large number of columns
- for( int bj=0; bj<clen; bj+=BLOCKSIZE_J )
- {
- for( int j=bj; j<Math.min(clen,bj+BLOCKSIZE_J); j++ )
- {
- if(src.get(i, j) != null)
- sb.append(src.get(i, j).toString());
-
+ String[] row = iter.next();
+ for( int bj=0; bj<clen; bj+=BLOCKSIZE_J ) {
+ for( int j=bj; j<Math.min(clen,bj+BLOCKSIZE_J); j++ ) {
+ if(row[j] != null)
+ sb.append(row[j]);
if( j != clen-1 )
sb.append(delim);
}
@@ -142,14 +139,12 @@ public class FrameWriterTextCSV extends FrameWriter
}
sb.append('\n');
- br.write( sb.toString() ); //same as append
+ br.write( sb.toString() );
sb.setLength(0);
}
}
- finally
- {
+ finally {
IOUtilFunctions.closeSilently(br);
}
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
index 2af9a60..c595448 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
@@ -22,6 +22,7 @@ package org.apache.sysml.runtime.io;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
+import java.util.Iterator;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -84,8 +85,7 @@ public class FrameWriterTextCell extends FrameWriter
int cols = src.getNumColumns();
//bound check per block
- if( rows > rlen || cols > clen )
- {
+ if( rows > rlen || cols > clen ) {
throw new IOException("Frame block [1:"+rows+",1:"+cols+"] " +
"out of overall frame range [1:"+rlen+",1:"+clen+"].");
}
@@ -95,21 +95,19 @@ public class FrameWriterTextCell extends FrameWriter
//for obj reuse and preventing repeated buffer re-allocations
StringBuilder sb = new StringBuilder();
- for( int i=0; i<rows; i++ )
- {
- String rowIndex = Integer.toString(i+1);
- for( int j=0; j<cols; j++ )
- {
- if(src.get(i, j) != null)
- {
- String lvalue = src.get(i, j).toString();
+ Iterator<String[]> iter = src.getStringRowIterator();
+ for( int i=0; iter.hasNext(); i++ ) { //for all rows
+ String rowIndex = Integer.toString(i+1);
+ String[] row = iter.next();
+ for( int j=0; j<cols; j++ ) {
+ if( row[j] != null ) {
sb.append(rowIndex);
sb.append(' ');
sb.append( j+1 );
sb.append(' ');
- sb.append( lvalue );
+ sb.append( row[j] );
sb.append('\n');
- br.write( sb.toString() ); //same as append
+ br.write( sb.toString() );
sb.setLength(0);
entriesWritten = true;
}
@@ -121,10 +119,8 @@ public class FrameWriterTextCell extends FrameWriter
br.write("1 1 0\n");
}
}
- finally
- {
+ finally {
IOUtilFunctions.closeSilently(br);
}
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java b/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
index e48c868..d40857f 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
@@ -196,8 +196,8 @@ public class CombineMR
{
Pair<Integer, Integer> blockSize=outputBlockSizes.get(ins.output);
keyBuff.setIndexes(
- UtilFunctions.cellIndexCalculation(in1.getIndexes().getRowIndex(), blockSize.getKey(), r),
- UtilFunctions.cellIndexCalculation(in1.getIndexes().getColumnIndex(), blockSize.getValue(), c)
+ UtilFunctions.computeCellIndex(in1.getIndexes().getRowIndex(), blockSize.getKey(), r),
+ UtilFunctions.computeCellIndex(in1.getIndexes().getColumnIndex(), blockSize.getValue(), c)
);
valueBuff.setValue(in1.getValue().getValue(r, c));
valueBuff.setOtherValue(in2.getValue().getValue(r, c));
@@ -259,8 +259,8 @@ public class CombineMR
{
Pair<Integer, Integer> blockSize=outputBlockSizes.get(ins.output);
keyBuff.setIndexes(
- UtilFunctions.cellIndexCalculation(indexes.getRowIndex(), blockSize.getKey(), r),
- UtilFunctions.cellIndexCalculation(indexes.getColumnIndex(), blockSize.getValue(), c)
+ UtilFunctions.computeCellIndex(indexes.getRowIndex(), blockSize.getKey(), r),
+ UtilFunctions.computeCellIndex(indexes.getColumnIndex(), blockSize.getValue(), c)
);
valueBuff.setValue(in1.getValue().getValue(r, c));
double temp=in2.getValue().getValue(r, c);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java
index e4fefe4..42c2095 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java
@@ -58,8 +58,8 @@ Converter<MatrixIndexes, MatrixBlock, MatrixIndexes, MatrixCell>
@Override
public void convert(MatrixIndexes k1, MatrixBlock v1) {
reset();
- startIndexes.setIndexes(UtilFunctions.cellIndexCalculation(k1.getRowIndex(), brow,0),
- UtilFunctions.cellIndexCalculation(k1.getColumnIndex(),bcolumn,0));
+ startIndexes.setIndexes(UtilFunctions.computeCellIndex(k1.getRowIndex(), brow,0),
+ UtilFunctions.computeCellIndex(k1.getColumnIndex(),bcolumn,0));
sparse=v1.isInSparseFormat();
thisBlockWidth=v1.getNumColumns();
if(sparse)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java
index 26ea776..bfbd5bd 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java
@@ -58,7 +58,7 @@ public class BinaryBlockToRowBlockConverter implements Converter<MatrixIndexes,
@Override
public void convert(MatrixIndexes k1, MatrixBlock v1) {
reset();
- _startRowID = UtilFunctions.cellIndexCalculation(k1.getRowIndex(), _srcBrlen, 0);
+ _startRowID = UtilFunctions.computeCellIndex(k1.getRowIndex(), _srcBrlen, 0);
_colID = k1.getColumnIndex();
_destBlock.reset(1, v1.getNumColumns());
_srcRlen = v1.getNumRows();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java
index 0004adc..95cb5ec 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java
@@ -62,8 +62,8 @@ Converter<MatrixIndexes, MatrixBlock, NullWritable, Text>
@Override
public void convert(MatrixIndexes k1, MatrixBlock v1) {
reset();
- startIndexes.setIndexes(UtilFunctions.cellIndexCalculation(k1.getRowIndex(), brow,0),
- UtilFunctions.cellIndexCalculation(k1.getColumnIndex(),bcolumn,0));
+ startIndexes.setIndexes(UtilFunctions.computeCellIndex(k1.getRowIndex(), brow,0),
+ UtilFunctions.computeCellIndex(k1.getColumnIndex(),bcolumn,0));
sparse=v1.isInSparseFormat();
thisBlockWidth=v1.getNumColumns();
if(sparse)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java
index d48a004..a5620e5 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java
@@ -610,7 +610,7 @@ public class LibMatrixAgg
int m = out.rlen;
double[] c = out.getDenseBlock();
for( int i=0, cix=0; i<m; i++, cix+=2 )
- c[cix] = UtilFunctions.cellIndexCalculation(ix.getColumnIndex(), bclen, (int)c[cix]-1);
+ c[cix] = UtilFunctions.computeCellIndex(ix.getColumnIndex(), bclen, (int)c[cix]-1);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index 06ed711..034aeba 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -4520,7 +4520,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
|| ((Builtin)(op.aggOp.increOp.fn)).bFunc == Builtin.BuiltinFunctionCode.MININDEX)
){
double currMaxValue = result.quickGetValue(i, 1);
- long newMaxIndex = UtilFunctions.cellIndexCalculation(indexesIn.getColumnIndex(), blockingFactorCol, j);
+ long newMaxIndex = UtilFunctions.computeCellIndex(indexesIn.getColumnIndex(), blockingFactorCol, j);
double newMaxValue = quickGetValue(i, j);
double update = op.aggOp.increOp.fn.execute(newMaxValue, currMaxValue);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
index d9c8853..ea33de3 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
@@ -305,10 +305,10 @@ public class OperationsOnMatrixValues
public static void performSlice(IndexedMatrixValue in, IndexRange ixrange, int brlen, int bclen, ArrayList<IndexedMatrixValue> outlist)
throws DMLUnsupportedOperationException, DMLRuntimeException
{
- long cellIndexTopRow = UtilFunctions.cellIndexCalculation(in.getIndexes().getRowIndex(), brlen, 0);
- long cellIndexBottomRow = UtilFunctions.cellIndexCalculation(in.getIndexes().getRowIndex(), brlen, in.getValue().getNumRows()-1);
- long cellIndexLeftCol = UtilFunctions.cellIndexCalculation(in.getIndexes().getColumnIndex(), bclen, 0);
- long cellIndexRightCol = UtilFunctions.cellIndexCalculation(in.getIndexes().getColumnIndex(), bclen, in.getValue().getNumColumns()-1);
+ long cellIndexTopRow = UtilFunctions.computeCellIndex(in.getIndexes().getRowIndex(), brlen, 0);
+ long cellIndexBottomRow = UtilFunctions.computeCellIndex(in.getIndexes().getRowIndex(), brlen, in.getValue().getNumRows()-1);
+ long cellIndexLeftCol = UtilFunctions.computeCellIndex(in.getIndexes().getColumnIndex(), bclen, 0);
+ long cellIndexRightCol = UtilFunctions.computeCellIndex(in.getIndexes().getColumnIndex(), bclen, in.getValue().getNumColumns()-1);
long cellIndexOverlapTop = Math.max(cellIndexTopRow, ixrange.rowStart);
long cellIndexOverlapBottom = Math.min(cellIndexBottomRow, ixrange.rowEnd);
@@ -321,13 +321,13 @@ public class OperationsOnMatrixValues
}
IndexRange tmpRange = new IndexRange(
- UtilFunctions.cellInBlockCalculation(cellIndexOverlapTop, brlen),
- UtilFunctions.cellInBlockCalculation(cellIndexOverlapBottom, brlen),
- UtilFunctions.cellInBlockCalculation(cellIndexOverlapLeft, bclen),
- UtilFunctions.cellInBlockCalculation(cellIndexOverlapRight, bclen));
+ UtilFunctions.computeCellInBlock(cellIndexOverlapTop, brlen),
+ UtilFunctions.computeCellInBlock(cellIndexOverlapBottom, brlen),
+ UtilFunctions.computeCellInBlock(cellIndexOverlapLeft, bclen),
+ UtilFunctions.computeCellInBlock(cellIndexOverlapRight, bclen));
- int rowCut=UtilFunctions.cellInBlockCalculation(ixrange.rowStart, brlen);
- int colCut=UtilFunctions.cellInBlockCalculation(ixrange.colStart, bclen);
+ int rowCut=UtilFunctions.computeCellInBlock(ixrange.rowStart, brlen);
+ int colCut=UtilFunctions.computeCellInBlock(ixrange.colStart, bclen);
int rowsInLastBlock = (int)((ixrange.rowEnd-ixrange.rowStart+1)%brlen);
if(rowsInLastBlock==0)
@@ -336,15 +336,15 @@ public class OperationsOnMatrixValues
if(colsInLastBlock==0)
colsInLastBlock=bclen;
- long resultBlockIndexTop=UtilFunctions.blockIndexCalculation(cellIndexOverlapTop-ixrange.rowStart+1, brlen);
- long resultBlockIndexBottom=UtilFunctions.blockIndexCalculation(cellIndexOverlapBottom-ixrange.rowStart+1, brlen);
- long resultBlockIndexLeft=UtilFunctions.blockIndexCalculation(cellIndexOverlapLeft-ixrange.colStart+1, bclen);
- long resultBlockIndexRight=UtilFunctions.blockIndexCalculation(cellIndexOverlapRight-ixrange.colStart+1, bclen);
+ long resultBlockIndexTop=UtilFunctions.computeBlockIndex(cellIndexOverlapTop-ixrange.rowStart+1, brlen);
+ long resultBlockIndexBottom=UtilFunctions.computeBlockIndex(cellIndexOverlapBottom-ixrange.rowStart+1, brlen);
+ long resultBlockIndexLeft=UtilFunctions.computeBlockIndex(cellIndexOverlapLeft-ixrange.colStart+1, bclen);
+ long resultBlockIndexRight=UtilFunctions.computeBlockIndex(cellIndexOverlapRight-ixrange.colStart+1, bclen);
int boundaryRlen = brlen;
int boundaryClen = bclen;
- long finalBlockIndexBottom=UtilFunctions.blockIndexCalculation(ixrange.rowEnd-ixrange.rowStart+1, brlen);
- long finalBlockIndexRight=UtilFunctions.blockIndexCalculation(ixrange.colEnd-ixrange.colStart+1, bclen);
+ long finalBlockIndexBottom=UtilFunctions.computeBlockIndex(ixrange.rowEnd-ixrange.rowStart+1, brlen);
+ long finalBlockIndexRight=UtilFunctions.computeBlockIndex(ixrange.colEnd-ixrange.colStart+1, bclen);
if(resultBlockIndexBottom==finalBlockIndexBottom)
boundaryRlen=rowsInLastBlock;
if(resultBlockIndexRight==finalBlockIndexRight)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java
index bf00997..f371b70 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java
@@ -75,8 +75,8 @@ public class CSVReblockMapper extends MapperBase implements Mapper<LongWritable,
{
int start=0;
row.getIndexes().setTag(outTag);
- long rowIndex=UtilFunctions.blockIndexCalculation(rowOffset+num+1, brlen);
- row.getRow().indexInBlock=UtilFunctions.cellInBlockCalculation(rowOffset+num+1, brlen);
+ long rowIndex=UtilFunctions.computeBlockIndex(rowOffset+num+1, brlen);
+ row.getRow().indexInBlock=UtilFunctions.computeCellInBlock(rowOffset+num+1, brlen);
long col=0;
for(; col<cells.length/bclen; col++)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
index c1d83e7..40fb329 100644
--- a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
+++ b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
@@ -42,34 +42,6 @@ public class UtilFunctions
return (int)((pow2>Integer.MAX_VALUE)?Integer.MAX_VALUE : pow2);
}
- //return one block index given the index in cell format and block size
- //TODO to be deleted
- public static long blockIndexCalculation(long cellIndex, int blockSize)
- {
- if(cellIndex>0)
- return (cellIndex-1)/blockSize+1;
- else
- return (long)Math.floor((double)(cellIndex-1)/(double)blockSize)+1;
- }
-
- //return cell index in the block, for given index in the cell format and block size
- //TODO to be deleted
- public static int cellInBlockCalculation(long cellIndex, int blockSize)
- {
- if(cellIndex>0)
- return (int) ((cellIndex-1)%blockSize);
- else
- //return (int) Math.abs((cellIndex-1)%blockSize);
- return (int) ((cellIndex-1)%blockSize)+blockSize;
- }
-
- //given block index and block size and cells in block, return the index in cell format
- //TODO to be deleted
- public static long cellIndexCalculation(long blockIndex, int blockSize, int cellInBlock)
- {
- return (blockIndex-1)*blockSize+1+cellInBlock;
- }
-
/**
* Computes the 1-based block index based on the global cell index and block size meta
* data. See computeCellIndex for the inverse operation.
@@ -78,8 +50,7 @@ public class UtilFunctions
* @param blockSize
* @return
*/
- public static long computeBlockIndex(long cellIndex, int blockSize)
- {
+ public static long computeBlockIndex(long cellIndex, int blockSize) {
return (cellIndex-1)/blockSize + 1;
}
@@ -91,8 +62,7 @@ public class UtilFunctions
* @param blockSize
* @return
*/
- public static int computeCellInBlock(long cellIndex, int blockSize)
- {
+ public static int computeCellInBlock(long cellIndex, int blockSize) {
return (int) ((cellIndex-1)%blockSize);
}
@@ -107,9 +77,7 @@ public class UtilFunctions
* @param cellInBlock
* @return
*/
- public static long computeCellIndex( long blockIndex, int blockSize, int cellInBlock )
- {
- //
+ public static long computeCellIndex( long blockIndex, int blockSize, int cellInBlock ) {
return (blockIndex-1)*blockSize + 1 + cellInBlock;
}
@@ -123,8 +91,7 @@ public class UtilFunctions
* @param blockSize
* @return
*/
- public static int computeBlockSize( long len, long blockIndex, long blockSize )
- {
+ public static int computeBlockSize( long len, long blockIndex, long blockSize ) {
long remain = len - (blockIndex-1)*blockSize;
return (int)Math.min(blockSize, remain);
}
@@ -189,15 +156,15 @@ public class UtilFunctions
{
IndexRange tempRange = new IndexRange(-1, -1, -1, -1);
- long topBlockRowIndex=UtilFunctions.blockIndexCalculation(indexRange.rowStart, blockRowFactor);
- int topRowInTopBlock=UtilFunctions.cellInBlockCalculation(indexRange.rowStart, blockRowFactor);
- long bottomBlockRowIndex=UtilFunctions.blockIndexCalculation(indexRange.rowEnd, blockRowFactor);
- int bottomRowInBottomBlock=UtilFunctions.cellInBlockCalculation(indexRange.rowEnd, blockRowFactor);
+ long topBlockRowIndex=UtilFunctions.computeBlockIndex(indexRange.rowStart, blockRowFactor);
+ int topRowInTopBlock=UtilFunctions.computeCellInBlock(indexRange.rowStart, blockRowFactor);
+ long bottomBlockRowIndex=UtilFunctions.computeBlockIndex(indexRange.rowEnd, blockRowFactor);
+ int bottomRowInBottomBlock=UtilFunctions.computeCellInBlock(indexRange.rowEnd, blockRowFactor);
- long leftBlockColIndex=UtilFunctions.blockIndexCalculation(indexRange.colStart, blockColFactor);
- int leftColInLeftBlock=UtilFunctions.cellInBlockCalculation(indexRange.colStart, blockColFactor);
- long rightBlockColIndex=UtilFunctions.blockIndexCalculation(indexRange.colEnd, blockColFactor);
- int rightColInRightBlock=UtilFunctions.cellInBlockCalculation(indexRange.colEnd, blockColFactor);
+ long leftBlockColIndex=UtilFunctions.computeBlockIndex(indexRange.colStart, blockColFactor);
+ int leftColInLeftBlock=UtilFunctions.computeCellInBlock(indexRange.colStart, blockColFactor);
+ long rightBlockColIndex=UtilFunctions.computeBlockIndex(indexRange.colEnd, blockColFactor);
+ int rightColInRightBlock=UtilFunctions.computeCellInBlock(indexRange.colEnd, blockColFactor);
//no overlap
if(in.getIndexes().getRowIndex()<topBlockRowIndex || in.getIndexes().getRowIndex()>bottomBlockRowIndex
@@ -234,7 +201,6 @@ public class UtilFunctions
long total = UtilFunctions.getTotalLength(metadata);
long lpos=(long)Math.ceil(total*p);//lower bound is inclusive
long upos=(long)Math.ceil(total*(1-p));//upper bound is inclusive
- //System.out.println("getLengthForInterQuantile(): " + (upos-lpos+1));
return upos-lpos+1;
}