You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/01/18 08:14:59 UTC
[1/7] incubator-systemml git commit: [SYSTEMML-382] Refactoring
matrix block (prep sparse block integration)
Repository: incubator-systemml
Updated Branches:
refs/heads/master c65e34ebb -> 82d5c5f16
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index f2c4030..fe691b7 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -88,7 +88,6 @@ import org.apache.sysml.runtime.util.UtilFunctions;
public class MatrixBlock extends MatrixValue implements Externalizable
{
-
private static final long serialVersionUID = 7319972089143154056L;
//sparsity nnz threshold, based on practical experiments on space consumption and performance
@@ -113,7 +112,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
//matrix data (sparse or dense)
protected double[] denseBlock = null;
- protected SparseRow[] sparseRows = null;
+ protected SparseRow[] sparseBlock = null;
//sparse-block-specific attributes (allocation only)
protected int estimatedNNzsPerRow = -1;
@@ -236,11 +235,11 @@ public class MatrixBlock extends MatrixValue implements Externalizable
public void resetSparse()
{
- if(sparseRows!=null)
+ if(sparseBlock!=null)
{
- for(int i=0; i<Math.min(rlen, sparseRows.length); i++)
- if(sparseRows[i]!=null)
- sparseRows[i].reset(estimatedNNzsPerRow, clen);
+ for(int i=0; i<Math.min(rlen, sparseBlock.length); i++)
+ if(sparseBlock[i]!=null)
+ sparseBlock[i].reset(estimatedNNzsPerRow, clen);
}
}
@@ -369,7 +368,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
public boolean isAllocated()
{
if( sparse )
- return (sparseRows!=null);
+ return (sparseBlock!=null);
else
return (denseBlock!=null);
}
@@ -437,14 +436,14 @@ public class MatrixBlock extends MatrixValue implements Externalizable
public void allocateSparseRowsBlock(boolean clearNNZ)
{
//allocate block if non-existing or too small (guaranteed to be 0-initialized),
- if( sparseRows == null ) {
- sparseRows=new SparseRow[rlen];
+ if( sparseBlock == null ) {
+ sparseBlock=new SparseRow[rlen];
}
- else if( sparseRows.length < rlen ) {
- SparseRow[] oldSparseRows=sparseRows;
- sparseRows = new SparseRow[rlen];
+ else if( sparseBlock.length < rlen ) {
+ SparseRow[] oldSparseRows=sparseBlock;
+ sparseBlock = new SparseRow[rlen];
for(int i=0; i<Math.min(oldSparseRows.length, rlen); i++) {
- sparseRows[i]=oldSparseRows[i];
+ sparseBlock[i]=oldSparseRows[i];
}
}
@@ -487,7 +486,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
if(dense)
denseBlock = null;
if(sparse)
- sparseRows = null;
+ sparseBlock = null;
}
////////
@@ -584,7 +583,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
public boolean isEmptyBlock(boolean safe)
{
boolean ret = false;
- if( sparse && sparseRows==null )
+ if( sparse && sparseBlock==null )
ret = true;
else if( !sparse && denseBlock==null )
ret = true;
@@ -611,36 +610,36 @@ public class MatrixBlock extends MatrixValue implements Externalizable
////////
// Data handling
- public double[] getDenseArray()
+ public double[] getDenseBlock()
{
if(sparse)
return null;
return denseBlock;
}
- public SparseRow[] getSparseRows()
+ public SparseRow[] getSparseBlock()
{
if(!sparse)
return null;
- return sparseRows;
+ return sparseBlock;
}
- public SparseRowsIterator getSparseRowsIterator()
+ public Iterator<IJV> getSparseBlockIterator()
{
//check for valid format, should have been checked from outside
if( !sparse )
throw new RuntimeException("getSparseCellInterator should not be called for dense format");
- return new SparseRowsIterator(rlen, sparseRows);
+ return new SparseRowsIterator(rlen, sparseBlock);
}
- public SparseRowsIterator getSparseRowsIterator(int rl, int ru)
+ public SparseRowsIterator getSparseBlockIterator(int rl, int ru)
{
//check for valid format, should have been checked from outside
if( !sparse )
throw new RuntimeException("getSparseCellInterator should not be called for dense format");
- return new SparseRowsIterator(rl, ru, sparseRows);
+ return new SparseRowsIterator(rl, ru, sparseBlock);
}
@Override
@@ -673,9 +672,9 @@ public class MatrixBlock extends MatrixValue implements Externalizable
{
if(sparse)
{
- if( sparseRows==null || sparseRows.length<=r || sparseRows[r]==null )
+ if( sparseBlock==null || sparseBlock.length<=r || sparseBlock[r]==null )
return 0;
- return sparseRows[r].get(c);
+ return sparseBlock[r].get(c);
}
else
{
@@ -696,16 +695,16 @@ public class MatrixBlock extends MatrixValue implements Externalizable
if(sparse)
{
//early abort
- if( (sparseRows==null || sparseRows.length<=r || sparseRows[r]==null) && v==0 )
+ if( (sparseBlock==null || sparseBlock.length<=r || sparseBlock[r]==null) && v==0 )
return;
//allocation on demand
allocateSparseRowsBlock(false);
- if( sparseRows[r]==null )
- sparseRows[r] = new SparseRow(estimatedNNzsPerRow, clen);
+ if( sparseBlock[r]==null )
+ sparseBlock[r] = new SparseRow(estimatedNNzsPerRow, clen);
//set value and maintain nnz
- if( sparseRows[r].set(c, v) )
+ if( sparseBlock[r].set(c, v) )
nonZeros += (v!=0) ? 1 : -1;
}
else
@@ -747,9 +746,9 @@ public class MatrixBlock extends MatrixValue implements Externalizable
public double getValueSparseUnsafe(int r, int c)
{
- if(sparseRows==null || sparseRows.length<=r || sparseRows[r]==null)
+ if(sparseBlock==null || sparseBlock.length<=r || sparseBlock[r]==null)
return 0;
- return sparseRows[r].get(c);
+ return sparseBlock[r].get(c);
}
/**
@@ -779,11 +778,11 @@ public class MatrixBlock extends MatrixValue implements Externalizable
{
//allocation on demand (w/o overwriting nnz)
allocateSparseRowsBlock(false);
- if(sparseRows[r]==null)
- sparseRows[r]=new SparseRow(estimatedNNzsPerRow, clen);
+ if(sparseBlock[r]==null)
+ sparseBlock[r]=new SparseRow(estimatedNNzsPerRow, clen);
//set value and maintain nnz
- sparseRows[r].append(c, v);
+ sparseBlock[r].append(c, v);
nonZeros++;
}
}
@@ -796,10 +795,10 @@ public class MatrixBlock extends MatrixValue implements Externalizable
{
//allocation on demand
allocateSparseRowsBlock(false);
- if(sparseRows[r]==null)
- sparseRows[r]=new SparseRow(values);
+ if(sparseBlock[r]==null)
+ sparseBlock[r]=new SparseRow(values);
else
- sparseRows[r].copy(values);
+ sparseBlock[r].copy(values);
nonZeros+=values.size();
}
@@ -830,7 +829,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
{
for( int i=0; i<that.rlen; i++ )
{
- SparseRow brow = that.sparseRows[i];
+ SparseRow brow = that.sparseBlock[i];
if( brow!=null && !brow.isEmpty() )
{
int aix = rowoffset+i;
@@ -838,11 +837,11 @@ public class MatrixBlock extends MatrixValue implements Externalizable
int[] ix = brow.getIndexContainer();
double[] val = brow.getValueContainer();
- if( sparseRows[aix]==null )
- sparseRows[aix] = new SparseRow(estimatedNNzsPerRow,clen);
+ if( sparseBlock[aix]==null )
+ sparseBlock[aix] = new SparseRow(estimatedNNzsPerRow,clen);
for( int j=0; j<len; j++ )
- sparseRows[aix].append(coloffset+ix[j], val[j]);
+ sparseBlock[aix].append(coloffset+ix[j], val[j]);
}
}
}
@@ -856,9 +855,9 @@ public class MatrixBlock extends MatrixValue implements Externalizable
double val = that.denseBlock[bix+j];
if( val != 0 )
{
- if( sparseRows[aix]==null )//create sparserow only if required
- sparseRows[aix] = new SparseRow(estimatedNNzsPerRow,clen);
- sparseRows[aix].append(coloffset+j, val);
+ if( sparseBlock[aix]==null )//create sparserow only if required
+ sparseBlock[aix] = new SparseRow(estimatedNNzsPerRow,clen);
+ sparseBlock[aix].append(coloffset+j, val);
}
}
}
@@ -870,10 +869,10 @@ public class MatrixBlock extends MatrixValue implements Externalizable
*/
public void sortSparseRows()
{
- if( !sparse || sparseRows==null )
+ if( !sparse || sparseBlock==null )
return;
- for( SparseRow arow : sparseRows )
+ for( SparseRow arow : sparseBlock )
if( arow!=null && arow.size()>1 )
arow.sort();
}
@@ -1175,7 +1174,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
//copy dense to sparse
double[] a = denseBlock;
- SparseRow[] c = sparseRows;
+ SparseRow[] c = sparseBlock;
for( int i=0, aix=0; i<rlen; i++ )
for(int j=0; j<clen; j++, aix++)
@@ -1201,7 +1200,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
sparse = false;
//early abort on empty blocks
- if(sparseRows==null)
+ if(sparseBlock==null)
return;
int limit=rlen*clen;
@@ -1214,7 +1213,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
Arrays.fill(denseBlock, 0, limit, 0);
//copy sparse to dense
- SparseRow[] a = sparseRows;
+ SparseRow[] a = sparseBlock;
double[] c = denseBlock;
for( int i=0, cix=0; i<rlen; i++, cix+=clen)
@@ -1228,18 +1227,18 @@ public class MatrixBlock extends MatrixValue implements Externalizable
}
//cleanup sparse rows
- sparseRows = null;
+ sparseBlock = null;
}
public void recomputeNonZeros()
{
nonZeros=0;
- if( sparse && sparseRows!=null )
+ if( sparse && sparseBlock!=null )
{
- int limit = Math.min(rlen, sparseRows.length);
+ int limit = Math.min(rlen, sparseBlock.length);
for(int i=0; i<limit; i++)
- if(sparseRows[i]!=null)
- nonZeros += sparseRows[i].size();
+ if(sparseBlock[i]!=null)
+ nonZeros += sparseBlock[i].size();
}
else if( !sparse && denseBlock!=null )
{
@@ -1259,28 +1258,28 @@ public class MatrixBlock extends MatrixValue implements Externalizable
long nnz = 0;
if(sparse)
{
- if(sparseRows!=null)
+ if(sparseBlock!=null)
{
- int rlimit = Math.min( ru+1, Math.min(rlen, sparseRows.length) );
+ int rlimit = Math.min( ru+1, Math.min(rlen, sparseBlock.length) );
if( cl==0 && cu==clen-1 ) //specific case: all cols
{
for(int i=rl; i<rlimit; i++)
- if(sparseRows[i]!=null && !sparseRows[i].isEmpty())
- nnz+=sparseRows[i].size();
+ if(sparseBlock[i]!=null && !sparseBlock[i].isEmpty())
+ nnz+=sparseBlock[i].size();
}
else if( cl==cu ) //specific case: one column
{
for(int i=rl; i<rlimit; i++)
- if(sparseRows[i]!=null && !sparseRows[i].isEmpty())
- nnz += (sparseRows[i].get(cl)!=0) ? 1 : 0;
+ if(sparseBlock[i]!=null && !sparseBlock[i].isEmpty())
+ nnz += (sparseBlock[i].get(cl)!=0) ? 1 : 0;
}
else //general case
{
int astart,aend;
for(int i=rl; i<rlimit; i++)
- if(sparseRows[i]!=null && !sparseRows[i].isEmpty())
+ if(sparseBlock[i]!=null && !sparseBlock[i].isEmpty())
{
- SparseRow arow = sparseRows[i];
+ SparseRow arow = sparseBlock[i];
astart = arow.searchIndexesFirstGTE(cl);
aend = arow.searchIndexesFirstGTE(cu);
nnz += (astart!=-1) ? (aend-astart+1) : 0;
@@ -1366,16 +1365,16 @@ public class MatrixBlock extends MatrixValue implements Externalizable
}
allocateSparseRowsBlock(false);
- for(int i=0; i<Math.min(that.sparseRows.length, rlen); i++)
+ for(int i=0; i<Math.min(that.sparseBlock.length, rlen); i++)
{
- if(that.sparseRows[i]!=null)
+ if(that.sparseBlock[i]!=null)
{
- if(sparseRows[i]==null)
- sparseRows[i]=new SparseRow(that.sparseRows[i]);
+ if(sparseBlock[i]==null)
+ sparseBlock[i]=new SparseRow(that.sparseBlock[i]);
else
- sparseRows[i].copy(that.sparseRows[i]);
- }else if(this.sparseRows[i]!=null)
- this.sparseRows[i].reset(estimatedNNzsPerRow, clen);
+ sparseBlock[i].copy(that.sparseBlock[i]);
+ }else if(this.sparseBlock[i]!=null)
+ this.sparseBlock[i].reset(estimatedNNzsPerRow, clen);
}
}
@@ -1413,13 +1412,13 @@ public class MatrixBlock extends MatrixValue implements Externalizable
allocateDenseBlock(false);
int start=0;
- for(int r=0; r<Math.min(that.sparseRows.length, rlen); r++, start+=clen)
+ for(int r=0; r<Math.min(that.sparseBlock.length, rlen); r++, start+=clen)
{
- if(that.sparseRows[r]==null)
+ if(that.sparseBlock[r]==null)
continue;
- double[] values=that.sparseRows[r].getValueContainer();
- int[] cols=that.sparseRows[r].getIndexContainer();
- for(int i=0; i<that.sparseRows[r].size(); i++)
+ double[] values=that.sparseBlock[r].getValueContainer();
+ int[] cols=that.sparseBlock[r].getIndexContainer();
+ for(int i=0; i<that.sparseBlock[r].size(); i++)
{
denseBlock[start+cols[i]]=values[i];
}
@@ -1439,17 +1438,17 @@ public class MatrixBlock extends MatrixValue implements Externalizable
for(int i=0, ix=0; i<rlen; i++)
{
- if( sparseRows[i]!=null )
- sparseRows[i].reset(estimatedNNzsPerRow, clen);
+ if( sparseBlock[i]!=null )
+ sparseBlock[i].reset(estimatedNNzsPerRow, clen);
for(int j=0; j<clen; j++)
{
double val = that.denseBlock[ix++];
if( val != 0 )
{
- if(sparseRows[i]==null) //create sparse row only if required
- sparseRows[i]=new SparseRow(estimatedNNzsPerRow, clen);
- sparseRows[i].append(j, val);
+ if(sparseBlock[i]==null) //create sparse row only if required
+ sparseBlock[i]=new SparseRow(estimatedNNzsPerRow, clen);
+ sparseBlock[i].append(j, val);
}
}
}
@@ -1491,12 +1490,12 @@ public class MatrixBlock extends MatrixValue implements Externalizable
//handle empty src and dest
if( src.isEmptyBlock(false) )
{
- if( awareDestNZ && sparseRows != null )
+ if( awareDestNZ && sparseBlock != null )
copyEmptyToSparse(rl, ru, cl, cu, true);
return;
}
- if(sparseRows==null)
- sparseRows=new SparseRow[rlen];
+ if(sparseBlock==null)
+ sparseBlock=new SparseRow[rlen];
else if( awareDestNZ )
{
copyEmptyToSparse(rl, ru, cl, cu, true);
@@ -1511,17 +1510,17 @@ public class MatrixBlock extends MatrixValue implements Externalizable
for( int i=0; i<src.rlen; i++ )
{
- SparseRow arow = src.sparseRows[i];
+ SparseRow arow = src.sparseBlock[i];
if( arow != null && !arow.isEmpty() )
{
alen = arow.size();
aix = arow.getIndexContainer();
avals = arow.getValueContainer();
- if( sparseRows[rl+i] == null || sparseRows[rl+i].isEmpty() )
+ if( sparseBlock[rl+i] == null || sparseBlock[rl+i].isEmpty() )
{
- sparseRows[rl+i] = new SparseRow(estimatedNNzsPerRow, clen);
- SparseRow brow = sparseRows[rl+i];
+ sparseBlock[rl+i] = new SparseRow(estimatedNNzsPerRow, clen);
+ SparseRow brow = sparseBlock[rl+i];
for( int j=0; j<alen; j++ )
brow.append(cl+aix[j], avals[j]);
@@ -1530,7 +1529,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
}
else if( awareDestNZ ) //general case (w/ awareness NNZ)
{
- SparseRow brow = sparseRows[rl+i];
+ SparseRow brow = sparseBlock[rl+i];
int lnnz = brow.size();
if( cl==cu && cl==aix[0] )
{
@@ -1549,7 +1548,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
}
else //general case (w/o awareness NNZ)
{
- SparseRow brow = sparseRows[rl+i];
+ SparseRow brow = sparseBlock[rl+i];
//brow.set(cl, arow);
for( int j=0; j<alen; j++ )
@@ -1586,7 +1585,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
for( int i=0, ix=rl*clen; i<src.rlen; i++, ix+=clen )
{
- SparseRow arow = src.sparseRows[i];
+ SparseRow arow = src.sparseBlock[i];
if( arow != null && !arow.isEmpty() )
{
alen = arow.size();
@@ -1607,12 +1606,12 @@ public class MatrixBlock extends MatrixValue implements Externalizable
//handle empty src and dest
if( src.isEmptyBlock(false) )
{
- if( awareDestNZ && sparseRows != null )
+ if( awareDestNZ && sparseBlock != null )
copyEmptyToSparse(rl, ru, cl, cu, true);
return;
}
- if(sparseRows==null)
- sparseRows=new SparseRow[rlen];
+ if(sparseBlock==null)
+ sparseBlock=new SparseRow[rlen];
//no need to clear for awareDestNZ since overwritten
//copy values
@@ -1620,22 +1619,22 @@ public class MatrixBlock extends MatrixValue implements Externalizable
for( int i=0, ix=0; i<src.rlen; i++, ix+=src.clen )
{
int rix = rl + i;
- if( sparseRows[rix]==null || sparseRows[rix].isEmpty() )
+ if( sparseBlock[rix]==null || sparseBlock[rix].isEmpty() )
{
for( int j=0; j<src.clen; j++ )
if( (val = src.denseBlock[ix+j]) != 0 )
{
- if( sparseRows[rix]==null )
- sparseRows[rix] = new SparseRow(estimatedNNzsPerRow, clen);
- sparseRows[rix].append(cl+j, val);
+ if( sparseBlock[rix]==null )
+ sparseBlock[rix] = new SparseRow(estimatedNNzsPerRow, clen);
+ sparseBlock[rix].append(cl+j, val);
}
- if( awareDestNZ && sparseRows[rix]!=null )
- nonZeros += sparseRows[rix].size();
+ if( awareDestNZ && sparseBlock[rix]!=null )
+ nonZeros += sparseBlock[rix].size();
}
else if( awareDestNZ ) //general case (w/ awareness NNZ)
{
- SparseRow brow = sparseRows[rix];
+ SparseRow brow = sparseBlock[rix];
int lnnz = brow.size();
if( cl==cu )
{
@@ -1652,7 +1651,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
}
else //general case (w/o awareness NNZ)
{
- SparseRow brow = sparseRows[rix];
+ SparseRow brow = sparseBlock[rix];
for( int j=0; j<src.clen; j++ )
if( (val = src.denseBlock[ix+j]) != 0 )
brow.set(cl+j, val);
@@ -1696,18 +1695,18 @@ public class MatrixBlock extends MatrixValue implements Externalizable
if( updateNNZ )
{
for( int i=rl; i<=ru; i++ )
- if( sparseRows[i] != null && !sparseRows[i].isEmpty() )
+ if( sparseBlock[i] != null && !sparseBlock[i].isEmpty() )
{
- int lnnz = sparseRows[i].size();
- sparseRows[i].delete(cl);
- nonZeros += (sparseRows[i].size()-lnnz);
+ int lnnz = sparseBlock[i].size();
+ sparseBlock[i].delete(cl);
+ nonZeros += (sparseBlock[i].size()-lnnz);
}
}
else
{
for( int i=rl; i<=ru; i++ )
- if( sparseRows[i] != null && !sparseRows[i].isEmpty() )
- sparseRows[i].delete(cl);
+ if( sparseBlock[i] != null && !sparseBlock[i].isEmpty() )
+ sparseBlock[i].delete(cl);
}
}
else
@@ -1715,18 +1714,18 @@ public class MatrixBlock extends MatrixValue implements Externalizable
if( updateNNZ )
{
for( int i=rl; i<=ru; i++ )
- if( sparseRows[i] != null && !sparseRows[i].isEmpty() )
+ if( sparseBlock[i] != null && !sparseBlock[i].isEmpty() )
{
- int lnnz = sparseRows[i].size();
- sparseRows[i].deleteIndexRange(cl, cu);
- nonZeros += (sparseRows[i].size()-lnnz);
+ int lnnz = sparseBlock[i].size();
+ sparseBlock[i].deleteIndexRange(cl, cu);
+ nonZeros += (sparseBlock[i].size()-lnnz);
}
}
else
{
for( int i=rl; i<=ru; i++ )
- if( sparseRows[i] != null && !sparseRows[i].isEmpty() )
- sparseRows[i].deleteIndexRange(cl, cu);
+ if( sparseBlock[i] != null && !sparseBlock[i].isEmpty() )
+ sparseBlock[i].deleteIndexRange(cl, cu);
}
}
}
@@ -1795,7 +1794,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
if( that.sparse ) //DENSE <- SPARSE
{
double[] a = denseBlock;
- SparseRow[] b = that.sparseRows;
+ SparseRow[] b = that.sparseBlock;
int m = rlen;
int n = clen;
@@ -1831,8 +1830,8 @@ public class MatrixBlock extends MatrixValue implements Externalizable
{
if( that.sparse ) //SPARSE <- SPARSE
{
- SparseRow[] a = sparseRows;
- SparseRow[] b = that.sparseRows;
+ SparseRow[] a = sparseBlock;
+ SparseRow[] b = that.sparseBlock;
int m = rlen;
for( int i=0; i<m; i++ )
@@ -1866,7 +1865,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
}
else //SPARSE <- DENSE
{
- SparseRow[] a = sparseRows;
+ SparseRow[] a = sparseBlock;
double[] b = that.denseBlock;
int m = rlen;
int n = clen;
@@ -1995,14 +1994,14 @@ public class MatrixBlock extends MatrixValue implements Externalizable
if( in instanceof MatrixBlockDataInput ) //fast deserialize
{
MatrixBlockDataInput mbin = (MatrixBlockDataInput)in;
- nonZeros = mbin.readSparseRows(rlen, sparseRows);
+ nonZeros = mbin.readSparseRows(rlen, sparseBlock);
}
else if( in instanceof DataInputBuffer && MRJobConfiguration.USE_BINARYBLOCK_SERIALIZATION )
{
//workaround because sequencefile.reader.next(key, value) does not yet support serialization framework
DataInputBuffer din = (DataInputBuffer)in;
MatrixBlockDataInput mbin = new FastBufferedDataInputStream(din);
- nonZeros = mbin.readSparseRows(rlen, sparseRows);
+ nonZeros = mbin.readSparseRows(rlen, sparseBlock);
((FastBufferedDataInputStream)mbin).close();
}
else //default deserialize
@@ -2012,16 +2011,16 @@ public class MatrixBlock extends MatrixValue implements Externalizable
int nr=in.readInt();
if(nr==0)
{
- if(sparseRows[r]!=null)
- sparseRows[r].reset(estimatedNNzsPerRow, clen);
+ if(sparseBlock[r]!=null)
+ sparseBlock[r].reset(estimatedNNzsPerRow, clen);
continue;
}
- if(sparseRows[r]==null)
- sparseRows[r]=new SparseRow(nr);
+ if(sparseBlock[r]==null)
+ sparseBlock[r]=new SparseRow(nr);
else
- sparseRows[r].reset(nr, clen);
+ sparseBlock[r].reset(nr, clen);
for(int j=0; j<nr; j++)
- sparseRows[r].append(in.readInt(), in.readDouble());
+ sparseBlock[r].append(in.readInt(), in.readDouble());
}
}
}
@@ -2068,9 +2067,9 @@ public class MatrixBlock extends MatrixValue implements Externalizable
int r = in.readInt();
int c = in.readInt();
double val = in.readDouble();
- if(sparseRows[r]==null)
- sparseRows[r]=new SparseRow(1,clen);
- sparseRows[r].append(c, val);
+ if(sparseBlock[r]==null)
+ sparseBlock[r]=new SparseRow(1,clen);
+ sparseBlock[r].append(c, val);
}
}
else //ULTRA-SPARSE COL
@@ -2079,9 +2078,9 @@ public class MatrixBlock extends MatrixValue implements Externalizable
for(long i=0; i<nonZeros; i++) {
int r = in.readInt();
double val = in.readDouble();
- if(sparseRows[r]==null)
- sparseRows[r]=new SparseRow(1,1);
- sparseRows[r].append(0, val);
+ if(sparseBlock[r]==null)
+ sparseBlock[r]=new SparseRow(1,1);
+ sparseBlock[r].append(0, val);
}
}
}
@@ -2134,7 +2133,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
if( sparseSrc )
{
//write sparse to *
- if( sparseRows==null || nonZeros==0 )
+ if( sparseBlock==null || nonZeros==0 )
writeEmptyBlock(out);
else if( nonZeros<rlen && sparseDst )
writeSparseToUltraSparse(out);
@@ -2199,20 +2198,20 @@ public class MatrixBlock extends MatrixValue implements Externalizable
writeNnzInfo( out, false );
if( out instanceof MatrixBlockDataOutput ) //fast serialize
- ((MatrixBlockDataOutput)out).writeSparseRows(rlen, sparseRows);
+ ((MatrixBlockDataOutput)out).writeSparseRows(rlen, sparseBlock);
else //general case (if fast serialize not supported)
{
int r=0;
- for(;r<Math.min(rlen, sparseRows.length); r++)
+ for(;r<Math.min(rlen, sparseBlock.length); r++)
{
- if(sparseRows[r]==null)
+ if(sparseBlock[r]==null)
out.writeInt(0);
else
{
- int nr=sparseRows[r].size();
+ int nr=sparseBlock[r].size();
out.writeInt(nr);
- int[] cols=sparseRows[r].getIndexContainer();
- double[] values=sparseRows[r].getValueContainer();
+ int[] cols=sparseBlock[r].getIndexContainer();
+ double[] values=sparseBlock[r].getValueContainer();
for(int j=0; j<nr; j++)
{
out.writeInt(cols[j]);
@@ -2241,12 +2240,12 @@ public class MatrixBlock extends MatrixValue implements Externalizable
if( clen > 1 ) //ULTRA-SPARSE BLOCK
{
//block: write ijv-triples
- for(int r=0;r<Math.min(rlen, sparseRows.length); r++)
- if(sparseRows[r]!=null && !sparseRows[r].isEmpty() )
+ for(int r=0;r<Math.min(rlen, sparseBlock.length); r++)
+ if(sparseBlock[r]!=null && !sparseBlock[r].isEmpty() )
{
- int alen = sparseRows[r].size();
- int[] aix = sparseRows[r].getIndexContainer();
- double[] avals = sparseRows[r].getValueContainer();
+ int alen = sparseBlock[r].size();
+ int[] aix = sparseBlock[r].getIndexContainer();
+ double[] avals = sparseBlock[r].getValueContainer();
for(int j=0; j<alen; j++) {
//ultra-sparse block: write ijv-triples
out.writeInt(r);
@@ -2259,10 +2258,10 @@ public class MatrixBlock extends MatrixValue implements Externalizable
else //ULTRA-SPARSE COL
{
//block: write iv-pairs (should never happen since always dense)
- for(int r=0;r<Math.min(rlen, sparseRows.length); r++)
- if(sparseRows[r]!=null && !sparseRows[r].isEmpty() ) {
+ for(int r=0;r<Math.min(rlen, sparseBlock.length); r++)
+ if(sparseBlock[r]!=null && !sparseBlock[r].isEmpty() ) {
out.writeInt(r);
- out.writeDouble(sparseRows[r].getValueContainer()[0]);
+ out.writeDouble(sparseBlock[r].getValueContainer()[0]);
wnnz++;
}
}
@@ -2285,16 +2284,16 @@ public class MatrixBlock extends MatrixValue implements Externalizable
out.writeByte( BlockType.DENSE_BLOCK.ordinal() );
//write data (from sparse to dense)
- if( sparseRows==null ) //empty block
+ if( sparseBlock==null ) //empty block
for( int i=0; i<rlen*clen; i++ )
out.writeDouble(0);
else //existing sparse block
{
for( int i=0; i<rlen; i++ )
{
- if( i<sparseRows.length && sparseRows[i]!=null && !sparseRows[i].isEmpty() )
+ if( i<sparseBlock.length && sparseBlock[i]!=null && !sparseBlock[i].isEmpty() )
{
- SparseRow arow = sparseRows[i];
+ SparseRow arow = sparseBlock[i];
int alen = arow.size();
int[] aix = arow.getIndexContainer();
double[] avals = arow.getValueContainer();
@@ -2508,7 +2507,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
if( sparseSrc )
{
//write sparse to *
- if(sparseRows==null || lnonZeros==0)
+ if(sparseBlock==null || lnonZeros==0)
return HEADER_SIZE; //empty block
else if( lnonZeros<lrlen && sparseDst )
return estimateSizeUltraSparseOnDisk(lrlen, lclen, lnonZeros); //ultra sparse block
@@ -2899,7 +2898,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
if( sparse ) //SPARSE <- SPARSE
{
- SparseRow[] a = sparseRows;
+ SparseRow[] a = sparseBlock;
for(int i=0; i<m; i++) {
if( a[i]!=null && !a[i].isEmpty() )
@@ -2994,14 +2993,14 @@ public class MatrixBlock extends MatrixValue implements Externalizable
if(sparse)
{
nonZeros=0;
- for(int r=0; r<Math.min(rlen, sparseRows.length); r++)
+ for(int r=0; r<Math.min(rlen, sparseBlock.length); r++)
{
- if(sparseRows[r]==null)
+ if(sparseBlock[r]==null)
continue;
- double[] values=sparseRows[r].getValueContainer();
- int[] cols=sparseRows[r].getIndexContainer();
+ double[] values=sparseBlock[r].getValueContainer();
+ int[] cols=sparseBlock[r].getIndexContainer();
int pos=0;
- for(int i=0; i<sparseRows[r].size(); i++)
+ for(int i=0; i<sparseBlock[r].size(); i++)
{
double v=op.fn.execute(values[i]);
if(v!=0)
@@ -3012,7 +3011,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
nonZeros++;
}
}
- sparseRows[r].truncate(pos);
+ sparseBlock[r].truncate(pos);
}
}
@@ -3210,7 +3209,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
{
if( newWithCor.isInSparseFormat() && aggOp.sparseSafe ) //SPARSE
{
- SparseRow[] bRows = newWithCor.getSparseRows();
+ SparseRow[] bRows = newWithCor.getSparseBlock();
if( bRows==null ) //early abort on empty block
return;
for( int r=0; r<Math.min(rlen, bRows.length); r++ )
@@ -3608,15 +3607,15 @@ public class MatrixBlock extends MatrixValue implements Externalizable
CellIndex temp = new CellIndex(0, 0);
if(sparse)
{
- if(sparseRows!=null)
+ if(sparseBlock!=null)
{
- for(int r=0; r<Math.min(rlen, sparseRows.length); r++)
+ for(int r=0; r<Math.min(rlen, sparseBlock.length); r++)
{
- if(sparseRows[r]==null)
+ if(sparseBlock[r]==null)
continue;
- int[] cols=sparseRows[r].getIndexContainer();
- double[] values=sparseRows[r].getValueContainer();
- for(int i=0; i<sparseRows[r].size(); i++)
+ int[] cols=sparseBlock[r].getIndexContainer();
+ double[] values=sparseBlock[r].getValueContainer();
+ for(int i=0; i<sparseBlock[r].size(); i++)
{
tempCellIndex.set(r, cols[i]);
op.fn.execute(tempCellIndex, temp);
@@ -4062,7 +4061,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
//note: always dense dest
dest.allocateDenseBlock();
for( int i=rl; i<=ru; i++ ) {
- SparseRow arow = sparseRows[i];
+ SparseRow arow = sparseBlock[i];
if( arow != null && !arow.isEmpty() ) {
double val = arow.get(cl);
if( val != 0 ) {
@@ -4075,14 +4074,14 @@ public class MatrixBlock extends MatrixValue implements Externalizable
else if( rl==ru && cl==0 && cu==clen-1 ) //ROW VECTOR
{
//note: always sparse dest, but also works for dense
- dest.appendRow(0, sparseRows[rl]);
+ dest.appendRow(0, sparseBlock[rl]);
}
else //general case (sparse/dense dest)
{
for(int i=rl; i <= ru; i++)
- if(sparseRows[i] != null && !sparseRows[i].isEmpty())
+ if(sparseBlock[i] != null && !sparseBlock[i].isEmpty())
{
- SparseRow arow = sparseRows[i];
+ SparseRow arow = sparseBlock[i];
int alen = arow.size();
int[] aix = arow.getIndexContainer();
double[] avals = arow.getValueContainer();
@@ -4189,13 +4188,13 @@ public class MatrixBlock extends MatrixValue implements Externalizable
if(sparse)
{
- if(sparseRows!=null)
+ if(sparseBlock!=null)
{
int r=(int)range.rowStart;
- for(; r<Math.min(Math.min(rowCut, sparseRows.length), range.rowEnd+1); r++)
+ for(; r<Math.min(Math.min(rowCut, sparseBlock.length), range.rowEnd+1); r++)
sliceHelp(r, range, colCut, topleft, topright, normalBlockRowFactor-rowCut, normalBlockRowFactor, normalBlockColFactor);
- for(; r<=Math.min(range.rowEnd, sparseRows.length-1); r++)
+ for(; r<=Math.min(range.rowEnd, sparseBlock.length-1); r++)
sliceHelp(r, range, colCut, bottomleft, bottomright, -rowCut, normalBlockRowFactor, normalBlockColFactor);
//System.out.println("in: \n"+this);
//System.out.println("outlist: \n"+outlist);
@@ -4231,15 +4230,15 @@ public class MatrixBlock extends MatrixValue implements Externalizable
private void sliceHelp(int r, IndexRange range, int colCut, MatrixBlock left, MatrixBlock right, int rowOffset, int normalBlockRowFactor, int normalBlockColFactor)
{
- if(sparseRows[r]==null)
+ if(sparseBlock[r]==null)
return;
- int[] cols=sparseRows[r].getIndexContainer();
- double[] values=sparseRows[r].getValueContainer();
- int start=sparseRows[r].searchIndexesFirstGTE((int)range.colStart);
+ int[] cols=sparseBlock[r].getIndexContainer();
+ double[] values=sparseBlock[r].getValueContainer();
+ int start=sparseBlock[r].searchIndexesFirstGTE((int)range.colStart);
if(start<0)
return;
- int end=sparseRows[r].searchIndexesFirstLTE((int)range.colEnd);
+ int end=sparseBlock[r].searchIndexesFirstLTE((int)range.colEnd);
if(end<0 || start>end)
return;
@@ -4321,27 +4320,27 @@ public class MatrixBlock extends MatrixValue implements Externalizable
if(sparse)
{
- if(sparseRows!=null)
+ if(sparseBlock!=null)
{
if(!complementary)//if zero out
{
- for(int r=0; r<Math.min((int)range.rowStart, sparseRows.length); r++)
- ((MatrixBlock) result).appendRow(r, sparseRows[r]);
- for(int r=Math.min((int)range.rowEnd+1, sparseRows.length); r<Math.min(rlen, sparseRows.length); r++)
- ((MatrixBlock) result).appendRow(r, sparseRows[r]);
+ for(int r=0; r<Math.min((int)range.rowStart, sparseBlock.length); r++)
+ ((MatrixBlock) result).appendRow(r, sparseBlock[r]);
+ for(int r=Math.min((int)range.rowEnd+1, sparseBlock.length); r<Math.min(rlen, sparseBlock.length); r++)
+ ((MatrixBlock) result).appendRow(r, sparseBlock[r]);
}
- for(int r=(int)range.rowStart; r<=Math.min(range.rowEnd, sparseRows.length-1); r++)
+ for(int r=(int)range.rowStart; r<=Math.min(range.rowEnd, sparseBlock.length-1); r++)
{
- if(sparseRows[r]==null)
+ if(sparseBlock[r]==null)
continue;
- int[] cols=sparseRows[r].getIndexContainer();
- double[] values=sparseRows[r].getValueContainer();
+ int[] cols=sparseBlock[r].getIndexContainer();
+ double[] values=sparseBlock[r].getValueContainer();
if(complementary)//if selection
{
- int start=sparseRows[r].searchIndexesFirstGTE((int)range.colStart);
+ int start=sparseBlock[r].searchIndexesFirstGTE((int)range.colStart);
if(start<0) continue;
- int end=sparseRows[r].searchIndexesFirstGT((int)range.colEnd);
+ int end=sparseBlock[r].searchIndexesFirstGT((int)range.colEnd);
if(end<0 || start>end)
continue;
@@ -4351,16 +4350,16 @@ public class MatrixBlock extends MatrixValue implements Externalizable
}
}else
{
- int start=sparseRows[r].searchIndexesFirstGTE((int)range.colStart);
- if(start<0) start=sparseRows[r].size();
- int end=sparseRows[r].searchIndexesFirstGT((int)range.colEnd);
- if(end<0) end=sparseRows[r].size();
+ int start=sparseBlock[r].searchIndexesFirstGTE((int)range.colStart);
+ if(start<0) start=sparseBlock[r].size();
+ int end=sparseBlock[r].searchIndexesFirstGT((int)range.colEnd);
+ if(end<0) end=sparseBlock[r].size();
for(int i=0; i<start; i++)
{
((MatrixBlock) result).appendValue(r, cols[i], values[i]);
}
- for(int i=end; i<sparseRows[r].size(); i++)
+ for(int i=end; i<sparseBlock[r].size(); i++)
{
((MatrixBlock) result).appendValue(r, cols[i], values[i]);
}
@@ -4485,15 +4484,15 @@ public class MatrixBlock extends MatrixValue implements Externalizable
if(sparse)
{
- if(sparseRows!=null)
+ if(sparseBlock!=null)
{
- for(r=0; r<Math.min(rlen, sparseRows.length); r++)
+ for(r=0; r<Math.min(rlen, sparseBlock.length); r++)
{
- if(sparseRows[r]==null)
+ if(sparseBlock[r]==null)
continue;
- int[] cols=sparseRows[r].getIndexContainer();
- double[] values=sparseRows[r].getValueContainer();
- for(int i=0; i<sparseRows[r].size(); i++)
+ int[] cols=sparseBlock[r].getIndexContainer();
+ double[] values=sparseBlock[r].getValueContainer();
+ for(int i=0; i<sparseBlock[r].size(); i++)
{
tempCellIndex.set(r, cols[i]);
op.indexFn.execute(tempCellIndex, tempCellIndex);
@@ -4653,10 +4652,10 @@ public class MatrixBlock extends MatrixValue implements Externalizable
{
if( sparse ) //SPARSE
{
- if(sparseRows!=null)
+ if(sparseBlock!=null)
for(int i=1; i<=step; i++)
- if(sparseRows[rlen-i]!=null)
- this.nonZeros-=sparseRows[rlen-i].size();
+ if(sparseBlock[rlen-i]!=null)
+ this.nonZeros-=sparseBlock[rlen-i].size();
}
else //DENSE
{
@@ -4677,16 +4676,16 @@ public class MatrixBlock extends MatrixValue implements Externalizable
{
if(sparse) //SPARSE
{
- if(sparseRows!=null)
+ if(sparseBlock!=null)
{
- for(int r=0; r<Math.min(rlen, sparseRows.length); r++)
- if(sparseRows[r]!=null)
+ for(int r=0; r<Math.min(rlen, sparseBlock.length); r++)
+ if(sparseBlock[r]!=null)
{
- int newSize=sparseRows[r].searchIndexesFirstGTE(clen-step);
+ int newSize=sparseBlock[r].searchIndexesFirstGTE(clen-step);
if(newSize>=0)
{
- this.nonZeros-=sparseRows[r].size()-newSize;
- sparseRows[r].truncate(newSize);
+ this.nonZeros-=sparseBlock[r].size()-newSize;
+ sparseBlock[r].truncate(newSize);
}
}
}
@@ -4746,14 +4745,14 @@ public class MatrixBlock extends MatrixValue implements Externalizable
}
int nzcount = 0;
- if(sparse && sparseRows!=null) //SPARSE
+ if(sparse && sparseBlock!=null) //SPARSE
{
- for(int r=0; r<Math.min(rlen, sparseRows.length); r++)
+ for(int r=0; r<Math.min(rlen, sparseBlock.length); r++)
{
- if(sparseRows[r]==null)
+ if(sparseBlock[r]==null)
continue;
- double[] values=sparseRows[r].getValueContainer();
- for(int i=0; i<sparseRows[r].size(); i++) {
+ double[] values=sparseBlock[r].getValueContainer();
+ for(int i=0; i<sparseBlock[r].size(); i++) {
op.fn.execute(cmobj, values[i]);
nzcount++;
}
@@ -4785,7 +4784,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
}
CM_COV_Object cmobj = new CM_COV_Object();
- if (sparse && sparseRows!=null) //SPARSE
+ if (sparse && sparseBlock!=null) //SPARSE
{
for(int i=0; i < rlen; i++)
op.fn.execute(cmobj, this.quickGetValue(i,0), weights.quickGetValue(i,0));
@@ -4824,7 +4823,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
}
CM_COV_Object covobj = new CM_COV_Object();
- if(sparse && sparseRows!=null) //SPARSE
+ if(sparse && sparseBlock!=null) //SPARSE
{
for(int i=0; i < rlen; i++ )
op.fn.execute(covobj, this.quickGetValue(i,0), that.quickGetValue(i,0));
@@ -4868,7 +4867,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
}
CM_COV_Object covobj = new CM_COV_Object();
- if(sparse && sparseRows!=null) //SPARSE
+ if(sparse && sparseBlock!=null) //SPARSE
{
for(int i=0; i < rlen; i++ )
op.fn.execute(covobj, this.quickGetValue(i,0), that.quickGetValue(i,0), weights.quickGetValue(i,0));
@@ -5427,8 +5426,8 @@ public class MatrixBlock extends MatrixValue implements Externalizable
if( pattern != 0d ) //SPARSE <- SPARSE (sparse-safe)
{
ret.allocateSparseRowsBlock();
- SparseRow[] a = sparseRows;
- SparseRow[] c = ret.sparseRows;
+ SparseRow[] a = sparseBlock;
+ SparseRow[] c = ret.sparseBlock;
for( int i=0; i<rlen; i++ )
{
@@ -5455,7 +5454,7 @@ public class MatrixBlock extends MatrixValue implements Externalizable
{
ret.sparse = false;
ret.allocateDenseBlock();
- SparseRow[] a = sparseRows;
+ SparseRow[] a = sparseBlock;
double[] c = ret.denseBlock;
//initialize with replacement (since all 0 values, see SPARSITY_TURN_POINT)
@@ -5655,8 +5654,8 @@ public class MatrixBlock extends MatrixValue implements Externalizable
if( this.isEmptyBlock(false) && that.isEmptyBlock(false) )
return;
- SparseRow[] a = this.sparseRows;
- SparseRow[] b = that.sparseRows;
+ SparseRow[] a = this.sparseBlock;
+ SparseRow[] b = that.sparseBlock;
for( int i=0; i<rlen; i++ )
{
SparseRow arow = a[i];
@@ -6144,15 +6143,15 @@ public class MatrixBlock extends MatrixValue implements Externalizable
if(sparse)
{
int len=0;
- if(sparseRows!=null)
- len = Math.min(rlen, sparseRows.length);
+ if(sparseBlock!=null)
+ len = Math.min(rlen, sparseBlock.length);
int i=0;
for(; i<len; i++)
{
sb.append("row +");
sb.append(i);
sb.append(": ");
- sb.append(sparseRows[i]);
+ sb.append(sparseBlock[i]);
sb.append("\n");
}
for(; i<rlen; i++)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteReducer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteReducer.java
index 73dae15..95afd32 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteReducer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteReducer.java
@@ -38,7 +38,6 @@ import org.apache.sysml.runtime.instructions.mr.CSVWriteInstruction;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.IJV;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
import org.apache.sysml.runtime.matrix.data.TaggedFirstSecondIndexes;
import org.apache.sysml.runtime.matrix.mapred.CSVWriteReducer.RowBlockForTextOutput;
import org.apache.sysml.runtime.matrix.mapred.CSVWriteReducer.RowBlockForTextOutput.Situation;
@@ -324,7 +323,7 @@ public class CSVWriteReducer extends ReduceBase implements Reducer<TaggedFirstSe
}
else if( _data.isInSparseFormat() ) //SPARSE BLOCK
{
- SparseRowsIterator iter = _data.getSparseRowsIterator();
+ Iterator<IJV> iter = _data.getSparseBlockIterator();
int j = -1;
while( iter.hasNext() )
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java
index 4a75e8c..4c50e20 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReblockBuffer.java
@@ -24,17 +24,16 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
+import java.util.Iterator;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.OutputCollector;
-
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.matrix.data.AdaptivePartialBlock;
import org.apache.sysml.runtime.matrix.data.IJV;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.data.PartialBlock;
-import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
import org.apache.sysml.runtime.matrix.data.TaggedAdaptivePartialBlock;
/**
@@ -106,7 +105,7 @@ public class ReblockBuffer
{
if( inBlk.isInSparseFormat() ) //SPARSE
{
- SparseRowsIterator iter = inBlk.getSparseRowsIterator();
+ Iterator<IJV> iter = inBlk.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV cell = iter.next();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
index 4bce27a..00df11e 100644
--- a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
@@ -22,11 +22,11 @@ package org.apache.sysml.runtime.util;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;
import org.apache.commons.math3.linear.Array2DRowRealMatrix;
-
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
@@ -43,7 +43,6 @@ import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
-import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
import org.apache.sysml.udf.Matrix;
@@ -318,7 +317,7 @@ public class DataConverter
{
if( mb.isInSparseFormat() )
{
- SparseRowsIterator iter = mb.getSparseRowsIterator();
+ Iterator<IJV> iter = mb.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV cell = iter.next();
@@ -351,7 +350,7 @@ public class DataConverter
{
if( mb.isInSparseFormat() )
{
- SparseRowsIterator iter = mb.getSparseRowsIterator();
+ Iterator<IJV> iter = mb.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV cell = iter.next();
@@ -386,7 +385,7 @@ public class DataConverter
{
if( mb.isInSparseFormat() )
{
- SparseRowsIterator iter = mb.getSparseRowsIterator();
+ Iterator<IJV> iter = mb.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV cell = iter.next();
@@ -420,7 +419,7 @@ public class DataConverter
{
if( mb.isInSparseFormat() )
{
- SparseRowsIterator iter = mb.getSparseRowsIterator();
+ Iterator<IJV> iter = mb.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV cell = iter.next();
@@ -430,7 +429,7 @@ public class DataConverter
else
{
//memcopy row major representation if at least 1 non-zero
- System.arraycopy(mb.getDenseArray(), 0, ret, 0, rows*cols);
+ System.arraycopy(mb.getDenseBlock(), 0, ret, 0, rows*cols);
}
}
@@ -451,7 +450,7 @@ public class DataConverter
if( mb.isInSparseFormat() )
{
- SparseRowsIterator iter = mb.getSparseRowsIterator();
+ Iterator<IJV> iter = mb.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV cell = iter.next();
@@ -640,7 +639,7 @@ public class DataConverter
//cache-friendly sequential read/append
if( !mb.isEmptyBlock(false) ) {
if( sparse ){ //SPARSE
- SparseRowsIterator iter = mb.getSparseRowsIterator();
+ Iterator<IJV> iter = mb.getSparseBlockIterator();
while( iter.hasNext() ) {
IJV cell = iter.next();
ret[cell.j].appendValue(cell.i, 0, cell.v);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/test/java/org/apache/sysml/test/utils/TestUtils.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/utils/TestUtils.java b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
index f768870..e6e7c12 100644
--- a/src/test/java/org/apache/sysml/test/utils/TestUtils.java
+++ b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
@@ -40,6 +40,7 @@ import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Locale;
import java.util.Random;
import java.util.StringTokenizer;
@@ -53,14 +54,12 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
-
import org.apache.sysml.runtime.io.IOUtilFunctions;
import org.apache.sysml.runtime.matrix.data.IJV;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixCell;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
-import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
import org.apache.sysml.test.integration.BinaryMatrixCharacteristics;
@@ -1965,7 +1964,7 @@ public class TestUtils
}
if (value.isInSparseFormat()) {
- SparseRowsIterator iter = value.getSparseRowsIterator();
+ Iterator<IJV> iter = value.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV cell = iter.next();
@@ -1974,7 +1973,7 @@ public class TestUtils
}
} else {
- double[] valuesInBlock = value.getDenseArray();
+ double[] valuesInBlock = value.getDenseBlock();
for (int i = 0; i < value.getNumRows(); i++) {
for (int j = 0; j < value.getNumColumns(); j++) {
valueMap.put(new MatrixIndexes(((indexes.getRowIndex() - 1) * rowsInBlock + i),
[3/7] incubator-systemml git commit: [SYSTEMML-378] New sparse block
abstraction
Posted by mb...@apache.org.
[SYSTEMML-378] New sparse block abstraction
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/7c06ef60
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/7c06ef60
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/7c06ef60
Branch: refs/heads/master
Commit: 7c06ef6068ee4fc3633d63041e4b5174d902d516
Parents: cf7d206
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Jan 17 23:00:41 2016 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sun Jan 17 23:12:18 2016 -0800
----------------------------------------------------------------------
.../sysml/runtime/matrix/data/SparseBlock.java | 350 +++++++++++++++++++
1 file changed, 350 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7c06ef60/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlock.java
new file mode 100644
index 0000000..478e900
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlock.java
@@ -0,0 +1,350 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.apache.sysml.runtime.matrix.data;
+
+import java.io.Serializable;
+import java.util.Iterator;
+
+/**
+ * This SparseBlock is an abstraction for different sparse matrix formats.
+ * Since the design is a tradeoff between performance and generality, we
+ * restrict this abstraction to row-major sparse representations for now.
+ * All sparse matrix block operations are supposed to be implemented
+ * against this abstraction in order to enable variability/extensibility.
+ *
+ * Example sparse format that can be implemented efficiently include
+ * CSR, MCSR, and - with performance drawbacks - COO.
+ *
+ */
+public abstract class SparseBlock implements Serializable
+{
+ private static final long serialVersionUID = -5008747088111141395L;
+
+ //internal configuration parameters for all sparse blocks
+ protected static final int INIT_CAPACITY = 4; //initial array capacity
+ protected static final double RESIZE_FACTOR1 = 2; //factor until reaching est nnz
+ protected static final double RESIZE_FACTOR2 = 1.1; //factor after reaching est nnz
+
+ public enum Type {
+ MCSR,
+ CSR,
+ COO,
+ }
+
+
+ ////////////////////////
+ //basic allocation
+
+ /**
+ * Allocate the underlying data structure holding non-zero values
+ * of row r if necessary.
+ *
+ * @param r
+ */
+ public abstract void allocate(int r);
+
+ ////////////////////////
+ //obtain basic meta data
+
+ /**
+ * Get the number of rows in the sparse block.
+ *
+ * @return
+ */
+ public abstract int numRows();
+
+ /**
+ * Indicates if the underlying implementation allows thread-safe row
+ * updates if concurrent threads update disjoint rows.
+ *
+ * @return
+ */
+ public abstract boolean isThreadSafe();
+
+ /**
+ * Get the number of non-zero values in the sparse block.
+ *
+ * @return
+ */
+ public abstract long size();
+
+ /**
+ * Get the number of non-zero values in row r.
+ *
+ * @param r row index starting at 0
+ * @return
+ */
+ public abstract int size(int r);
+
+ /**
+ * Get information if row r is empty, i.e., does not contain non-zero
+ * values. Equivalent to size(r)==0. Users should do this check if
+ * it is unknown if the underlying row data structure is allocated.
+ *
+ * @param r row index starting at 0
+ * @return
+ */
+ public abstract boolean isEmpty(int r);
+
+
+ ////////////////////////
+ //obtain indexes/values/positions
+
+ /**
+ * Get the sorted array of column indexes of all non-zero entries in
+ * row r. Note that - for flexibility of the implementing format - the
+ * returned array may be larger, where the range for row r is given by
+ * [pos(r),pos(r)+size(r)).
+ *
+ * @param r row index starting at 0
+ * @return
+ */
+ public abstract int[] indexes(int r);
+
+ /**
+ * Get the array of all non-zero entries in row r, sorted by their column
+ * indexes. Note that - for flexibility of the implementing format - the
+ * returned array may be larger, where the range for row r is given by
+ * [pos(r),pos(r)+size(r)).
+ *
+ * @param r row index starting at 0
+ * @return
+ */
+ public abstract double[] values(int r);
+
+ /**
+ * Get the starting position of row r in the indexes/values arrays returned
+ * by indexes(r) and values(r).
+ *
+ * @param r row index starting at 0
+ * @return
+ */
+ public abstract int pos(int r);
+
+
+ ////////////////////////
+ //update operations
+
+ /**
+ * Set the value of a matrix cell (r,c). This might update an existing
+ * non-zero value, insert a new non-zero value, or delete a non-zero value.
+ *
+ * @param r row index starting at 0
+ * @param c column index starting at 0
+ * @param v zero or non-zero value
+ * @return
+ */
+ public abstract boolean set(int r, int c, double v);
+
+ /**
+ * Append a value to the end of the physical representation. This should
+ * only be used for operations with sequential write pattern or if followed
+ * by a sort() operation. Note that this operation does not perform any
+ * matrix cell updates.
+ *
+ * @param r row index starting at 0
+ * @param c column index starting at 0
+ * @param v zero or non-zero value
+ */
+ public abstract void append(int r, int c, double v);
+
+ /**
+ * Sets a sorted array of non-zeros values into the column range [cl,cu)
+ * in row r. The passed value array may be larger and the relevant range
+ * is given by [vix,vix+len).
+ *
+ * @param r row index starting at 0
+ * @param cl lower column index starting at 0
+ * @param cu upper column index starting at 0
+ * @param v value array
+ * @param vix start index in value array
+ * @param vlen number of relevant values
+ */
+ public abstract void setIndexRange(int r, int cl, int cu, double[] v, int vix, int vlen);
+
+ /**
+ * Deletes all non-zero values of the given column range [cl,cu) in row r.
+ *
+ * @param r row index starting at 0
+ * @param cl lower column index starting at 0
+ * @param cu upper column index starting at 0
+ */
+ public abstract void deleteIndexRange(int r, int cl, int cu);
+
+ /**
+ * Sort all non-zero value/index pairs of the sparse block by row
+ * and column index.
+ */
+ public abstract void sort();
+
+ /**
+ * Sort all non-zero value/index pairs of row r column index.
+ *
+ * @param r row index starting at 0
+ */
+ public abstract void sort(int r);
+
+
+ ////////////////////////
+ //search operations
+
+ /**
+ * Get value of matrix cell (r,c). In case of non existing values
+ * this call returns 0.
+ *
+ * @param r row index starting at 0
+ * @param c column index starting at 0
+ * @return
+ */
+ public abstract double get(int r, int c);
+
+ /**
+ * Get position of first column index lower than or equal column c
+ * in row r. The position is relative to the indexes/values arrays
+ * returned by indexes(r) and values(r). If no such value exists,
+ * this call returns -1.
+ *
+ * @param r row index starting at 0
+ * @param c column index starting at 0
+ * @return
+ */
+ public abstract int posFIndexLTE(int r, int c);
+
+ /**
+ * Get position of first column index greater than or equal column c
+ * in row r. The position is relative to the indexes/values arrays
+ * returned by indexes(r) and values(r). If no such value exists,
+ * this call returns -1.
+ *
+ * @param r
+ * @param c
+ * @return
+ */
+ public abstract int posFIndexGTE(int r, int c);
+
+ /**
+ * Get position of first column index greater than column c in row r.
+ * The position is relative to the indexes/values arrays returned by
+ * indexes(r) and values(r). If no such value exists, this call
+ * returns -1.
+ *
+ * @param r
+ * @param c
+ * @return
+ */
+ public abstract int posFIndexGT(int r, int c);
+
+
+ ////////////////////////
+ //iterators
+
+ /**
+ * Get a non-zero iterator over the entire sparse block. Note that
+ * the returned IJV object is reused across next calls and should
+ * be directly consumed or deep copied.
+ *
+ * @return
+ */
+ public Iterator<IJV> getIterator() {
+ //default generic iterator, override if necessary
+ return new SparseBlockIterator(numRows());
+ }
+
+ /**
+ * Get a non-zero iterator over the subblock [rl, ru). Note that
+ * the returned IJV object is reused across next calls and should
+ * be directly consumed or deep copied.
+ *
+ * @param rl inclusive lower row index starting at 0
+ * @param ru exclusive upper row index starting at 0
+ * @return
+ */
+ public Iterator<IJV> getIterator(int rl, int ru) {
+ //default generic iterator, override if necessary
+ return new SparseBlockIterator(rl, Math.min(ru,numRows()));
+ }
+
+
+ /**
+ * Default sparse block iterator implemented against the sparse block
+ * api in an implementation-agnostic manner.
+ *
+ */
+ private class SparseBlockIterator implements Iterator<IJV>
+ {
+ private int _rlen = 0; //row upper
+ private int _curRow = -1; //current row
+ private int _curColIx = -1; //current col index pos
+ private int[] _curIndexes = null; //current col indexes
+ private double[] _curValues = null; //current col values
+ private boolean _noNext = false; //end indicator
+ private IJV retijv = new IJV(); //reuse output tuple
+
+ protected SparseBlockIterator(int ru) {
+ _rlen = ru;
+ _curRow = 0;
+ findNextNonZeroRow();
+ }
+
+ protected SparseBlockIterator(int rl, int ru) {
+ _rlen = ru;
+ _curRow = rl;
+ findNextNonZeroRow();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return !_noNext;
+ }
+
+ @Override
+ public IJV next( ) {
+ retijv.set(_curRow, _curIndexes[_curColIx], _curValues[_curColIx]);
+ if( ++_curColIx >= pos(_curRow)+size(_curRow) ) {
+ _curRow++;
+ findNextNonZeroRow();
+ }
+
+ return retijv;
+ }
+
+ @Override
+ public void remove() {
+ throw new RuntimeException("SparseBlockIterator is unsupported!");
+ }
+
+ /**
+ * Moves cursor to next non-zero value or indicates that no more
+ * values are available.
+ */
+ private void findNextNonZeroRow() {
+ while( _curRow<_rlen && isEmpty(_curRow))
+ _curRow++;
+ if(_curRow >= _rlen)
+ _noNext = true;
+ else {
+ _curColIx = pos(_curRow);
+ _curIndexes = indexes(_curRow);
+ _curValues = values(_curRow);
+ }
+ }
+ }
+}
[4/7] incubator-systemml git commit: [SYSTEMML-379] New Modified CSR
sparse matrix block
Posted by mb...@apache.org.
[SYSTEMML-379] New Modified CSR sparse matrix block
This changes also includes some minor extensions of the underlying
SparseRow (method dispatching and default constructor).
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/34e8221d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/34e8221d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/34e8221d
Branch: refs/heads/master
Commit: 34e8221d53a39798277aad7a759dabce35c98950
Parents: 7c06ef6
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Jan 17 23:04:38 2016 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sun Jan 17 23:12:26 2016 -0800
----------------------------------------------------------------------
.../runtime/matrix/data/SparseBlockMCSR.java | 179 +++++++++++++++++++
.../sysml/runtime/matrix/data/SparseRow.java | 15 +-
2 files changed, 193 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/34e8221d/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockMCSR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockMCSR.java b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockMCSR.java
new file mode 100644
index 0000000..4a19c68
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockMCSR.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.apache.sysml.runtime.matrix.data;
+
+/**
+ * SparseBlock implementation that realizes a 'modified compressed sparse row'
+ * representation, where each compressed row is stored as a separate SparseRow
+ * object which provides flexibility for unsorted row appends without the need
+ * for global reshifting of values/indexes but it incurs additional memory
+ * overhead per row for object/array headers per row which also slows down
+ * memory-bound operations due to higher memory bandwidth requirements.
+ *
+ */
+public class SparseBlockMCSR extends SparseBlock
+{
+ private static final long serialVersionUID = -4743624499258436199L;
+
+ private SparseRow[] _rows = null;
+
+ /**
+ * Copy constructor old sparse row representation.
+ */
+ public SparseBlockMCSR(SparseRow[] rows, boolean deep) {
+ if( deep ) {
+ _rows = new SparseRow[rows.length];
+ for( int i=0; i<_rows.length; i++ )
+ _rows[i] = new SparseRow(rows[i]);
+ }
+ else {
+ _rows = rows;
+ }
+ }
+
+ public SparseBlockMCSR(int rlen, int clen) {
+ _rows = new SparseRow[rlen];
+ }
+
+ ///////////////////
+ //SparseBlock implementation
+
+ @Override
+ public void allocate(int r) {
+ if( _rows[r] == null )
+ _rows[r] = new SparseRow();
+ }
+
+ @Override
+ public int numRows() {
+ return _rows.length;
+ }
+
+ @Override
+ public boolean isThreadSafe() {
+ return true;
+ }
+
+ @Override
+ public long size() {
+ //recompute non-zeros to avoid redundant maintenance
+ long nnz = 0;
+ for( SparseRow row : _rows )
+ if( row != null )
+ nnz += row.size();
+ return nnz;
+ }
+
+ @Override
+ public int size(int r) {
+ //prior check with isEmpty(r) expected
+ return _rows[r].size();
+ }
+
+ @Override
+ public boolean isEmpty(int r) {
+ return (_rows[r]==null || _rows[r].isEmpty());
+ }
+
+ @Override
+ public int[] indexes(int r) {
+ //prior check with isEmpty(r) expected
+ return _rows[r].indexes();
+ }
+
+ @Override
+ public double[] values(int r) {
+ //prior check with isEmpty(r) expected
+ return _rows[r].values();
+ }
+
+ @Override
+ public int pos(int r) {
+ //arrays per row (always start 0)
+ return 0;
+ }
+
+ @Override
+ public boolean set(int r, int c, double v) {
+ if( _rows[r] == null )
+ _rows[r] = new SparseRow();
+ return _rows[r].set(c, v);
+ }
+
+ @Override
+ public void append(int r, int c, double v) {
+ if( _rows[r] == null )
+ _rows[r] = new SparseRow();
+ _rows[r].append(c, v);
+ }
+
+ @Override
+ public void setIndexRange(int r, int cl, int cu, double[] v, int vix, int len) {
+ if( _rows[r] == null )
+ _rows[r] = new SparseRow();
+ //different sparse row semantics: upper bound inclusive
+ _rows[r].setIndexRange(cl, cu-1, v, vix, len);
+ }
+
+ @Override
+ public void deleteIndexRange(int r, int cl, int cu) {
+ //prior check with isEmpty(r) expected
+ //different sparse row semantics: upper bound inclusive
+ _rows[r].deleteIndexRange(cl, cu-1);
+ }
+
+ @Override
+ public void sort() {
+ for( SparseRow row : _rows )
+ if( row != null && !row.isEmpty() )
+ row.sort();
+ }
+
+ @Override
+ public void sort(int r) {
+ //prior check with isEmpty(r) expected
+ _rows[r].sort();
+ }
+
+ @Override
+ public double get(int r, int c) {
+ //prior check with isEmpty(r) expected
+ return _rows[r].get(c);
+ }
+
+ @Override
+ public int posFIndexLTE(int r, int c) {
+ //prior check with isEmpty(r) expected
+ return _rows[r].searchIndexesFirstLTE(c);
+ }
+
+ @Override
+ public int posFIndexGTE(int r, int c) {
+ //prior check with isEmpty(r) expected
+ return _rows[r].searchIndexesFirstGTE(c);
+ }
+
+ @Override
+ public int posFIndexGT(int r, int c) {
+ //prior check with isEmpty(r) expected
+ return _rows[r].searchIndexesFirstGT(c);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/34e8221d/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRow.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRow.java b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRow.java
index b030c11..31d26ad 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRow.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRow.java
@@ -27,7 +27,6 @@ import org.apache.sysml.runtime.util.SortUtils;
public class SparseRow implements Serializable
{
-
private static final long serialVersionUID = 5806895317005796456L;
//initial capacity of any created sparse row
@@ -40,6 +39,10 @@ public class SparseRow implements Serializable
private double[] values = null;
private int[] indexes = null;
+ public SparseRow() {
+ this(initialCapacity);
+ }
+
public SparseRow(int capacity)
{
estimatedNzs = capacity;
@@ -90,11 +93,21 @@ public class SparseRow implements Serializable
return (size == 0);
}
+ public double[] values()
+ {
+ return getValueContainer();
+ }
+
public double[] getValueContainer()
{
return values;
}
+ public int[] indexes()
+ {
+ return getIndexContainer();
+ }
+
public int[] getIndexContainer()
{
return indexes;
[6/7] incubator-systemml git commit: [SYSTEMML-471] New COO sparse
matrix block, incl custom iterator
Posted by mb...@apache.org.
[SYSTEMML-471] New COO sparse matrix block, incl custom iterator
This change also includes an extension of our core SortUtils in order to
enable sorting index/index/value arrays by the first index array.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/7ce3707a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/7ce3707a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/7ce3707a
Branch: refs/heads/master
Commit: 7ce3707a3c200fb0b588c3faea8c8100d260e3ab
Parents: 6aa3437
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Jan 17 23:07:42 2016 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sun Jan 17 23:12:42 2016 -0800
----------------------------------------------------------------------
.../runtime/matrix/data/SparseBlockCOO.java | 496 +++++++++++++++++++
.../apache/sysml/runtime/util/SortUtils.java | 127 +++++
2 files changed, 623 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7ce3707a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockCOO.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockCOO.java b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockCOO.java
new file mode 100644
index 0000000..d2c6731
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockCOO.java
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.apache.sysml.runtime.matrix.data;
+
+import java.util.Arrays;
+import java.util.Iterator;
+
+import org.apache.sysml.runtime.util.SortUtils;
+
+/**
+ * SparseBlock implementation that realizes a traditional 'coordinate matrix'
+ * representation, where the entire sparse block is stored as triples in three arrays:
+ * row indexes, column indexes, and values, where row indexes and colunm indexes are
+ * sorted in order to allow binary search. This format is very memory efficient for
+ * ultra-sparse matrices, allows fast incremental construction but has performance
+ * drawbacks for row-major access through our sparse block abstraction since there
+ * is no constant-time random access to individual rows. Similar to CSR, the nnz
+ * is limited to Integer.MAX_VALUE.
+ *
+ */
+public class SparseBlockCOO extends SparseBlock
+{
+ private static final long serialVersionUID = 7223478015917668745L;
+
+ private int _rlen = -1;
+ private int[] _rindexes = null; //row index array (size: >=nnz)
+ private int[] _cindexes = null; //column index array (size: >=nnz)
+ private double[] _values = null; //value array (size: >=nnz)
+ private int _size = 0; //actual number of nnz
+
+ public SparseBlockCOO(int rlen) {
+ this(rlen, INIT_CAPACITY);
+ }
+
+ public SparseBlockCOO(int rlen, int capacity) {
+ _rlen = rlen;
+ _rindexes = new int[capacity];
+ _cindexes = new int[capacity];
+ _values = new double[capacity];
+ _size = 0;
+ }
+
+ /**
+ * Copy constructor old sparse row representation.
+ */
+ public SparseBlockCOO(SparseRow[] rows, int nnz)
+ {
+ _rlen = rows.length;
+
+ _rindexes = new int[nnz];
+ _cindexes = new int[nnz];
+ _values = new double[nnz];
+ _size = nnz;
+
+ for( int i=0, pos=0; i<_rlen; i++ ) {
+ int alen = rows[i].size();
+ int[] aix = rows[i].getIndexContainer();
+ double[] avals = rows[i].getValueContainer();
+ for( int j=0; j<alen; j++ ) {
+ _rindexes[pos] = i;
+ _cindexes[pos] = aix[j];
+ _values[pos] = avals[j];
+ pos++;
+ }
+ }
+ }
+
+ @Override
+ public void allocate(int r) {
+ //do nothing everything preallocated
+ }
+
+ @Override
+ public int numRows() {
+ return _rlen;
+ }
+
+ @Override
+ public boolean isThreadSafe() {
+ return false;
+ }
+
+ @Override
+ public long size() {
+ return _size;
+ }
+
+ @Override
+ public int size(int r) {
+ int pos = pos(r);
+ if( _rindexes[pos]!=r )
+ return 0;
+
+ //count number of equal row indexes
+ double rix0 = _rindexes[pos];
+ int cnt = 0;
+ while( pos<_size && rix0 == _rindexes[pos++] )
+ cnt ++;
+ return cnt;
+ }
+
+ @Override
+ public boolean isEmpty(int r) {
+ int pos = pos(r);
+ return (pos>=_size||_rindexes[pos]!=r);
+ }
+
+ @Override
+ public int[] indexes(int r) {
+ return _cindexes;
+ }
+
+ @Override
+ public double[] values(int r) {
+ return _values;
+ }
+
+ @Override
+ public int pos(int r) {
+ //find row index partition
+ int index = Arrays.binarySearch(_rindexes, 0, _size, r);
+ if( index < 0 )
+ return Math.abs(index+1);
+
+ //scan to begin of row index partition
+ while( index>0 && _rindexes[index-1]==r )
+ index--;
+ return index;
+ }
+
+ @Override
+ public boolean set(int r, int c, double v) {
+ int pos = pos(r);
+ int len = size(r);
+
+ //search for existing col index
+ int index = Arrays.binarySearch(_cindexes, pos, pos+len, c);
+
+ if( index >= 0 ) {
+ //delete/overwrite existing value (on value delete, we shift
+ //left for (1) correct nnz maintenance, and (2) smaller size)
+ if( v == 0 ) {
+ shiftLeftAndDelete(index);
+ return true; // nnz--
+ }
+ else {
+ _values[index] = v;
+ return false;
+ }
+ }
+
+ //early abort on zero (if no overwrite)
+ if( v==0 ) return false;
+
+ //insert new index-value pair
+ index = Math.abs( index+1 );
+ if( _size==_values.length )
+ resizeAndInsert(index, r, c, v);
+ else
+ shiftRightAndInsert(index, r, c, v);
+ return true; // nnz++
+ }
+
+ @Override
+ public void append(int r, int c, double v) {
+ //early abort on zero
+ if( v==0 ) return;
+
+ if( _size==_values.length )
+ resize();
+ insert(_size, r, c, v);
+ }
+
+ @Override
+ public void setIndexRange(int r, int cl, int cu, double[] v, int vix, int vlen) {
+ //delete existing values in range if necessary
+ deleteIndexRange(r, cl, cu);
+
+ //determine input nnz
+ int lnnz = 0;
+ for( int i=vix; i<vix+vlen; i++ )
+ lnnz += ( v[i] != 0 ) ? 1 : 0;
+
+ //prepare free space (allocate and shift)
+ int lsize = _size+lnnz;
+ if( _values.length < lsize )
+ resize(lsize);
+ int index = posFIndexGT(r, cl);
+ shiftRightByN((index>0)?index:pos(r+1), lnnz);
+
+ //insert values
+ for( int i=vix; i<vix+vlen; i++ )
+ if( v[i] != 0 ) {
+ _rindexes[ index ] = r;
+ _cindexes[ index ] = cl+i-vix;
+ _values[ index ] = v[i];
+ index++;
+ }
+ }
+
+ @Override
+ public void deleteIndexRange(int r, int cl, int cu) {
+ int start = posFIndexGTE(r,cl);
+ if( start < 0 ) //nothing to delete
+ return;
+
+ int len = size(r);
+ int end = posFIndexGTE(r, cu);
+ if( end < 0 ) //delete all remaining
+ end = start+len;
+
+ //overlapping array copy (shift rhs values left)
+ System.arraycopy(_rindexes, end, _rindexes, start, _size-end);
+ System.arraycopy(_cindexes, end, _cindexes, start, _size-end);
+ System.arraycopy(_values, end, _values, start, _size-end);
+ _size -= (end-start);
+ }
+
+ @Override
+ public void sort() {
+ //sort all three indexes by _rindexes
+ SortUtils.sortByIndex(0, _size, _rindexes, _cindexes, _values);
+
+ //sort _cindexes/_values by _cindexes per row partition
+ int index = 0;
+ while( index < _size ){
+ int r = _rindexes[index];
+ int len = 0;
+ while( r == _rindexes[index] ) {
+ len ++; index ++;
+ }
+ SortUtils.sortByIndex(index-len, index, _cindexes, _values);
+ }
+ }
+
+ @Override
+ public void sort(int r) {
+ int pos = pos(r);
+ int len = size(r);
+
+ if( len<=100 || !SortUtils.isSorted(pos, pos+len, _cindexes) )
+ SortUtils.sortByIndex(pos, pos+len, _cindexes, _values);
+ }
+
+ @Override
+ public double get(int r, int c) {
+ int pos = pos(r);
+ int len = size(r);
+
+ //search for existing col index in [pos,pos+len)
+ int index = Arrays.binarySearch(_cindexes, pos, pos+len, c);
+ return (index >= 0) ? _values[index] : 0;
+ }
+
+ @Override
+ public int posFIndexLTE(int r, int c) {
+ int pos = pos(r);
+ int len = size(r);
+
+ //search for existing col index in [pos,pos+len)
+ int index = Arrays.binarySearch(_cindexes, pos, pos+len, c);
+ if( index >= 0 )
+ return (index < pos+len) ? index : -1;
+
+ //search lt col index (see binary search)
+ index = Math.abs( index+1 );
+ return (index-1 >= pos) ? index-1 : -1;
+ }
+
+ @Override
+ public int posFIndexGTE(int r, int c) {
+ int pos = pos(r);
+ int len = size(r);
+
+ //search for existing col index
+ int index = Arrays.binarySearch(_cindexes, pos, pos+len, c);
+ if( index >= 0 )
+ return (index < pos+len) ? index : -1;
+
+ //search gt col index (see binary search)
+ index = Math.abs( index+1 );
+ return (index < pos+len) ? index : -1;
+ }
+
+ @Override
+ public int posFIndexGT(int r, int c) {
+ int pos = pos(r);
+ int len = size(r);
+
+ //search for existing col index
+ int index = Arrays.binarySearch(_cindexes, pos, pos+len, c);
+ if( index >= 0 )
+ return (index+1 < pos+len) ? index+1 : -1;
+
+ //search gt col index (see binary search)
+ index = Math.abs( index+1 );
+ return (index < pos+len) ? index : -1;
+ }
+
+ @Override
+ public Iterator<IJV> getIterator() {
+ return new SparseBlockCOOIterator(0, _size);
+ }
+
+ @Override
+ public Iterator<IJV> getIterator(int rl, int ru) {
+ return new SparseBlockCOOIterator(pos(rl), pos(ru));
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("SparseBlockCOO: rlen=");
+ sb.append(_rlen);
+ sb.append(", nnz=");
+ sb.append(_size);
+ sb.append("\n");
+ for( int i=0; i<_size; i++ ) {
+ sb.append(_rindexes[i]);
+ sb.append(",");
+ sb.append(_cindexes[i]);
+ sb.append(":");
+ sb.append(_values[i]);
+ sb.append("\n");
+ }
+ return sb.toString();
+ }
+
+ ///////////////////////////
+ // private helper methods
+
+ /**
+ *
+ */
+ private void resize() {
+ //compute new size
+ double tmpCap = _values.length * RESIZE_FACTOR1;
+ int newCap = (int)Math.min(tmpCap, Integer.MAX_VALUE);
+
+ resize(newCap);
+ }
+
+ /**
+ *
+ * @param capacity
+ */
+ private void resize(int capacity) {
+ //reallocate arrays and copy old values
+ _rindexes = Arrays.copyOf(_rindexes, capacity);
+ _cindexes = Arrays.copyOf(_cindexes, capacity);
+ _values = Arrays.copyOf(_values, capacity);
+ }
+
+ /**
+ *
+ * @param ix
+ * @param r
+ * @param c
+ * @param v
+ */
+ private void resizeAndInsert(int ix, int r, int c, double v) {
+ //compute new size
+ double tmpCap = _values.length * RESIZE_FACTOR1;
+ int newCap = (int)Math.min(tmpCap, Integer.MAX_VALUE);
+
+ int[] oldrindexes = _rindexes;
+ int[] oldcindexes = _cindexes;
+ double[] oldvalues = _values;
+ _rindexes = new int[newCap];
+ _cindexes = new int[newCap];
+ _values = new double[newCap];
+
+ //copy lhs values to new array
+ System.arraycopy(oldrindexes, 0, _rindexes, 0, ix);
+ System.arraycopy(oldcindexes, 0, _cindexes, 0, ix);
+ System.arraycopy(oldvalues, 0, _values, 0, ix);
+
+ //copy rhs values to new array
+ System.arraycopy(oldrindexes, ix, _rindexes, ix+1, _size-ix);
+ System.arraycopy(oldcindexes, ix, _cindexes, ix+1, _size-ix);
+ System.arraycopy(oldvalues, ix, _values, ix+1, _size-ix);
+
+ //insert new value
+ insert(ix, r, c, v);
+ }
+
+ /**
+ *
+ * @param ix
+ * @param r
+ * @param c
+ * @param v
+ */
+ private void shiftRightAndInsert(int ix, int r, int c, double v) {
+ //overlapping array copy (shift rhs values right by 1)
+ System.arraycopy(_rindexes, ix, _rindexes, ix+1, _size-ix);
+ System.arraycopy(_cindexes, ix, _cindexes, ix+1, _size-ix);
+ System.arraycopy(_values, ix, _values, ix+1, _size-ix);
+
+ //insert new value
+ insert(ix, r, c, v);
+ }
+
+ /**
+ *
+ * @param index
+ */
+ private void shiftLeftAndDelete(int ix)
+ {
+ //overlapping array copy (shift rhs values left by 1)
+ System.arraycopy(_rindexes, ix+1, _rindexes, ix, _size-ix-1);
+ System.arraycopy(_cindexes, ix+1, _cindexes, ix, _size-ix-1);
+ System.arraycopy(_values, ix+1, _values, ix, _size-ix-1);
+ _size--;
+ }
+
+ /**
+ *
+ * @param ix
+ * @param n
+ */
+ private void shiftRightByN(int ix, int n)
+ {
+ //overlapping array copy (shift rhs values right by 1)
+ System.arraycopy(_rindexes, ix, _rindexes, ix+n, _size-ix);
+ System.arraycopy(_cindexes, ix, _cindexes, ix+n, _size-ix);
+ System.arraycopy(_values, ix, _values, ix+n, _size-ix);
+ _size += n;
+ }
+
+ /**
+ *
+ * @param ix
+ * @param r
+ * @param c
+ * @param v
+ */
+ private void insert(int ix, int r, int c, double v) {
+ _rindexes[ix] = r;
+ _cindexes[ix] = c;
+ _values[ix] = v;
+ _size++;
+ }
+
+ /**
+ * Custom sparse block COO iterator implemented against the
+ * SparseBlockCOO data structure in order to avoid unnecessary
+ * binary search for row locations and lengths.
+ *
+ */
+ private class SparseBlockCOOIterator implements Iterator<IJV>
+ {
+ private int _pos = 0; //current nnz position
+ private int _len = 0; //upper nnz position (exclusive)
+ private IJV retijv = new IJV(); //reuse output tuple
+
+ protected SparseBlockCOOIterator(int posrl, int posru) {
+ _pos = posrl;
+ _len = posru;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return _pos<_len;
+ }
+
+ @Override
+ public IJV next( ) {
+ retijv.set(_rindexes[_pos], _cindexes[_pos], _values[_pos++]);
+ return retijv;
+ }
+
+ @Override
+ public void remove() {
+ throw new RuntimeException("SparseBlockCOOIterator is unsupported!");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7ce3707a/src/main/java/org/apache/sysml/runtime/util/SortUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/SortUtils.java b/src/main/java/org/apache/sysml/runtime/util/SortUtils.java
index f29033c..a0cc397 100644
--- a/src/main/java/org/apache/sysml/runtime/util/SortUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/util/SortUtils.java
@@ -175,6 +175,133 @@ public class SortUtils
}
}
+
+ /**
+ * In-place sort of three arrays, only first indexes is used for comparison and second
+ * indexes as well as values of same position are sorted accordingly.
+ *
+ * @param start
+ * @param end
+ * @param indexes
+ * @param values
+ */
+ public static void sortByIndex(int start, int end, int[] indexes, int[] indexes2, double[] values)
+ {
+ int tempIx;
+ int tempIx2;
+ double tempVal;
+
+ int length = end - start;
+ if (length < 7) {
+ for (int i = start + 1; i < end; i++) {
+ for (int j = i; j > start && indexes[j - 1] > indexes[j]; j--) {
+ tempIx = indexes[j];
+ indexes[j] = indexes[j - 1];
+ indexes[j - 1] = tempIx;
+ tempIx2 = indexes2[j];
+ indexes2[j] = indexes2[j - 1];
+ indexes2[j - 1] = tempIx2;
+ tempVal = values[j];
+ values[j] = values[j - 1];
+ values[j - 1] = tempVal;
+ }
+ }
+ return;
+ }
+ int middle = (start + end) / 2;
+ if (length > 7) {
+ int bottom = start;
+ int top = end - 1;
+ if (length > 40) {
+ length /= 8;
+ bottom = med3(indexes, bottom, bottom + length, bottom
+ + (2 * length));
+ middle = med3(indexes, middle - length, middle, middle + length);
+ top = med3(indexes, top - (2 * length), top - length, top);
+ }
+ middle = med3(indexes, bottom, middle, top);
+ }
+ int partionValue = indexes[middle];
+ int a, b, c, d;
+ a = b = start;
+ c = d = end - 1;
+ while (true) {
+ while (b <= c && indexes[b] <= partionValue) {
+ if (indexes[b] == partionValue) {
+ tempIx = indexes[a];
+ indexes[a] = indexes[b];
+ indexes[b] = tempIx;
+ tempIx2 = indexes2[a];
+ indexes2[a] = indexes2[b];
+ indexes2[b] = tempIx2;
+ tempVal = values[a];
+ values[a++] = values[b];
+ values[b] = tempVal;
+ }
+ b++;
+ }
+ while (c >= b && indexes[c] >= partionValue) {
+ if (indexes[c] == partionValue) {
+ tempIx = indexes[c];
+ indexes[c] = indexes[d];
+ indexes[d] = tempIx;
+ tempIx2 = indexes2[c];
+ indexes2[c] = indexes2[d];
+ indexes2[d] = tempIx2;
+ tempVal = values[c];
+ values[c] = values[d];
+ values[d--] = tempVal;
+ }
+ c--;
+ }
+ if (b > c) {
+ break;
+ }
+ tempIx = indexes[b];
+ indexes[b] = indexes[c];
+ indexes[c] = tempIx;
+ tempIx2 = indexes2[b];
+ indexes2[b] = indexes2[c];
+ indexes2[c] = tempIx2;
+ tempVal = values[b];
+ values[b++] = values[c];
+ values[c--] = tempVal;
+ }
+ length = a - start < b - a ? a - start : b - a;
+ int l = start;
+ int h = b - length;
+ while (length-- > 0) {
+ tempIx = indexes[l];
+ indexes[l] = indexes[h];
+ indexes[h] = tempIx;
+ tempIx2 = indexes2[l];
+ indexes2[l] = indexes2[h];
+ indexes2[h] = tempIx2;
+ tempVal = values[l];
+ values[l++] = values[h];
+ values[h++] = tempVal;
+ }
+ length = d - c < end - 1 - d ? d - c : end - 1 - d;
+ l = b;
+ h = end - length;
+ while (length-- > 0) {
+ tempIx = indexes[l];
+ indexes[l] = indexes[h];
+ indexes[h] = tempIx;
+ tempIx2 = indexes2[l];
+ indexes2[l] = indexes2[h];
+ indexes2[h] = tempIx2;
+ tempVal = values[l];
+ values[l++] = values[h];
+ values[h++] = tempVal;
+ }
+ if ((length = b - a) > 0) {
+ sortByIndex(start, start + length, indexes, indexes2, values);
+ }
+ if ((length = d - c) > 0) {
+ sortByIndex(end - length, end, indexes, indexes2, values);
+ }
+ }
/**
*
[2/7] incubator-systemml git commit: [SYSTEMML-382] Refactoring
matrix block (prep sparse block integration)
Posted by mb...@apache.org.
[SYSTEMML-382] Refactoring matrix block (prep sparse block integration)
This changes isolates mechanic refactorings due to renaming and
signature changes from the upcoming actual runtime integration.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/cf7d206c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/cf7d206c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/cf7d206c
Branch: refs/heads/master
Commit: cf7d206c64d7ccf85b5864e7d35d7c1e3bfedfce
Parents: c65e34e
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Jan 17 22:36:06 2016 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sun Jan 17 23:12:10 2016 -0800
----------------------------------------------------------------------
.../parfor/DataPartitionerLocal.java | 5 +-
.../parfor/DataPartitionerRemoteMapper.java | 15 +-
.../parfor/ResultMergeLocalFile.java | 7 +-
.../spark/functions/GetMLLibBlocks.java | 10 +-
.../spark/utils/RDDConverterUtils.java | 2 +-
.../sysml/runtime/io/WriterBinaryCell.java | 5 +-
.../sysml/runtime/io/WriterMatrixMarket.java | 5 +-
.../runtime/io/WriterMatrixMarketParallel.java | 5 +-
.../apache/sysml/runtime/io/WriterTextCSV.java | 2 +-
.../sysml/runtime/io/WriterTextCSVParallel.java | 2 +-
.../apache/sysml/runtime/io/WriterTextCell.java | 5 +-
.../runtime/io/WriterTextCellParallel.java | 5 +-
.../data/BinaryBlockToBinaryCellConverter.java | 10 +-
.../data/BinaryBlockToTextCellConverter.java | 14 +-
.../sysml/runtime/matrix/data/LibMatrixAgg.java | 56 +--
.../runtime/matrix/data/LibMatrixBincell.java | 92 ++--
.../runtime/matrix/data/LibMatrixDatagen.java | 4 +-
.../runtime/matrix/data/LibMatrixMult.java | 100 ++---
.../runtime/matrix/data/LibMatrixOuterAgg.java | 12 +-
.../runtime/matrix/data/LibMatrixReorg.java | 52 +--
.../sysml/runtime/matrix/data/MatrixBlock.java | 441 +++++++++----------
.../runtime/matrix/mapred/CSVWriteReducer.java | 3 +-
.../runtime/matrix/mapred/ReblockBuffer.java | 5 +-
.../sysml/runtime/util/DataConverter.java | 17 +-
.../org/apache/sysml/test/utils/TestUtils.java | 7 +-
25 files changed, 435 insertions(+), 446 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java
index 162330c..421ed86 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java
@@ -24,6 +24,7 @@ import java.io.File;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.LinkedList;
import java.util.Map.Entry;
@@ -38,7 +39,6 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
-
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.DMLUnsupportedOperationException;
@@ -54,7 +54,6 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixCell;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
-import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
import org.apache.sysml.runtime.util.FastStringTokenizer;
import org.apache.sysml.runtime.util.LocalFileUtils;
@@ -476,7 +475,7 @@ public class DataPartitionerLocal extends DataPartitioner
boolean sparse = value.isInSparseFormat();
if( sparse ) //SPARSE
{
- SparseRowsIterator iter = value.getSparseRowsIterator();
+ Iterator<IJV> iter = value.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV lcell = iter.next();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java
index 491dfc2..440a4cb 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMapper.java
@@ -20,6 +20,7 @@
package org.apache.sysml.runtime.controlprogram.parfor;
import java.io.IOException;
+import java.util.Iterator;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Writable;
@@ -27,7 +28,6 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
-
import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
import org.apache.sysml.runtime.controlprogram.parfor.util.PairWritableBlock;
import org.apache.sysml.runtime.controlprogram.parfor.util.PairWritableCell;
@@ -37,7 +37,6 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixCell;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
-import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
import org.apache.sysml.runtime.matrix.mapred.MRJobConfiguration;
import org.apache.sysml.runtime.util.FastStringTokenizer;
import org.apache.sysml.runtime.util.MapReduceTool;
@@ -466,7 +465,7 @@ public class DataPartitionerRemoteMapper
case ROW_WISE:
if( sparse )
{
- SparseRowsIterator iter = value2.getSparseRowsIterator();
+ Iterator<IJV> iter = value2.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV lcell = iter.next();
@@ -500,7 +499,7 @@ public class DataPartitionerRemoteMapper
longKey.set((row_offset/_brlen+1));
if( sparse )
{
- SparseRowsIterator iter = value2.getSparseRowsIterator();
+ Iterator<IJV> iter = value2.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV lcell = iter.next();
@@ -534,7 +533,7 @@ public class DataPartitionerRemoteMapper
rowBlockIndex = ((row_offset%_n)/_brlen)+1;
if( sparse )
{
- SparseRowsIterator iter = value2.getSparseRowsIterator();
+ Iterator<IJV> iter = value2.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV lcell = iter.next();
@@ -563,7 +562,7 @@ public class DataPartitionerRemoteMapper
case COLUMN_WISE:
if( sparse )
{
- SparseRowsIterator iter = value2.getSparseRowsIterator();
+ Iterator<IJV> iter = value2.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV lcell = iter.next();
@@ -597,7 +596,7 @@ public class DataPartitionerRemoteMapper
longKey.set(col_offset/_bclen+1);
if( sparse )
{
- SparseRowsIterator iter = value2.getSparseRowsIterator();
+ Iterator<IJV> iter = value2.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV lcell = iter.next();
@@ -631,7 +630,7 @@ public class DataPartitionerRemoteMapper
colBlockIndex = ((col_offset%_n)/_bclen)+1;
if( sparse )
{
- SparseRowsIterator iter = value2.getSparseRowsIterator();
+ Iterator<IJV> iter = value2.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV lcell = iter.next();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
index 5ef713b..33ee581 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.LinkedList;
import java.util.Map.Entry;
@@ -39,7 +40,6 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
-
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
@@ -58,7 +58,6 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixCell;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
-import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
import org.apache.sysml.runtime.util.DataConverter;
import org.apache.sysml.runtime.util.FastStringTokenizer;
import org.apache.sysml.runtime.util.LocalFileUtils;
@@ -955,7 +954,7 @@ public class ResultMergeLocalFile extends ResultMerge
{
if( mb.isInSparseFormat() )
{
- SparseRowsIterator iter = mb.getSparseRowsIterator();
+ Iterator<IJV> iter = mb.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV lcell = iter.next();
@@ -1105,7 +1104,7 @@ public class ResultMergeLocalFile extends ResultMerge
{
if( mb.isInSparseFormat() )
{
- SparseRowsIterator iter = mb.getSparseRowsIterator();
+ Iterator<IJV> iter = mb.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV lcell = iter.next();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLLibBlocks.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLLibBlocks.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLLibBlocks.java
index 768a0eb..9d78650 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLLibBlocks.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMLLibBlocks.java
@@ -20,6 +20,7 @@
package org.apache.sysml.runtime.instructions.spark.functions;
import java.util.ArrayList;
+import java.util.Iterator;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.mllib.linalg.DenseMatrix;
@@ -31,7 +32,6 @@ import scala.Tuple2;
import org.apache.sysml.runtime.matrix.data.IJV;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
-import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
import org.apache.sysml.runtime.util.UtilFunctions;
public class GetMLLibBlocks implements PairFunction<Tuple2<MatrixIndexes,MatrixBlock>, Tuple2<Object, Object>, Matrix> {
@@ -51,7 +51,7 @@ public class GetMLLibBlocks implements PairFunction<Tuple2<MatrixIndexes,MatrixB
return (int) blk.getNonZeros();
}
else if(blk.isInSparseFormat()) {
- SparseRowsIterator iter = blk.getSparseRowsIterator();
+ Iterator<IJV> iter = blk.getSparseBlockIterator();
int nnz = 0;
while( iter.hasNext() ) {
nnz++;
@@ -59,7 +59,7 @@ public class GetMLLibBlocks implements PairFunction<Tuple2<MatrixIndexes,MatrixB
return nnz;
}
else {
- return blk.getDenseArray().length;
+ return blk.getDenseBlock().length;
}
}
@@ -88,7 +88,7 @@ public class GetMLLibBlocks implements PairFunction<Tuple2<MatrixIndexes,MatrixB
// ------------------------------------------------------------------
if(blk.isInSparseFormat()) {
- SparseRowsIterator iter = blk.getSparseRowsIterator();
+ Iterator<IJV> iter = blk.getSparseBlockIterator();
int nnz = getNNZ(blk);
double [] values = new double[nnz];
int [] rowIndices = new int[nnz];
@@ -115,7 +115,7 @@ public class GetMLLibBlocks implements PairFunction<Tuple2<MatrixIndexes,MatrixB
mllibBlock = new SparseMatrix(lrlen, lclen, colPtrs, rowIndices, values);
}
else {
- mllibBlock = new DenseMatrix(lrlen, lclen, blk.getDenseArray());
+ mllibBlock = new DenseMatrix(lrlen, lclen, blk.getDenseBlock());
}
return new Tuple2<Tuple2<Object,Object>, Matrix>(new Tuple2<Object,Object>(blockRowIndex, blockColIndex), mllibBlock);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java
index 9681803..656a618 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java
@@ -280,7 +280,7 @@ public class RDDConverterUtils
}
else if( tmp.isInSparseFormat() ) //SPARSE ROW
{
- SparseRow row = tmp.getSparseRows()[0];
+ SparseRow row = tmp.getSparseBlock()[0];
int rlen = row.size();
int[] rix = row.getIndexContainer();
double[] rvals = row.getValueContainer();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/io/WriterBinaryCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryCell.java b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryCell.java
index 4b6bc5b..7f5d4b2 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryCell.java
@@ -20,12 +20,12 @@
package org.apache.sysml.runtime.io;
import java.io.IOException;
+import java.util.Iterator;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.mapred.JobConf;
-
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.DMLUnsupportedOperationException;
@@ -33,7 +33,6 @@ import org.apache.sysml.runtime.matrix.data.IJV;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixCell;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
-import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
import org.apache.sysml.runtime.util.MapReduceTool;
public class WriterBinaryCell extends MatrixWriter
@@ -110,7 +109,7 @@ public class WriterBinaryCell extends MatrixWriter
if( sparse ) //SPARSE
{
- SparseRowsIterator iter = src.getSparseRowsIterator();
+ Iterator<IJV> iter = src.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV lcell = iter.next();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java
index 0f9bb06..3615573 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
+import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -32,13 +33,11 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapred.JobConf;
-
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.DMLUnsupportedOperationException;
import org.apache.sysml.runtime.matrix.data.IJV;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
import org.apache.sysml.runtime.util.MapReduceTool;
/**
@@ -121,7 +120,7 @@ public class WriterMatrixMarket extends MatrixWriter
// output matrix cell
if( sparse ) //SPARSE
{
- SparseRowsIterator iter = src.getSparseRowsIterator();
+ Iterator<IJV> iter = src.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV cell = iter.next();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java
index 23c2e51..eae4c46 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java
@@ -23,6 +23,7 @@ import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
@@ -32,14 +33,12 @@ import java.util.concurrent.Future;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
-
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
import org.apache.sysml.runtime.matrix.data.IJV;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
-import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
import org.apache.sysml.runtime.util.MapReduceTool;
/**
@@ -152,7 +151,7 @@ public class WriterMatrixMarketParallel extends WriterMatrixMarket
if( _src.isInSparseFormat() ) //SPARSE
{
- SparseRowsIterator iter = _src.getSparseRowsIterator(_rl, _ru);
+ Iterator<IJV> iter = _src.getSparseBlockIterator(_rl, _ru);
while( iter.hasNext() )
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java
index b6f56eb..96e31db 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java
@@ -138,7 +138,7 @@ public class WriterTextCSV extends MatrixWriter
// Write data lines
if( sparse ) //SPARSE
{
- SparseRow[] sparseRows = src.getSparseRows();
+ SparseRow[] sparseRows = src.getSparseBlock();
for(int i=0; i < rlen; i++)
{
//write row chunk-wise to prevent OOM on large number of columns
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java
index e1e849b..9758dc5 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java
@@ -172,7 +172,7 @@ public class WriterTextCSVParallel extends WriterTextCSV
// Write data lines
if( sparse ) //SPARSE
{
- SparseRow[] sparseRows = _src.getSparseRows();
+ SparseRow[] sparseRows = _src.getSparseBlock();
for( int i=_rl; i<_ru; i++ )
{
//write row chunk-wise to prevent OOM on large number of columns
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java
index 1c6eb69..66f6ec4 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java
@@ -22,18 +22,17 @@ package org.apache.sysml.runtime.io;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
+import java.util.Iterator;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
-
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.DMLUnsupportedOperationException;
import org.apache.sysml.runtime.matrix.data.IJV;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
import org.apache.sysml.runtime.util.MapReduceTool;
public class WriterTextCell extends MatrixWriter
@@ -106,7 +105,7 @@ public class WriterTextCell extends MatrixWriter
if( sparse ) //SPARSE
{
- SparseRowsIterator iter = src.getSparseRowsIterator();
+ Iterator<IJV> iter = src.getSparseBlockIterator();
while( iter.hasNext() )
{
IJV cell = iter.next();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java
index 809f33f..6c1312c 100644
--- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java
+++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java
@@ -23,6 +23,7 @@ import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
@@ -32,14 +33,12 @@ import java.util.concurrent.Future;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
-
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
import org.apache.sysml.runtime.matrix.data.IJV;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
-import org.apache.sysml.runtime.matrix.data.SparseRowsIterator;
import org.apache.sysml.runtime.util.MapReduceTool;
public class WriterTextCellParallel extends WriterTextCell
@@ -139,7 +138,7 @@ public class WriterTextCellParallel extends WriterTextCell
if( _src.isInSparseFormat() ) //SPARSE
{
- SparseRowsIterator iter = _src.getSparseRowsIterator(_rl, _ru);
+ Iterator<IJV> iter = _src.getSparseBlockIterator(_rl, _ru);
while( iter.hasNext() )
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java
index fcbb79c..e4fefe4 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java
@@ -20,6 +20,8 @@
package org.apache.sysml.runtime.matrix.data;
+import java.util.Iterator;
+
import org.apache.sysml.runtime.util.UtilFunctions;
@@ -28,7 +30,7 @@ public class BinaryBlockToBinaryCellConverter implements
Converter<MatrixIndexes, MatrixBlock, MatrixIndexes, MatrixCell>
{
- private SparseRowsIterator sparseIterator=null;
+ private Iterator<IJV> sparseIterator=null;
private double[] denseArray=null;
private int denseArraySize=0;
private int nextInDenseArray=-1;
@@ -62,13 +64,13 @@ Converter<MatrixIndexes, MatrixBlock, MatrixIndexes, MatrixCell>
thisBlockWidth=v1.getNumColumns();
if(sparse)
{
- sparseIterator=v1.getSparseRowsIterator();
+ sparseIterator=v1.getSparseBlockIterator();
}
else
{
- if(v1.getDenseArray()==null)
+ if(v1.getDenseBlock()==null)
return;
- denseArray=v1.getDenseArray();
+ denseArray=v1.getDenseBlock();
nextInDenseArray=0;
denseArraySize=v1.getNumRows()*v1.getNumColumns();
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java
index da63d64..0004adc 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java
@@ -21,18 +21,18 @@
package org.apache.sysml.runtime.matrix.data;
+import java.util.Iterator;
+
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
-
import org.apache.sysml.runtime.util.UtilFunctions;
public class BinaryBlockToTextCellConverter implements
Converter<MatrixIndexes, MatrixBlock, NullWritable, Text>
-{
-
- private SparseRowsIterator sparseIterator=null;
+{
+ private Iterator<IJV> sparseIterator=null;
private double[] denseArray=null;
private int denseArraySize=0;
private int nextInDenseArray=-1;
@@ -68,13 +68,13 @@ Converter<MatrixIndexes, MatrixBlock, NullWritable, Text>
thisBlockWidth=v1.getNumColumns();
if(sparse)
{
- sparseIterator=v1.getSparseRowsIterator();
+ sparseIterator=v1.getSparseBlockIterator();
}
else
{
- if(v1.getDenseArray()==null)
+ if(v1.getDenseBlock()==null)
return;
- denseArray=v1.getDenseArray();
+ denseArray=v1.getDenseBlock();
nextInDenseArray=0;
denseArraySize=v1.getNumRows()*v1.getNumColumns();
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java
index ee20f25..f856e06 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java
@@ -512,7 +512,7 @@ public class LibMatrixAgg
if( (type == AggType.MAX_INDEX || type == AggType.MIN_INDEX) && ix.getColumnIndex()!=1 ) //MAXINDEX or MININDEX
{
int m = out.rlen;
- double[] c = out.getDenseArray();
+ double[] c = out.getDenseBlock();
for( int i=0, cix=0; i<m; i++, cix+=2 )
c[cix] = UtilFunctions.cellIndexCalculation(ix.getColumnIndex(), bclen, (int)c[cix]-1);
}
@@ -689,7 +689,7 @@ public class LibMatrixAgg
if( in1.sparse )
{
- SparseRow[] a = in1.sparseRows;
+ SparseRow[] a = in1.sparseBlock;
for( int i=rl; i<ru; i++ )
if( a[i]!=null && !a[i].isEmpty() ) {
@@ -757,11 +757,11 @@ public class LibMatrixAgg
if( target.sparse ) //SPARSE target
{
- if( target.sparseRows[0]!=null )
+ if( target.sparseBlock[0]!=null )
{
- int len = target.sparseRows[0].size();
- int[] aix = target.sparseRows[0].getIndexContainer();
- double[] avals = target.sparseRows[0].getValueContainer();
+ int len = target.sparseBlock[0].size();
+ int[] aix = target.sparseBlock[0].getIndexContainer();
+ double[] avals = target.sparseBlock[0].getValueContainer();
for( int j=0; j<len; j++ ) //for each nnz
{
int g = (int) groups.quickGetValue(aix[j], 0);
@@ -795,7 +795,7 @@ public class LibMatrixAgg
{
if( target.sparse ) //SPARSE target
{
- SparseRow[] a = target.sparseRows;
+ SparseRow[] a = target.sparseBlock;
for( int i=0; i < groups.getNumRows(); i++ )
{
@@ -873,7 +873,7 @@ public class LibMatrixAgg
//column vector or matrix
if( target.sparse ) //SPARSE target
{
- SparseRow[] a = target.sparseRows;
+ SparseRow[] a = target.sparseBlock;
for( int i=0; i < groups.getNumRows(); i++ )
{
@@ -976,9 +976,9 @@ public class LibMatrixAgg
aggVal.allocateDenseBlock(); //should always stay in dense
aggCorr.allocateDenseBlock(); //should always stay in dense
- double[] a = in.getDenseArray();
- double[] c = aggVal.getDenseArray();
- double[] cc = aggCorr.getDenseArray();
+ double[] a = in.getDenseBlock();
+ double[] c = aggVal.getDenseBlock();
+ double[] cc = aggCorr.getDenseBlock();
KahanObject buffer1 = new KahanObject(0, 0);
KahanPlus akplus = KahanPlus.getKahanPlusFnObject();
@@ -1015,16 +1015,16 @@ public class LibMatrixAgg
private static void aggregateBinaryMatrixSparseDense(MatrixBlock in, MatrixBlock aggVal, MatrixBlock aggCorr)
throws DMLRuntimeException
{
- if( in.sparseRows==null || in.isEmptyBlock(false) )
+ if( in.isEmptyBlock(false) )
return;
//allocate output arrays (if required)
aggVal.allocateDenseBlock(); //should always stay in dense
aggCorr.allocateDenseBlock(); //should always stay in dense
- SparseRow[] a = in.getSparseRows();
- double[] c = aggVal.getDenseArray();
- double[] cc = aggCorr.getDenseArray();
+ SparseRow[] a = in.getSparseBlock();
+ double[] c = aggVal.getDenseBlock();
+ double[] cc = aggCorr.getDenseBlock();
KahanObject buffer1 = new KahanObject(0, 0);
KahanPlus akplus = KahanPlus.getKahanPlusFnObject();
@@ -1070,10 +1070,10 @@ public class LibMatrixAgg
private static void aggregateBinaryMatrixSparseGeneric(MatrixBlock in, MatrixBlock aggVal, MatrixBlock aggCorr)
throws DMLRuntimeException
{
- if( in.sparseRows==null || in.isEmptyBlock(false) )
+ if( in.isEmptyBlock(false) )
return;
- SparseRow[] a = in.getSparseRows();
+ SparseRow[] a = in.getSparseBlock();
KahanObject buffer1 = new KahanObject(0, 0);
KahanPlus akplus = KahanPlus.getKahanPlusFnObject();
@@ -1124,7 +1124,7 @@ public class LibMatrixAgg
final int m = in.rlen;
final int n = in.clen;
- double[] a = in.getDenseArray();
+ double[] a = in.getDenseBlock();
KahanObject buffer = new KahanObject(0, 0);
KahanPlus akplus = KahanPlus.getKahanPlusFnObject();
@@ -1161,7 +1161,7 @@ public class LibMatrixAgg
final int n = in.clen;
final int cix = (m-1)*n;
- double[] a = in.getDenseArray();
+ double[] a = in.getDenseBlock();
KahanObject buffer = new KahanObject(0, 0);
KahanPlus akplus = KahanPlus.getKahanPlusFnObject();
@@ -1191,10 +1191,10 @@ public class LibMatrixAgg
throws DMLRuntimeException
{
//sparse-safe operation
- if( in.sparseRows==null || in.isEmptyBlock(false) )
+ if( in.isEmptyBlock(false) )
return;
- SparseRow[] a = in.getSparseRows();
+ SparseRow[] a = in.getSparseBlock();
KahanObject buffer1 = new KahanObject(0, 0);
KahanPlus akplus = KahanPlus.getKahanPlusFnObject();
@@ -1243,7 +1243,7 @@ public class LibMatrixAgg
final int m = in.rlen;
final int n = in.clen;
- double[] a = in.getDenseArray();
+ double[] a = in.getDenseBlock();
KahanObject buffer = new KahanObject(0, 0);
KahanPlus akplus = KahanPlus.getKahanPlusFnObject();
@@ -1273,10 +1273,10 @@ public class LibMatrixAgg
throws DMLRuntimeException
{
//sparse-safe operation
- if( in.sparseRows==null || in.isEmptyBlock(false) )
+ if( in.isEmptyBlock(false) )
return;
- SparseRow[] a = in.getSparseRows();
+ SparseRow[] a = in.getSparseBlock();
KahanObject buffer1 = new KahanObject(0, 0);
KahanPlus akplus = KahanPlus.getKahanPlusFnObject();
@@ -1325,8 +1325,8 @@ public class LibMatrixAgg
final int m = in.rlen;
final int n = in.clen;
- double[] a = in.getDenseArray();
- double[] c = out.getDenseArray();
+ double[] a = in.getDenseBlock();
+ double[] c = out.getDenseBlock();
switch( optype )
{
@@ -1455,8 +1455,8 @@ public class LibMatrixAgg
final int m = in.rlen;
final int n = in.clen;
- SparseRow[] a = in.getSparseRows();
- double[] c = out.getDenseArray();
+ SparseRow[] a = in.getSparseBlock();
+ double[] c = out.getDenseBlock();
switch( optype )
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixBincell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixBincell.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixBincell.java
index cead016..1147c5e 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixBincell.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixBincell.java
@@ -262,12 +262,12 @@ public class LibMatrixBincell
ret.allocateSparseRowsBlock();
//both sparse blocks existing
- if(m1.sparseRows!=null && m2.sparseRows!=null)
+ if(m1.sparseBlock!=null && m2.sparseBlock!=null)
{
for(int r=0; r<rlen; r++)
{
- SparseRow lrow = (m1.sparseRows.length>r && m1.sparseRows[r]!=null) ? m1.sparseRows[r] : null;
- SparseRow rrow = (m2.sparseRows.length>r && m2.sparseRows[r]!=null) ? m2.sparseRows[r] : null;
+ SparseRow lrow = (m1.sparseBlock.length>r && m1.sparseBlock[r]!=null) ? m1.sparseBlock[r] : null;
+ SparseRow rrow = (m2.sparseBlock.length>r && m2.sparseBlock[r]!=null) ? m2.sparseBlock[r] : null;
if( lrow!=null && rrow!=null)
{
@@ -289,23 +289,23 @@ public class LibMatrixBincell
}
}
//right sparse block existing
- else if( m2.sparseRows!=null )
+ else if( m2.sparseBlock!=null )
{
- for(int r=0; r<Math.min(rlen, m2.sparseRows.length); r++)
- if(m2.sparseRows[r]!=null)
+ for(int r=0; r<Math.min(rlen, m2.sparseBlock.length); r++)
+ if(m2.sparseBlock[r]!=null)
{
- appendRightForSparseBinary(op, m2.sparseRows[r].getValueContainer(),
- m2.sparseRows[r].getIndexContainer(), m2.sparseRows[r].size(), 0, r, ret);
+ appendRightForSparseBinary(op, m2.sparseBlock[r].getValueContainer(),
+ m2.sparseBlock[r].getIndexContainer(), m2.sparseBlock[r].size(), 0, r, ret);
}
}
//left sparse block existing
else
{
for(int r=0; r<rlen; r++)
- if( m1.sparseRows[r]!=null )
+ if( m1.sparseBlock[r]!=null )
{
- appendLeftForSparseBinary(op, m1.sparseRows[r].getValueContainer(),
- m1.sparseRows[r].getIndexContainer(), m1.sparseRows[r].size(), 0, r, ret);
+ appendLeftForSparseBinary(op, m1.sparseBlock[r].getValueContainer(),
+ m1.sparseBlock[r].getIndexContainer(), m1.sparseBlock[r].size(), 0, r, ret);
}
}
}
@@ -328,10 +328,10 @@ public class LibMatrixBincell
{
Arrays.fill(ret.denseBlock, 0, ret.denseBlock.length, 0);
- if( m1.sparseRows != null )
+ if( m1.sparseBlock != null )
{
for( int i=0, ix=0; i<m; i++, ix+=n ) {
- SparseRow arow = m1.sparseRows[i];
+ SparseRow arow = m1.sparseBlock[i];
if( arow != null && !arow.isEmpty() )
{
alen = arow.size();
@@ -354,10 +354,10 @@ public class LibMatrixBincell
//2) process right input: op.fn (+,-,*), * only if dense
if( m2.sparse ) //SPARSE right
{
- if(m2.sparseRows!=null)
+ if(m2.sparseBlock!=null)
{
for( int i=0, ix=0; i<m; i++, ix+=n ) {
- SparseRow arow = m2.sparseRows[i];
+ SparseRow arow = m2.sparseBlock[i];
if( arow != null && !arow.isEmpty() )
{
alen = arow.size();
@@ -509,7 +509,7 @@ public class LibMatrixBincell
int rlen = m1.rlen;
int clen = m1.clen;
- SparseRow[] a = m1.sparseRows;
+ SparseRow[] a = m1.sparseBlock;
BinaryAccessType atype = getBinaryAccessType(m1, m2);
//early abort on skip and empty
@@ -672,7 +672,7 @@ public class LibMatrixBincell
if( m2.sparse && isMultiply ) //SPARSE *
{
//note: sparse block guaranteed to be allocated (otherwise early about)
- SparseRow brow = m2.sparseRows[0];
+ SparseRow brow = m2.sparseBlock[0];
if( brow != null && !brow.isEmpty() )
{
int blen = brow.size();
@@ -786,12 +786,12 @@ public class LibMatrixBincell
if(bOp.fn instanceof LessThan || bOp.fn instanceof GreaterThanEquals
|| bOp.fn instanceof GreaterThan || bOp.fn instanceof LessThanEquals
|| bOp.fn instanceof Equals) {
- Arrays.fill(mbOut.getDenseArray(), iOffSet+iStartPos, iOffSet+iEndPos, 1.0);
+ Arrays.fill(mbOut.getDenseBlock(), iOffSet+iStartPos, iOffSet+iEndPos, 1.0);
lNNZ += (iEndPos-iStartPos);
}
else if (bOp.fn instanceof NotEquals) {
- Arrays.fill(mbOut.getDenseArray(), iOffSet, iOffSet+iStartPos, 1.0);
- Arrays.fill(mbOut.getDenseArray(), iOffSet+iEndPos, iOffSet+bv.length, 1.0);
+ Arrays.fill(mbOut.getDenseBlock(), iOffSet, iOffSet+iStartPos, 1.0);
+ Arrays.fill(mbOut.getDenseBlock(), iOffSet+iEndPos, iOffSet+bv.length, 1.0);
lNNZ += (iStartPos+(bv.length-iEndPos));
}
}
@@ -921,10 +921,10 @@ public class LibMatrixBincell
{
//allocate sparse row structure
ret.allocateSparseRowsBlock();
- SparseRow[] a = m1.sparseRows;
- SparseRow[] c = ret.sparseRows;
+ SparseRow[] a = m1.sparseBlock;
+ SparseRow[] c = ret.sparseBlock;
- for(int r=0; r<Math.min(m1.rlen, m1.sparseRows.length); r++) {
+ for(int r=0; r<Math.min(m1.rlen, m1.sparseBlock.length); r++) {
if( a[r]!=null && !a[r].isEmpty() )
{
int alen = a[r].size();
@@ -1012,7 +1012,7 @@ public class LibMatrixBincell
{
ret.allocateDenseBlock();
- SparseRow[] a = m1.sparseRows;
+ SparseRow[] a = m1.sparseBlock;
double[] c = ret.denseBlock;
int m = m1.rlen;
int n = m1.clen;
@@ -1082,61 +1082,61 @@ public class LibMatrixBincell
if(m1ret.sparse && m2.sparse)
{
- if(m1ret.sparseRows!=null)
+ if(m1ret.sparseBlock!=null)
m1ret.allocateSparseRowsBlock(false);
- if(m2.sparseRows!=null)
+ if(m2.sparseBlock!=null)
m2.allocateSparseRowsBlock(false);
- if(m1ret.sparseRows!=null && m2.sparseRows!=null)
+ if(m1ret.sparseBlock!=null && m2.sparseBlock!=null)
{
for(int r=0; r<rlen; r++)
{
- if(m1ret.sparseRows[r]==null && m2.sparseRows[r]==null)
+ if(m1ret.sparseBlock[r]==null && m2.sparseBlock[r]==null)
continue;
- if(m2.sparseRows[r]==null)
+ if(m2.sparseBlock[r]==null)
{
- double[] values=m1ret.sparseRows[r].getValueContainer();
- for(int i=0; i<m1ret.sparseRows[r].size(); i++)
+ double[] values=m1ret.sparseBlock[r].getValueContainer();
+ for(int i=0; i<m1ret.sparseBlock[r].size(); i++)
values[i]=op.fn.execute(values[i], 0);
}else
{
int estimateSize=0;
- if(m1ret.sparseRows[r]!=null)
- estimateSize+=m1ret.sparseRows[r].size();
- if(m2.sparseRows[r]!=null)
- estimateSize+=m2.sparseRows[r].size();
+ if(m1ret.sparseBlock[r]!=null)
+ estimateSize+=m1ret.sparseBlock[r].size();
+ if(m2.sparseBlock[r]!=null)
+ estimateSize+=m2.sparseBlock[r].size();
estimateSize=Math.min(clen, estimateSize);
//temp
- SparseRow thisRow=m1ret.sparseRows[r];
- m1ret.sparseRows[r]=new SparseRow(estimateSize, clen);
+ SparseRow thisRow=m1ret.sparseBlock[r];
+ m1ret.sparseBlock[r]=new SparseRow(estimateSize, clen);
if(thisRow!=null)
{
m1ret.nonZeros-=thisRow.size();
mergeForSparseBinary(op, thisRow.getValueContainer(),
thisRow.getIndexContainer(), thisRow.size(),
- m2.sparseRows[r].getValueContainer(),
- m2.sparseRows[r].getIndexContainer(), m2.sparseRows[r].size(), r, m1ret);
+ m2.sparseBlock[r].getValueContainer(),
+ m2.sparseBlock[r].getIndexContainer(), m2.sparseBlock[r].size(), r, m1ret);
}else
{
- appendRightForSparseBinary(op, m2.sparseRows[r].getValueContainer(),
- m2.sparseRows[r].getIndexContainer(), m2.sparseRows[r].size(), 0, r, m1ret);
+ appendRightForSparseBinary(op, m2.sparseBlock[r].getValueContainer(),
+ m2.sparseBlock[r].getIndexContainer(), m2.sparseBlock[r].size(), 0, r, m1ret);
}
}
}
}
- else if(m1ret.sparseRows==null)
+ else if(m1ret.sparseBlock==null)
{
- m1ret.sparseRows=new SparseRow[rlen];
+ m1ret.sparseBlock=new SparseRow[rlen];
for(int r=0; r<rlen; r++)
{
- SparseRow brow = m2.sparseRows[r];
+ SparseRow brow = m2.sparseBlock[r];
if( brow!=null && !brow.isEmpty() )
{
- m1ret.sparseRows[r] = new SparseRow( brow.size(), clen );
+ m1ret.sparseBlock[r] = new SparseRow( brow.size(), clen );
appendRightForSparseBinary(op, brow.getValueContainer(), brow.getIndexContainer(), brow.size(), 0, r, m1ret);
}
}
@@ -1145,7 +1145,7 @@ public class LibMatrixBincell
{
if( !(op.fn instanceof Plus || op.fn instanceof Minus || op.fn instanceof Or) ){
for(int r=0; r<rlen; r++){
- SparseRow arow = m1ret.sparseRows[r];
+ SparseRow arow = m1ret.sparseBlock[r];
if( arow!=null && !arow.isEmpty() )
{
int alen = arow.size();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java
index 7844ac3..96b929f 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java
@@ -671,7 +671,7 @@ public class LibMatrixDatagen
// irrelevant but we need to ensure consistency with MR)
boolean localSparse = MatrixBlock.evalSparseFormatInMemory(blockrows, blockcols, nnzInBlocks[blockID] ); //(long)(sparsity*blockrows*blockcols));
if ( localSparse ) {
- SparseRow[] c = out.sparseRows;
+ SparseRow[] c = out.sparseBlock;
int idx = 0; // takes values in range [1, brlen*bclen] (both ends including)
int ridx=0, cidx=0; // idx translates into (ridx, cidx) entry within the block
@@ -716,7 +716,7 @@ public class LibMatrixDatagen
*
*/
// In this case, entire matrix is in sparse format but the current block is dense
- SparseRow[] c = out.sparseRows;
+ SparseRow[] c = out.sparseBlock;
for(int ii=0; ii < blockrows; ii++) {
for(int jj=0; jj < blockcols; jj++) {
if(nnzPRNG.nextDouble() <= sparsity) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
index 50247b9..f617f40 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
@@ -1170,7 +1170,7 @@ public class LibMatrixMult
final int blocksizeK = 32;
//note: in contrast to dense-dense, no blocking over j (would require maintaining blocksizeK indexes, counter-productive on skew)
- SparseRow[] b = m2.sparseRows;
+ SparseRow[] b = m2.sparseBlock;
if( pm2 && m==1 ) //VECTOR-MATRIX
{
@@ -1219,7 +1219,7 @@ public class LibMatrixMult
double val = a[aix];
if( val!=0 )
{
- SparseRow brow = m2.sparseRows[ k ];
+ SparseRow brow = m2.sparseBlock[ k ];
if( brow != null && !brow.isEmpty() )
{
int blen = brow.size();
@@ -1253,7 +1253,7 @@ public class LibMatrixMult
{
if( m==1 && n==1 ) //DOT PRODUCT
{
- SparseRow arow = m1.sparseRows[0];
+ SparseRow arow = m1.sparseBlock[0];
if( arow != null && !arow.isEmpty() )
{
int alen = arow.size();
@@ -1265,9 +1265,9 @@ public class LibMatrixMult
}
else if( n==1 ) //MATRIX-VECTOR
{
- for( int i=rl; i<Math.min(ru, m1.sparseRows.length); i++ )
+ for( int i=rl; i<Math.min(ru, m1.sparseBlock.length); i++ )
{
- SparseRow arow = m1.sparseRows[i];
+ SparseRow arow = m1.sparseBlock[i];
if( arow != null && !arow.isEmpty() )
{
int alen = arow.size();
@@ -1281,7 +1281,7 @@ public class LibMatrixMult
else if( pm2 && m==1 ) //VECTOR-MATRIX
{
//parallelization over rows in rhs matrix
- SparseRow arow = m1.sparseRows[0];
+ SparseRow arow = m1.sparseBlock[0];
if( arow != null && !arow.isEmpty() )
{
int alen = arow.size();
@@ -1300,7 +1300,7 @@ public class LibMatrixMult
}
else if( pm2 && m<=16 ) //MATRIX-MATRIX (short lhs)
{
- SparseRow[] a = m1.sparseRows;
+ SparseRow[] a = m1.sparseBlock;
for( int i=0, cix=0; i<a.length; i++, cix+=n )
if( a[i] != null && !a[i].isEmpty() )
{
@@ -1330,9 +1330,9 @@ public class LibMatrixMult
}
else //MATRIX-MATRIX
{
- for( int i=rl, cix=rl*n; i<Math.min(ru, m1.sparseRows.length); i++, cix+=n )
+ for( int i=rl, cix=rl*n; i<Math.min(ru, m1.sparseBlock.length); i++, cix+=n )
{
- SparseRow arow = m1.sparseRows[i];
+ SparseRow arow = m1.sparseBlock[i];
if( arow != null && !arow.isEmpty() )
{
int alen = arow.size();
@@ -1366,9 +1366,9 @@ public class LibMatrixMult
}
else
{
- for( int i=rl, cix=rl*n; i<Math.min(ru, m1.sparseRows.length); i++, cix+=n )
+ for( int i=rl, cix=rl*n; i<Math.min(ru, m1.sparseBlock.length); i++, cix+=n )
{
- SparseRow arow = m1.sparseRows[i];
+ SparseRow arow = m1.sparseBlock[i];
if( arow != null && !arow.isEmpty() )
{
int alen = arow.size();
@@ -1396,7 +1396,7 @@ public class LibMatrixMult
private static void matrixMultSparseSparse(MatrixBlock m1, MatrixBlock m2, MatrixBlock ret, boolean pm2, int rl, int ru)
throws DMLRuntimeException
{
- SparseRow[] b = m2.sparseRows;
+ SparseRow[] b = m2.sparseBlock;
double[] c = ret.denseBlock;
int m = m1.rlen;
int n = m2.clen;
@@ -1407,7 +1407,7 @@ public class LibMatrixMult
if( pm2 && m==1 ) //VECTOR-MATRIX
{
//parallelization over rows in rhs matrix
- SparseRow arow = m1.sparseRows[0];
+ SparseRow arow = m1.sparseBlock[0];
if( arow != null && !arow.isEmpty() )
{
int alen = arow.size();
@@ -1428,9 +1428,9 @@ public class LibMatrixMult
}
else //MATRIX-MATRIX
{
- for( int i=rl, cix=rl*n; i<Math.min(ru, m1.sparseRows.length); i++, cix+=n )
+ for( int i=rl, cix=rl*n; i<Math.min(ru, m1.sparseBlock.length); i++, cix+=n )
{
- SparseRow arow = m1.sparseRows[i];
+ SparseRow arow = m1.sparseBlock[i];
if( arow != null && !arow.isEmpty() )
{
int alen = arow.size();
@@ -1456,9 +1456,9 @@ public class LibMatrixMult
}
else
{
- for( int i=rl, cix=rl*n; i<Math.min(ru, m1.sparseRows.length); i++, cix+=n )
+ for( int i=rl, cix=rl*n; i<Math.min(ru, m1.sparseBlock.length); i++, cix+=n )
{
- SparseRow arow = m1.sparseRows[i];
+ SparseRow arow = m1.sparseBlock[i];
if( arow != null && !arow.isEmpty() )
{
int alen = arow.size();
@@ -1468,7 +1468,7 @@ public class LibMatrixMult
for(int k = 0; k < alen; k++)
{
double val = avals[k];
- SparseRow brow = m2.sparseRows[ aix[k] ];
+ SparseRow brow = m2.sparseBlock[ aix[k] ];
if( brow != null && !brow.isEmpty() )
{
int blen = brow.size();
@@ -1510,7 +1510,7 @@ public class LibMatrixMult
for( int i=rl; i<ru; i++ )
{
- SparseRow arow = m1.sparseRows[ i ];
+ SparseRow arow = m1.sparseBlock[ i ];
if( arow != null && !arow.isEmpty() )
{
int alen = arow.size();
@@ -1521,11 +1521,11 @@ public class LibMatrixMult
{
int aix = aixs[0];
if( rightSparse ) { //sparse right matrix (full row copy)
- if( m2.sparseRows!=null && m2.sparseRows[aix]!=null ) {
+ if( m2.sparseBlock!=null && m2.sparseBlock[aix]!=null ) {
ret.rlen=m;
ret.allocateSparseRowsBlock(false); //allocation on demand
- ret.sparseRows[i] = new SparseRow(m2.sparseRows[aix]);
- ret.nonZeros += ret.sparseRows[i].size();
+ ret.sparseBlock[i] = new SparseRow(m2.sparseBlock[aix]);
+ ret.nonZeros += ret.sparseBlock[i].size();
}
}
else { //dense right matrix (append all values)
@@ -1555,7 +1555,7 @@ public class LibMatrixMult
{
for(int k = 0; k < cd; k++ )
{
- SparseRow brow = m2.sparseRows[ k ];
+ SparseRow brow = m2.sparseBlock[ k ];
if( brow != null && !brow.isEmpty() )
{
int blen = brow.size();
@@ -1647,7 +1647,7 @@ public class LibMatrixMult
*/
private static void matrixMultChainSparse(MatrixBlock mX, MatrixBlock mV, MatrixBlock mW, MatrixBlock ret, ChainType ct, int rl, int ru)
{
- SparseRow[] a = mX.sparseRows;
+ SparseRow[] a = mX.sparseBlock;
double[] b = mV.denseBlock;
double[] w = (mW!=null) ? mW.denseBlock : null;
double[] c = ret.denseBlock;
@@ -1858,7 +1858,7 @@ public class LibMatrixMult
//algorithm: scan rows, foreach row self join (KIJ)
if( LOW_LEVEL_OPTIMIZATION )
{
- for( SparseRow arow : m1.sparseRows )
+ for( SparseRow arow : m1.sparseBlock )
if( arow != null && !arow.isEmpty() )
{
int alen = arow.size();
@@ -1879,7 +1879,7 @@ public class LibMatrixMult
}
else
{
- for( SparseRow arow : m1.sparseRows )
+ for( SparseRow arow : m1.sparseBlock )
if( arow != null && !arow.isEmpty() )
{
int alen = arow.size();
@@ -1902,7 +1902,7 @@ public class LibMatrixMult
{
if( m==1 ) //VECTOR
{
- SparseRow arow = m1.sparseRows[0];
+ SparseRow arow = m1.sparseBlock[0];
if( arow !=null && !arow.isEmpty() )
{
int alen = arow.size();
@@ -1921,7 +1921,7 @@ public class LibMatrixMult
//algorithm: scan rows, foreach row self join (KIJ)
if( LOW_LEVEL_OPTIMIZATION )
{
- for( SparseRow arow : m1.sparseRows )
+ for( SparseRow arow : m1.sparseBlock )
if( arow != null && !arow.isEmpty() )
{
int alen = arow.size();
@@ -1942,7 +1942,7 @@ public class LibMatrixMult
}
else
{
- for( SparseRow arow : m1.sparseRows )
+ for( SparseRow arow : m1.sparseBlock )
if( arow != null && !arow.isEmpty() )
{
int alen = arow.size();
@@ -2023,7 +2023,7 @@ public class LibMatrixMult
{
double[] a = pm1.denseBlock;
double[] b = m2.denseBlock;
- SparseRow[] c = ret1.sparseRows;
+ SparseRow[] c = ret1.sparseBlock;
final int n = m2.clen;
final int brlen = ret1.getNumRows();
@@ -2044,7 +2044,7 @@ public class LibMatrixMult
ret2.sparse = true;
ret2.rlen=ret1.rlen;
ret2.allocateSparseRowsBlock();
- c = ret2.sparseRows;
+ c = ret2.sparseBlock;
}
//append entire dense row into sparse target position
@@ -2069,8 +2069,8 @@ public class LibMatrixMult
private static void matrixMultPermuteSparse( MatrixBlock pm1, MatrixBlock m2, MatrixBlock ret1, MatrixBlock ret2, int rl, int ru)
{
double[] a = pm1.denseBlock;
- SparseRow[] b = m2.sparseRows;
- SparseRow[] c = ret1.sparseRows;
+ SparseRow[] b = m2.sparseBlock;
+ SparseRow[] c = ret1.sparseBlock;
final int brlen = ret1.getNumRows();
@@ -2089,7 +2089,7 @@ public class LibMatrixMult
if( lastblk!=-1 && lastblk<blk ){
ret2.sparse = true;
ret2.allocateSparseRowsBlock();
- c = ret2.sparseRows;
+ c = ret2.sparseBlock;
}
//memcopy entire sparse row into target position
@@ -2198,8 +2198,8 @@ public class LibMatrixMult
*/
private static void matrixMultWSLossSparseDense(MatrixBlock mX, MatrixBlock mU, MatrixBlock mV, MatrixBlock mW, MatrixBlock ret, WeightsType wt, int rl, int ru)
{
- SparseRow[] x = mX.sparseRows;
- SparseRow[] w = (mW!=null)? mW.sparseRows : null;
+ SparseRow[] x = mX.sparseBlock;
+ SparseRow[] w = (mW!=null)? mW.sparseBlock : null;
double[] u = mU.denseBlock;
double[] v = mV.denseBlock;
final int n = mX.clen;
@@ -2316,7 +2316,7 @@ public class LibMatrixMult
// approach: iterate over W, point-wise in order to exploit sparsity
if( mW.sparse ) //SPARSE
{
- SparseRow[] wrows = mW.sparseRows;
+ SparseRow[] wrows = mW.sparseBlock;
for( int i=rl; i<ru; i++ )
if( wrows[i] != null && !wrows[i].isEmpty() ){
@@ -2349,7 +2349,7 @@ public class LibMatrixMult
// approach: iterate over W, point-wise in order to exploit sparsity
if( mW.sparse ) //SPARSE
{
- SparseRow[] xrows = mX.sparseRows;
+ SparseRow[] xrows = mX.sparseBlock;
for( int i=rl; i<ru; i++ )
if( xrows[i] != null && !xrows[i].isEmpty() ){
@@ -2469,8 +2469,8 @@ public class LibMatrixMult
private static void matrixMultWSigmoidSparseDense(MatrixBlock mW, MatrixBlock mU, MatrixBlock mV, MatrixBlock ret, WSigmoidType wt, int rl, int ru)
throws DMLRuntimeException
{
- SparseRow[] w = mW.sparseRows;
- SparseRow[] c = ret.sparseRows;
+ SparseRow[] w = mW.sparseBlock;
+ SparseRow[] c = ret.sparseBlock;
double[] u = mU.denseBlock;
double[] v = mV.denseBlock;
final int n = mW.clen;
@@ -2519,8 +2519,8 @@ public class LibMatrixMult
if( mW.sparse ) //SPARSE
{
//w and c always in same representation
- SparseRow[] w = mW.sparseRows;
- SparseRow[] c = ret.sparseRows;
+ SparseRow[] w = mW.sparseBlock;
+ SparseRow[] c = ret.sparseBlock;
for( int i=rl; i<ru; i++ )
if( w[i] != null && !w[i].isEmpty() ) {
@@ -2622,7 +2622,7 @@ public class LibMatrixMult
final boolean minus = wt.isMinus();
final int cd = mU.clen;
- SparseRow[] w = mW.sparseRows;
+ SparseRow[] w = mW.sparseBlock;
double[] u = mU.denseBlock;
double[] v = mV.denseBlock;
double[] c = ret.denseBlock;
@@ -2676,7 +2676,7 @@ public class LibMatrixMult
//approach: iterate over non-zeros of w, selective mm computation
if( mW.sparse ) //SPARSE
{
- SparseRow[] w = mW.sparseRows;
+ SparseRow[] w = mW.sparseBlock;
for( int i=rl; i<ru; i++ ) {
if( w[i] != null && !w[i].isEmpty() ) {
@@ -2769,7 +2769,7 @@ public class LibMatrixMult
*/
private static void matrixMultWCeMMSparseDense(MatrixBlock mW, MatrixBlock mU, MatrixBlock mV, MatrixBlock ret, WCeMMType wt, int rl, int ru)
{
- SparseRow[] w = mW.sparseRows;
+ SparseRow[] w = mW.sparseBlock;
double[] u = mU.denseBlock;
double[] v = mV.denseBlock;
final int cd = mU.clen;
@@ -2811,7 +2811,7 @@ public class LibMatrixMult
//approach: iterate over non-zeros of w, selective mm computation
if( mW.sparse ) //SPARSE
{
- SparseRow[] w = mW.sparseRows;
+ SparseRow[] w = mW.sparseBlock;
for( int i=rl; i<ru; i++ )
if( w[i] != null && !w[i].isEmpty() ) {
@@ -2905,8 +2905,8 @@ public class LibMatrixMult
private static void matrixMultWuMMSparseDense(MatrixBlock mW, MatrixBlock mU, MatrixBlock mV, MatrixBlock ret, WUMMType wt, ValueFunction fn, int rl, int ru)
throws DMLRuntimeException
{
- SparseRow[] w = mW.sparseRows;
- SparseRow[] c = ret.sparseRows;
+ SparseRow[] w = mW.sparseBlock;
+ SparseRow[] c = ret.sparseBlock;
double[] u = mU.denseBlock;
double[] v = mV.denseBlock;
final int n = mW.clen;
@@ -2953,8 +2953,8 @@ public class LibMatrixMult
if( mW.sparse ) //SPARSE
{
//w and c always in same representation
- SparseRow[] w = mW.sparseRows;
- SparseRow[] c = ret.sparseRows;
+ SparseRow[] w = mW.sparseBlock;
+ SparseRow[] c = ret.sparseBlock;
for( int i=rl; i<ru; i++ )
if( w[i] != null && !w[i].isEmpty() ) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixOuterAgg.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixOuterAgg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixOuterAgg.java
index 01c7751..c46e2c5 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixOuterAgg.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixOuterAgg.java
@@ -894,14 +894,14 @@ public class LibMatrixOuterAgg
//allocate and initialize output values (not indices)
out.allocateDenseBlock(true);
- Arrays.fill(out.getDenseArray(), 0, out.getNumColumns(), agg0);
+ Arrays.fill(out.getDenseBlock(), 0, out.getNumColumns(), agg0);
if(agg0 != 0.0)
out.setNonZeros(out.getNumColumns());
if( in.isEmptyBlock(false) )
return;
- SparseRow[] aSparseRows = in.getSparseRows();
+ SparseRow[] aSparseRows = in.getSparseBlock();
for (int j = 0; j < aSparseRows.length; ++j)
if( aSparseRows[j]!=null && !aSparseRows[j].isEmpty() )
{
@@ -954,14 +954,14 @@ public class LibMatrixOuterAgg
//allocate and initialize output values (not indices)
out.allocateDenseBlock(true);
- Arrays.fill(out.getDenseArray(), 0, out.getNumColumns(), agg0);
+ Arrays.fill(out.getDenseBlock(), 0, out.getNumColumns(), agg0);
if(agg0 != 0.0)
out.setNonZeros(out.getNumColumns());
if( in.isEmptyBlock(false) )
return;
- SparseRow[] aSparseRows = in.getSparseRows();
+ SparseRow[] aSparseRows = in.getSparseBlock();
for (int j = 0; j < aSparseRows.length; ++j)
if( aSparseRows[j]!=null && !aSparseRows[j].isEmpty() )
{
@@ -1014,14 +1014,14 @@ public class LibMatrixOuterAgg
//allocate and initialize output values (not indices)
out.allocateDenseBlock(true);
- Arrays.fill(out.getDenseArray(), 0, out.getNumColumns(), agg0);
+ Arrays.fill(out.getDenseBlock(), 0, out.getNumColumns(), agg0);
if(agg0 != 0.0)
out.setNonZeros(out.getNumColumns());
if( in.isEmptyBlock(false) )
return;
- SparseRow[] aSparseRows = in.getSparseRows();
+ SparseRow[] aSparseRows = in.getSparseBlock();
for (int j = 0; j < aSparseRows.length; ++j)
if( aSparseRows[j]!=null && !aSparseRows[j].isEmpty() )
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cf7d206c/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
index 545d425..4ea4d1a 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
@@ -365,8 +365,8 @@ public class LibMatrixReorg
out.allocateSparseRowsBlock(false);
for( int i=0; i<rlen; i++ ) {
int ix = vix[i];
- if( in.sparseRows[ix]!=null && !in.sparseRows[ix].isEmpty() ) {
- out.sparseRows[i] = new SparseRow(in.sparseRows[ix]);
+ if( in.sparseBlock[ix]!=null && !in.sparseBlock[ix].isEmpty() ) {
+ out.sparseBlock[i] = new SparseRow(in.sparseBlock[ix]);
}
}
}
@@ -751,8 +751,8 @@ public class LibMatrixReorg
//allocate output arrays (if required)
out.allocateDenseBlock(false);
- double[] a = in.getDenseArray();
- double[] c = out.getDenseArray();
+ double[] a = in.getDenseBlock();
+ double[] c = out.getDenseBlock();
if( m==1 || n==1 ) //VECTOR TRANSPOSE
{
@@ -798,8 +798,8 @@ public class LibMatrixReorg
out.reset(m2, n2, true); //always sparse
out.allocateSparseRowsBlock();
- double[] a = in.getDenseArray();
- SparseRow[] c = out.getSparseRows();
+ double[] a = in.getDenseBlock();
+ SparseRow[] c = out.getSparseBlock();
//blocking according to typical L2 cache sizes
final int blocksizeI = 128;
@@ -841,8 +841,8 @@ public class LibMatrixReorg
out.reset(m2, n2, true); //always sparse
out.allocateSparseRowsBlock();
- SparseRow[] a = in.getSparseRows();
- SparseRow[] c = out.getSparseRows();
+ SparseRow[] a = in.getSparseBlock();
+ SparseRow[] c = out.getSparseBlock();
//initial pass to determine capacity (this helps to prevent
//sparse row reallocations and mem inefficiency w/ skew
@@ -920,8 +920,8 @@ public class LibMatrixReorg
out.reset(m2, n2, false); //always dense
out.allocateDenseBlock();
- SparseRow[] a = in.getSparseRows();
- double[] c = out.getDenseArray();
+ SparseRow[] a = in.getSparseBlock();
+ double[] c = out.getDenseBlock();
if( m==1 ) //ROW VECTOR TRANSPOSE
{
@@ -1020,8 +1020,8 @@ public class LibMatrixReorg
out.nonZeros = in.nonZeros;
out.allocateDenseBlock(false);
- double[] a = in.getDenseArray();
- double[] c = out.getDenseArray();
+ double[] a = in.getDenseBlock();
+ double[] c = out.getDenseBlock();
//copy all rows into target positions
if( n == 1 ) { //column vector
@@ -1051,8 +1051,8 @@ public class LibMatrixReorg
out.allocateSparseRowsBlock(false);
- SparseRow[] a = in.getSparseRows();
- SparseRow[] c = out.getSparseRows();
+ SparseRow[] a = in.getSparseBlock();
+ SparseRow[] c = out.getSparseBlock();
//copy all rows into target positions
for( int i=0; i<m; i++ ) {
@@ -1200,8 +1200,8 @@ public class LibMatrixReorg
int estnnz = (int) (in.nonZeros/rows);
//sparse reshape
- SparseRow[] aRows = in.sparseRows;
- SparseRow[] cRows = out.sparseRows;
+ SparseRow[] aRows = in.sparseBlock;
+ SparseRow[] cRows = out.sparseBlock;
if( rowwise )
{
@@ -1323,13 +1323,13 @@ public class LibMatrixReorg
return;
//allocate block if necessary
- if(out.sparseRows==null)
- out.sparseRows=new SparseRow[rows];
+ if(out.sparseBlock==null)
+ out.sparseBlock=new SparseRow[rows];
int estnnz = (int) (in.nonZeros/rows);
//sparse reshape
double[] a = in.denseBlock;
- SparseRow[] cRows = out.sparseRows;
+ SparseRow[] cRows = out.sparseBlock;
if( rowwise )
{
@@ -1403,14 +1403,14 @@ public class LibMatrixReorg
int clen = in.clen;
//reshape empty block
- if( in.sparseRows == null )
+ if( in.sparseBlock == null )
return;
//allocate block if necessary
out.allocateDenseBlock(false);
//sparse/dense reshape
- SparseRow[] aRows = in.sparseRows;
+ SparseRow[] aRows = in.sparseBlock;
double[] c = out.denseBlock;
if( rowwise )
@@ -1699,7 +1699,7 @@ public class LibMatrixReorg
return;
int rlen = in.rlen;
- SparseRow[] aRows = in.sparseRows;
+ SparseRow[] aRows = in.sparseBlock;
//append all values to right blocks
MatrixIndexes ixtmp = new MatrixIndexes();
@@ -1831,7 +1831,7 @@ public class LibMatrixReorg
if( in.sparse ) //SPARSE
{
- SparseRow[] a = in.sparseRows;
+ SparseRow[] a = in.sparseBlock;
for ( int i=0; i < m; i++ )
if ( a[i] != null && !a[i].isEmpty() ) {
@@ -1872,7 +1872,7 @@ public class LibMatrixReorg
//note: output dense or sparse
for( int i=0, cix=0; i<m; i++ )
if( flags[i] )
- ret.appendRow(cix++, in.sparseRows[i]);
+ ret.appendRow(cix++, in.sparseBlock[i]);
}
else if( !in.sparse && !ret.sparse ) //DENSE <- DENSE
{
@@ -1930,7 +1930,7 @@ public class LibMatrixReorg
flags = new boolean[ n ]; //false
if( in.sparse ) //SPARSE
{
- SparseRow[] a = in.sparseRows;
+ SparseRow[] a = in.sparseBlock;
for( int i=0; i<m; i++ )
if ( a[i] != null && !a[i].isEmpty() ) {
@@ -1978,7 +1978,7 @@ public class LibMatrixReorg
if( in.sparse ) //* <- SPARSE
{
//note: output dense or sparse
- SparseRow[] a = in.sparseRows;
+ SparseRow[] a = in.sparseBlock;
for( int i=0; i<m; i++ )
if ( a[i] != null && !a[i].isEmpty() ) {
[7/7] incubator-systemml git commit: [SYSTEMML-472] New component
test suite for sparse matrix blocks
Posted by mb...@apache.org.
[SYSTEMML-472] New component test suite for sparse matrix blocks
This test suite covers over 100 tests of important SparseBlock
primitives of all SparseBlock implementations. The entire test suite
runs in less than 10s because these tests run purely in-memory and
without the need for R comparisons.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/82d5c5f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/82d5c5f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/82d5c5f1
Branch: refs/heads/master
Commit: 82d5c5f16d581aaa91bcf09854c2cfd6af427072
Parents: 7ce3707
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Jan 17 23:11:48 2016 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sun Jan 17 23:12:50 2016 -0800
----------------------------------------------------------------------
.../functions/sparse/SparseBlockAppendSort.java | 214 ++++++++++++++
.../functions/sparse/SparseBlockDelete.java | 164 +++++++++++
.../sparse/SparseBlockGetFirstIndex.java | 284 +++++++++++++++++++
.../functions/sparse/SparseBlockGetSet.java | 275 ++++++++++++++++++
.../functions/sparse/SparseBlockIndexRange.java | 225 +++++++++++++++
.../functions/sparse/SparseBlockIterator.java | 205 +++++++++++++
.../functions/sparse/SparseBlockScan.java | 157 ++++++++++
.../functions/sparse/ZPackageSuite.java | 42 +++
8 files changed, 1566 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/82d5c5f1/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockAppendSort.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockAppendSort.java b/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockAppendSort.java
new file mode 100644
index 0000000..a563fde
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockAppendSort.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.sparse;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.sysml.runtime.matrix.data.SparseBlock;
+import org.apache.sysml.runtime.matrix.data.SparseBlockCOO;
+import org.apache.sysml.runtime.matrix.data.SparseBlockCSR;
+import org.apache.sysml.runtime.matrix.data.SparseBlockMCSR;
+import org.apache.sysml.runtime.util.LongLongDoubleHashMap;
+import org.apache.sysml.runtime.util.LongLongDoubleHashMap.LLDoubleEntry;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
+
+/**
+ * This is a sparse matrix block component test for append,
+ * and sort functionality. In order to achieve broad coverage, we
+ * test against different init methods and sparsity values.
+ *
+ */
+public class SparseBlockAppendSort extends AutomatedTestBase
+{
+ private final static int rows = 632;
+ private final static int cols = 454;
+ private final static double sparsity1 = 0.11;
+ private final static double sparsity2 = 0.21;
+ private final static double sparsity3 = 0.31;
+
+ private enum InitType {
+ SEQ_SET,
+ RAND_SET,
+ }
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ }
+
+ @Test
+ public void testSparseBlockMCSR1Seq() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.MCSR, sparsity1, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockMCSR2Seq() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.MCSR, sparsity2, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockMCSR3Seq() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.MCSR, sparsity3, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockMCSR1Rand() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.MCSR, sparsity1, InitType.RAND_SET);
+ }
+
+ @Test
+ public void testSparseBlockMCSR2Rand() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.MCSR, sparsity2, InitType.RAND_SET);
+ }
+
+ @Test
+ public void testSparseBlockMCSR3Rand() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.MCSR, sparsity3, InitType.RAND_SET);
+ }
+
+ @Test
+ public void testSparseBlockCSR1Seq() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.CSR, sparsity1, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockCSR2Seq() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.CSR, sparsity2, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockCSR3Seq() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.CSR, sparsity3, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockCSR1Rand() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.CSR, sparsity1, InitType.RAND_SET);
+ }
+
+ @Test
+ public void testSparseBlockCSR2Rand() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.CSR, sparsity2, InitType.RAND_SET);
+ }
+
+ @Test
+ public void testSparseBlockCSR3Rand() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.CSR, sparsity3, InitType.RAND_SET);
+ }
+
+ @Test
+ public void testSparseBlockCOO1Seq() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.COO, sparsity1, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockCOO2Seq() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.COO, sparsity2, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockCOO3Seq() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.COO, sparsity3, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockCOO1Rand() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.COO, sparsity1, InitType.RAND_SET);
+ }
+
+ @Test
+ public void testSparseBlockCOO2Rand() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.COO, sparsity2, InitType.RAND_SET);
+ }
+
+ @Test
+ public void testSparseBlockCOO3Rand() {
+ runSparseBlockAppendSortTest(SparseBlock.Type.COO, sparsity3, InitType.RAND_SET);
+ }
+
+ /**
+ *
+ * @param sparseM1
+ * @param sparseM2
+ * @param instType
+ */
+ private void runSparseBlockAppendSortTest( SparseBlock.Type btype, double sparsity, InitType itype)
+ {
+ try
+ {
+ //data generation
+ double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 7654321);
+
+ //init sparse block
+ SparseBlock sblock = null;
+ switch( btype ) {
+ case MCSR: sblock = new SparseBlockMCSR(rows, cols); break;
+ case CSR: sblock = new SparseBlockCSR(rows, cols); break;
+ case COO: sblock = new SparseBlockCOO(rows, cols); break;
+ }
+
+ if(itype == InitType.SEQ_SET) {
+ for( int i=0; i<rows; i++ )
+ for( int j=0; j<cols; j++ )
+ sblock.append(i, j, A[i][j]);
+ }
+ else if( itype == InitType.RAND_SET ) {
+ LongLongDoubleHashMap map = new LongLongDoubleHashMap();
+ for( int i=0; i<rows; i++ )
+ for( int j=0; j<cols; j++ )
+ map.addValue(i, j, A[i][j]);
+ for( LLDoubleEntry e : map.extractValues() ) //random hash order
+ sblock.append((int)e.key1, (int)e.key2, e.value);
+ }
+
+ //sort appended values
+ sblock.sort();
+
+ //check for correct number of non-zeros
+ int[] rnnz = new int[rows]; int nnz = 0;
+ for( int i=0; i<rows; i++ ) {
+ for( int j=0; j<cols; j++ )
+ rnnz[i] += (A[i][j]!=0) ? 1 : 0;
+ nnz += rnnz[i];
+ }
+ if( nnz != sblock.size() )
+ Assert.fail("Wrong number of non-zeros: "+sblock.size()+", expected: "+nnz);
+
+ //check correct isEmpty return
+ for( int i=0; i<rows; i++ )
+ if( sblock.isEmpty(i) != (rnnz[i]==0) )
+ Assert.fail("Wrong isEmpty(row) result for row nnz: "+rnnz[i]);
+
+ //check correct values
+ for( int i=0; i<rows; i++ )
+ if( !sblock.isEmpty(i) )
+ for( int j=0; j<cols; j++ ) {
+ double tmp = sblock.get(i, j);
+ if( tmp != A[i][j] )
+ Assert.fail("Wrong get value for cell ("+i+","+j+"): "+tmp+", expected: "+A[i][j]);
+ }
+ }
+ catch(Exception ex) {
+ ex.printStackTrace();
+ throw new RuntimeException(ex);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/82d5c5f1/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockDelete.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockDelete.java b/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockDelete.java
new file mode 100644
index 0000000..c719bcb
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockDelete.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.sparse;
+
+import java.util.Iterator;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.SparseBlock;
+import org.apache.sysml.runtime.matrix.data.SparseBlockCOO;
+import org.apache.sysml.runtime.matrix.data.SparseBlockCSR;
+import org.apache.sysml.runtime.matrix.data.SparseBlockMCSR;
+import org.apache.sysml.runtime.matrix.data.SparseRow;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
+
+/**
+ * This is a sparse matrix block component test for sparse block delete
+ * via set functionality. In order to achieve broad coverage, we test against
+ * different sparsity values.
+ *
+ */
+public class SparseBlockDelete extends AutomatedTestBase
+{
+ private final static int rows = 662;
+ private final static int cols = 444;
+ private final static int cl = 145;
+ private final static int cu = 225;
+ private final static double sparsity1 = 0.12;
+ private final static double sparsity2 = 0.22;
+ private final static double sparsity3 = 0.32;
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ }
+
+ @Test
+ public void testSparseBlockMCSR1() {
+ runSparseBlockDeleteTest(SparseBlock.Type.MCSR, sparsity1);
+ }
+
+ @Test
+ public void testSparseBlockMCSR2() {
+ runSparseBlockDeleteTest(SparseBlock.Type.MCSR, sparsity2);
+ }
+
+ @Test
+ public void testSparseBlockMCSR3() {
+ runSparseBlockDeleteTest(SparseBlock.Type.MCSR, sparsity3);
+ }
+
+ @Test
+ public void testSparseBlockCSR1() {
+ runSparseBlockDeleteTest(SparseBlock.Type.CSR, sparsity1);
+ }
+
+ @Test
+ public void testSparseBlockCSR2() {
+ runSparseBlockDeleteTest(SparseBlock.Type.CSR, sparsity2);
+ }
+
+ @Test
+ public void testSparseBlockCSR3() {
+ runSparseBlockDeleteTest(SparseBlock.Type.CSR, sparsity3);
+ }
+
+ @Test
+ public void testSparseBlockCOO1() {
+ runSparseBlockDeleteTest(SparseBlock.Type.COO, sparsity1);
+ }
+
+ @Test
+ public void testSparseBlockCOO2() {
+ runSparseBlockDeleteTest(SparseBlock.Type.COO, sparsity2);
+ }
+
+ @Test
+ public void testSparseBlockCOO3() {
+ runSparseBlockDeleteTest(SparseBlock.Type.COO, sparsity3);
+ }
+
+ /**
+ *
+ * @param btype
+ * @param sparsity
+ */
+ private void runSparseBlockDeleteTest( SparseBlock.Type btype, double sparsity)
+ {
+ try
+ {
+ //data generation
+ double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 456);
+
+ //init sparse block
+ SparseBlock sblock = null;
+ MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A);
+ SparseRow[] srtmp = mbtmp.getSparseBlock();
+ switch( btype ) {
+ case MCSR: sblock = new SparseBlockMCSR(srtmp,true); break;
+ case CSR: sblock = new SparseBlockCSR(srtmp, (int)mbtmp.getNonZeros()); break;
+ case COO: sblock = new SparseBlockCOO(srtmp, (int)mbtmp.getNonZeros()); break;
+ }
+
+ //delete range per row via set
+ for( int i=0; i<rows; i++ )
+ for( int j=cl; j<cu; j++ ) {
+ A[i][j] = 0;
+ sblock.set(i, j, 0);
+ }
+
+ //check for correct number of non-zeros
+ int[] rnnz = new int[rows]; int nnz = 0;
+ for( int i=0; i<rows; i++ ) {
+ for( int j=0; j<cols; j++ )
+ rnnz[i] += (A[i][j]!=0) ? 1 : 0;
+ nnz += rnnz[i];
+ }
+ if( nnz != sblock.size() )
+ Assert.fail("Wrong number of non-zeros: "+sblock.size()+", expected: "+nnz);
+
+ //check correct isEmpty return
+ for( int i=0; i<rows; i++ )
+ if( sblock.isEmpty(i) != (rnnz[i]==0) )
+ Assert.fail("Wrong isEmpty(row) result for row nnz: "+rnnz[i]);
+
+ //check correct values
+ Iterator<IJV> iter = sblock.getIterator();
+ int count = 0;
+ while( iter.hasNext() ) {
+ IJV cell = iter.next();
+ if( cell.v != A[cell.i][cell.j] )
+ Assert.fail("Wrong value returned by iterator: "+cell.v+", expected: "+A[cell.i][cell.j]);
+ count++;
+ }
+ if( count != nnz )
+ Assert.fail("Wrong number of values returned by iterator: "+count+", expected: "+nnz);
+ }
+ catch(Exception ex) {
+ ex.printStackTrace();
+ throw new RuntimeException(ex);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/82d5c5f1/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockGetFirstIndex.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockGetFirstIndex.java b/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockGetFirstIndex.java
new file mode 100644
index 0000000..7e23dd0
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockGetFirstIndex.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.sparse;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.SparseBlock;
+import org.apache.sysml.runtime.matrix.data.SparseBlockCOO;
+import org.apache.sysml.runtime.matrix.data.SparseBlockCSR;
+import org.apache.sysml.runtime.matrix.data.SparseBlockMCSR;
+import org.apache.sysml.runtime.matrix.data.SparseRow;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
+
+/**
+ * This is a sparse matrix block component test for sparse block get
+ * first index functionality. In order to achieve broad coverage, we
+ * test against GT, GTE, and LTE as well as different sparsity values.
+ *
+ */
+public class SparseBlockGetFirstIndex extends AutomatedTestBase
+{
+ private final static int rows = 571;
+ private final static int cols = 595;
+ private final static double sparsity1 = 0.09;
+ private final static double sparsity2 = 0.19;
+ private final static double sparsity3 = 0.29;
+
+ public enum IndexType {
+ GT,
+ GTE,
+ LTE,
+ }
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ }
+
+ @Test
+ public void testSparseBlockMCSR1GT() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.MCSR, sparsity1, IndexType.GT);
+ }
+
+ @Test
+ public void testSparseBlockMCSR2GT() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.MCSR, sparsity2, IndexType.GT);
+ }
+
+ @Test
+ public void testSparseBlockMCSR3GT() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.MCSR, sparsity3, IndexType.GT);
+ }
+
+ @Test
+ public void testSparseBlockMCSR1GTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.MCSR, sparsity1, IndexType.GTE);
+ }
+
+ @Test
+ public void testSparseBlockMCSR2GTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.MCSR, sparsity2, IndexType.GTE);
+ }
+
+ @Test
+ public void testSparseBlockMCSR3GTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.MCSR, sparsity3, IndexType.GTE);
+ }
+
+ @Test
+ public void testSparseBlockMCSR1LTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.MCSR, sparsity1, IndexType.LTE);
+ }
+
+ @Test
+ public void testSparseBlockMCSR2LTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.MCSR, sparsity2, IndexType.LTE);
+ }
+
+ @Test
+ public void testSparseBlockMCSR3LTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.MCSR, sparsity3, IndexType.LTE);
+ }
+
+ @Test
+ public void testSparseBlockCSR1GT() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.CSR, sparsity1, IndexType.GT);
+ }
+
+ @Test
+ public void testSparseBlockCSR2GT() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.CSR, sparsity2, IndexType.GT);
+ }
+
+ @Test
+ public void testSparseBlockCSR3GT() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.CSR, sparsity3, IndexType.GT);
+ }
+
+ @Test
+ public void testSparseBlockCSR1GTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.CSR, sparsity1, IndexType.GTE);
+ }
+
+ @Test
+ public void testSparseBlockCSR2GTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.CSR, sparsity2, IndexType.GTE);
+ }
+
+ @Test
+ public void testSparseBlockCSR3GTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.CSR, sparsity3, IndexType.GTE);
+ }
+
+ @Test
+ public void testSparseBlockCSR1LTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.CSR, sparsity1, IndexType.LTE);
+ }
+
+ @Test
+ public void testSparseBlockCSR2LTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.CSR, sparsity2, IndexType.LTE);
+ }
+
+ @Test
+ public void testSparseBlockCSR3LTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.CSR, sparsity3, IndexType.LTE);
+ }
+
+ @Test
+ public void testSparseBlockCOO1GT() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.COO, sparsity1, IndexType.GT);
+ }
+
+ @Test
+ public void testSparseBlockCOO2GT() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.COO, sparsity2, IndexType.GT);
+ }
+
+ @Test
+ public void testSparseBlockCOO3GT() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.COO, sparsity3, IndexType.GT);
+ }
+
+ @Test
+ public void testSparseBlockCOO1GTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.COO, sparsity1, IndexType.GTE);
+ }
+
+ @Test
+ public void testSparseBlockCOO2GTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.COO, sparsity2, IndexType.GTE);
+ }
+
+ @Test
+ public void testSparseBlockCOO3GTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.COO, sparsity3, IndexType.GTE);
+ }
+
+ @Test
+ public void testSparseBlockCOO1LTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.COO, sparsity1, IndexType.LTE);
+ }
+
+ @Test
+ public void testSparseBlockCOO2LTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.COO, sparsity2, IndexType.LTE);
+ }
+
+ @Test
+ public void testSparseBlockCOO3LTE() {
+ runSparseBlockGetFirstIndexTest(SparseBlock.Type.COO, sparsity3, IndexType.LTE);
+ }
+
+ /**
+ *
+ * @param sparseM1
+ * @param sparseM2
+ * @param instType
+ */
+ private void runSparseBlockGetFirstIndexTest( SparseBlock.Type btype, double sparsity, IndexType itype)
+ {
+ try
+ {
+ //data generation
+ double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 3456);
+
+ //init sparse block
+ SparseBlock sblock = null;
+ MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A);
+ SparseRow[] srtmp = mbtmp.getSparseBlock();
+ switch( btype ) {
+ case MCSR: sblock = new SparseBlockMCSR(srtmp,true); break;
+ case CSR: sblock = new SparseBlockCSR(srtmp, (int)mbtmp.getNonZeros()); break;
+ case COO: sblock = new SparseBlockCOO(srtmp, (int)mbtmp.getNonZeros()); break;
+ }
+
+ //check for correct number of non-zeros
+ int[] rnnz = new int[rows]; int nnz = 0;
+ for( int i=0; i<rows; i++ ) {
+ for( int j=0; j<cols; j++ )
+ rnnz[i] += (A[i][j]!=0) ? 1 : 0;
+ nnz += rnnz[i];
+ }
+ if( nnz != sblock.size() )
+ Assert.fail("Wrong number of non-zeros: "+sblock.size()+", expected: "+nnz);
+
+ //check correct isEmpty return
+ for( int i=0; i<rows; i++ )
+ if( sblock.isEmpty(i) != (rnnz[i]==0) )
+ Assert.fail("Wrong isEmpty(row) result for row nnz: "+rnnz[i]);
+
+ //check correct index values
+ for( int i=0; i<rows; i++ ) {
+ int ix = getFirstIx(A, i, i, itype);
+ int sixpos = -1;
+ switch( itype ) {
+ case GT: sixpos = sblock.posFIndexGT(i, i); break;
+ case GTE: sixpos = sblock.posFIndexGTE(i, i); break;
+ case LTE: sixpos = sblock.posFIndexLTE(i, i); break;
+ }
+ int six = (sixpos>=0) ? sblock.indexes(i)[sixpos] : -1;
+ if( six != ix ) {
+ Assert.fail("Wrong index returned by index probe ("+
+ itype.toString()+","+i+"): "+six+", expected: "+ix);
+ }
+ }
+ }
+ catch(Exception ex) {
+ ex.printStackTrace();
+ throw new RuntimeException(ex);
+ }
+ }
+
+ /**
+ *
+ * @param A
+ * @param rix
+ * @param cix
+ * @param type
+ * @return
+ */
+ private int getFirstIx( double[][] A, int rix, int cix, IndexType type ) {
+ if( type==IndexType.GT ) {
+ for( int j=cix+1; j<cols; j++ )
+ if( A[rix][j] != 0 )
+ return j;
+ return -1;
+ }
+ else if( type==IndexType.GTE ) {
+ for( int j=cix; j<cols; j++ )
+ if( A[rix][j] != 0 )
+ return j;
+ return -1;
+ }
+ else if( type==IndexType.LTE ) {
+ for( int j=cix; j>=0; j-- )
+ if( A[rix][j] != 0 )
+ return j;
+ return -1;
+ }
+
+ return -1;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/82d5c5f1/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockGetSet.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockGetSet.java b/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockGetSet.java
new file mode 100644
index 0000000..9c9856e
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockGetSet.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.sparse;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.SparseBlock;
+import org.apache.sysml.runtime.matrix.data.SparseBlockCOO;
+import org.apache.sysml.runtime.matrix.data.SparseBlockCSR;
+import org.apache.sysml.runtime.matrix.data.SparseBlockMCSR;
+import org.apache.sysml.runtime.matrix.data.SparseRow;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.runtime.util.LongLongDoubleHashMap;
+import org.apache.sysml.runtime.util.LongLongDoubleHashMap.LLDoubleEntry;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
+
+/**
+ * This is a sparse matrix block component test for init, get, set,
+ * and append functionality. In order to achieve broad coverage, we
+ * test against different init methods and sparsity values.
+ *
+ */
+public class SparseBlockGetSet extends AutomatedTestBase
+{
+ private final static int rows = 732;
+ private final static int cols = 354;
+ private final static double sparsity1 = 0.1;
+ private final static double sparsity2 = 0.2;
+ private final static double sparsity3 = 0.3;
+
+ private enum InitType {
+ BULK,
+ SEQ_SET,
+ RAND_SET,
+ }
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ }
+
+ @Test
+ public void testSparseBlockMCSR1Bulk() {
+ runSparseBlockGetSetTest(SparseBlock.Type.MCSR, sparsity1, InitType.BULK);
+ }
+
+ @Test
+ public void testSparseBlockMCSR2Bulk() {
+ runSparseBlockGetSetTest(SparseBlock.Type.MCSR, sparsity2, InitType.BULK);
+ }
+
+ @Test
+ public void testSparseBlockMCSR3Bulk() {
+ runSparseBlockGetSetTest(SparseBlock.Type.MCSR, sparsity3, InitType.BULK);
+ }
+
+ @Test
+ public void testSparseBlockMCSR1Seq() {
+ runSparseBlockGetSetTest(SparseBlock.Type.MCSR, sparsity1, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockMCSR2Seq() {
+ runSparseBlockGetSetTest(SparseBlock.Type.MCSR, sparsity2, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockMCSR3Seq() {
+ runSparseBlockGetSetTest(SparseBlock.Type.MCSR, sparsity3, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockMCSR1Rand() {
+ runSparseBlockGetSetTest(SparseBlock.Type.MCSR, sparsity1, InitType.RAND_SET);
+ }
+
+ @Test
+ public void testSparseBlockMCSR2Rand() {
+ runSparseBlockGetSetTest(SparseBlock.Type.MCSR, sparsity2, InitType.RAND_SET);
+ }
+
+ @Test
+ public void testSparseBlockMCSR3Rand() {
+ runSparseBlockGetSetTest(SparseBlock.Type.MCSR, sparsity3, InitType.RAND_SET);
+ }
+
+ @Test
+ public void testSparseBlockCSR1Bulk() {
+ runSparseBlockGetSetTest(SparseBlock.Type.CSR, sparsity1, InitType.BULK);
+ }
+
+ @Test
+ public void testSparseBlockCSR2Bulk() {
+ runSparseBlockGetSetTest(SparseBlock.Type.CSR, sparsity2, InitType.BULK);
+ }
+
+ @Test
+ public void testSparseBlockCSR3Bulk() {
+ runSparseBlockGetSetTest(SparseBlock.Type.CSR, sparsity3, InitType.BULK);
+ }
+
+ @Test
+ public void testSparseBlockCSR1Seq() {
+ runSparseBlockGetSetTest(SparseBlock.Type.CSR, sparsity1, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockCSR2Seq() {
+ runSparseBlockGetSetTest(SparseBlock.Type.CSR, sparsity2, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockCSR3Seq() {
+ runSparseBlockGetSetTest(SparseBlock.Type.CSR, sparsity3, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockCSR1Rand() {
+ runSparseBlockGetSetTest(SparseBlock.Type.CSR, sparsity1, InitType.RAND_SET);
+ }
+
+ @Test
+ public void testSparseBlockCSR2Rand() {
+ runSparseBlockGetSetTest(SparseBlock.Type.CSR, sparsity2, InitType.RAND_SET);
+ }
+
+ @Test
+ public void testSparseBlockCSR3Rand() {
+ runSparseBlockGetSetTest(SparseBlock.Type.CSR, sparsity3, InitType.RAND_SET);
+ }
+
+ @Test
+ public void testSparseBlockCOO1Bulk() {
+ runSparseBlockGetSetTest(SparseBlock.Type.COO, sparsity1, InitType.BULK);
+ }
+
+ @Test
+ public void testSparseBlockCOO2Bulk() {
+ runSparseBlockGetSetTest(SparseBlock.Type.COO, sparsity2, InitType.BULK);
+ }
+
+ @Test
+ public void testSparseBlockCOO3Bulk() {
+ runSparseBlockGetSetTest(SparseBlock.Type.COO, sparsity3, InitType.BULK);
+ }
+
+ @Test
+ public void testSparseBlockCOO1Seq() {
+ runSparseBlockGetSetTest(SparseBlock.Type.COO, sparsity1, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockCOO2Seq() {
+ runSparseBlockGetSetTest(SparseBlock.Type.COO, sparsity2, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockCOO3Seq() {
+ runSparseBlockGetSetTest(SparseBlock.Type.COO, sparsity3, InitType.SEQ_SET);
+ }
+
+ @Test
+ public void testSparseBlockCOO1Rand() {
+ runSparseBlockGetSetTest(SparseBlock.Type.COO, sparsity1, InitType.RAND_SET);
+ }
+
+ @Test
+ public void testSparseBlockCOO2Rand() {
+ runSparseBlockGetSetTest(SparseBlock.Type.COO, sparsity2, InitType.RAND_SET);
+ }
+
+ @Test
+ public void testSparseBlockCOO3Rand() {
+ runSparseBlockGetSetTest(SparseBlock.Type.COO, sparsity3, InitType.RAND_SET);
+ }
+
+ /**
+ *
+ * @param sparseM1
+ * @param sparseM2
+ * @param instType
+ */
+ private void runSparseBlockGetSetTest( SparseBlock.Type btype, double sparsity, InitType itype)
+ {
+ try
+ {
+ //data generation
+ double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 7654321);
+
+ //init sparse block
+ SparseBlock sblock = null;
+ if( itype == InitType.BULK ) {
+ MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A);
+ SparseRow[] srtmp = mbtmp.getSparseBlock();
+ switch( btype ) {
+ case MCSR: sblock = new SparseBlockMCSR(srtmp,true); break;
+ case CSR: sblock = new SparseBlockCSR(srtmp, (int)mbtmp.getNonZeros()); break;
+ case COO: sblock = new SparseBlockCOO(srtmp, (int)mbtmp.getNonZeros()); break;
+ }
+ }
+ else if( itype == InitType.SEQ_SET || itype == InitType.RAND_SET ) {
+ switch( btype ) {
+ case MCSR: sblock = new SparseBlockMCSR(rows, cols); break;
+ case CSR: sblock = new SparseBlockCSR(rows, cols); break;
+ case COO: sblock = new SparseBlockCOO(rows, cols); break;
+ }
+
+ if(itype == InitType.SEQ_SET) {
+ for( int i=0; i<rows; i++ )
+ for( int j=0; j<cols; j++ )
+ sblock.append(i, j, A[i][j]);
+ }
+ else if( itype == InitType.RAND_SET ) {
+ LongLongDoubleHashMap map = new LongLongDoubleHashMap();
+ for( int i=0; i<rows; i++ )
+ for( int j=0; j<cols; j++ )
+ map.addValue(i, j, A[i][j]);
+ for( LLDoubleEntry e : map.extractValues() ) //random hash order
+ sblock.set((int)e.key1, (int)e.key2, e.value);
+ }
+ }
+
+ //check basic meta data
+ if( sblock.numRows() != rows )
+ Assert.fail("Wrong number of rows: "+sblock.numRows()+", expected: "+rows);
+
+ //check for correct number of non-zeros
+ int[] rnnz = new int[rows]; int nnz = 0;
+ for( int i=0; i<rows; i++ ) {
+ for( int j=0; j<cols; j++ )
+ rnnz[i] += (A[i][j]!=0) ? 1 : 0;
+ nnz += rnnz[i];
+ }
+ if( nnz != sblock.size() )
+ Assert.fail("Wrong number of non-zeros: "+sblock.size()+", expected: "+nnz);
+
+ //check correct isEmpty return
+ for( int i=0; i<rows; i++ )
+ if( sblock.isEmpty(i) != (rnnz[i]==0) )
+ Assert.fail("Wrong isEmpty(row) result for row nnz: "+rnnz[i]);
+
+ //check correct values
+ for( int i=0; i<rows; i++ )
+ if( !sblock.isEmpty(i) )
+ for( int j=0; j<cols; j++ ) {
+ double tmp = sblock.get(i, j);
+ if( tmp != A[i][j] )
+ Assert.fail("Wrong get value for cell ("+i+","+j+"): "+tmp+", expected: "+A[i][j]);
+ }
+ }
+ catch(Exception ex) {
+ ex.printStackTrace();
+ throw new RuntimeException(ex);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/82d5c5f1/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockIndexRange.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockIndexRange.java b/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockIndexRange.java
new file mode 100644
index 0000000..5a4ea4a
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockIndexRange.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.sparse;
+
+import java.util.Arrays;
+import java.util.Iterator;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.SparseBlock;
+import org.apache.sysml.runtime.matrix.data.SparseBlockCOO;
+import org.apache.sysml.runtime.matrix.data.SparseBlockCSR;
+import org.apache.sysml.runtime.matrix.data.SparseBlockMCSR;
+import org.apache.sysml.runtime.matrix.data.SparseRow;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
+
+/**
+ * This is a sparse matrix block component test for sparse block deleteIndexRange
+ * and setIndexRange functionality. In order to achieve broad coverage, we test
+ * against different update types and sparsity values.
+ *
+ */
+public class SparseBlockIndexRange extends AutomatedTestBase
+{
+ private final static int rows = 662;
+ private final static int cols = 549;
+ private final static int cl = 245;
+ private final static int cu = 425;
+ private final static double sparsity1 = 0.12;
+ private final static double sparsity2 = 0.22;
+ private final static double sparsity3 = 0.32;
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ }
+
+ public enum UpdateType {
+ DELETE,
+ INSERT,
+ }
+
+ @Test
+ public void testSparseBlockMCSR1Delete() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.MCSR, sparsity1, UpdateType.DELETE);
+ }
+
+ @Test
+ public void testSparseBlockMCSR2Delete() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.MCSR, sparsity2, UpdateType.DELETE);
+ }
+
+ @Test
+ public void testSparseBlockMCSR3Delete() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.MCSR, sparsity3, UpdateType.DELETE);
+ }
+
+ @Test
+ public void testSparseBlockMCSR1Insert() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.MCSR, sparsity1, UpdateType.INSERT);
+ }
+
+ @Test
+ public void testSparseBlockMCSR2Insert() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.MCSR, sparsity2, UpdateType.INSERT);
+ }
+
+ @Test
+ public void testSparseBlockMCSR3Insert() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.MCSR, sparsity3, UpdateType.INSERT);
+ }
+
+ @Test
+ public void testSparseBlockCSR1Delete() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.CSR, sparsity1, UpdateType.DELETE);
+ }
+
+ @Test
+ public void testSparseBlockCSR2Delete() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.CSR, sparsity2, UpdateType.DELETE);
+ }
+
+ @Test
+ public void testSparseBlockCSR3Delete() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.CSR, sparsity3, UpdateType.DELETE);
+ }
+
+ @Test
+ public void testSparseBlockCSR1Insert() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.CSR, sparsity1, UpdateType.INSERT);
+ }
+
+ @Test
+ public void testSparseBlockCSR2Insert() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.CSR, sparsity2, UpdateType.INSERT);
+ }
+
+ @Test
+ public void testSparseBlockCSR3Insert() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.CSR, sparsity3, UpdateType.INSERT);
+ }
+
+ @Test
+ public void testSparseBlockCOO1Delete() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.COO, sparsity1, UpdateType.DELETE);
+ }
+
+ @Test
+ public void testSparseBlockCOO2Delete() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.COO, sparsity2, UpdateType.DELETE);
+ }
+
+ @Test
+ public void testSparseBlockCOO3Delete() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.COO, sparsity3, UpdateType.DELETE);
+ }
+
+ @Test
+ public void testSparseBlockCOO1Insert() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.COO, sparsity1, UpdateType.INSERT);
+ }
+
+ @Test
+ public void testSparseBlockCOO2Insert() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.COO, sparsity2, UpdateType.INSERT);
+ }
+
+ @Test
+ public void testSparseBlockCOO3Insert() {
+ runSparseBlockIndexRangeTest(SparseBlock.Type.COO, sparsity3, UpdateType.INSERT);
+ }
+
+ /**
+ *
+ * @param btype
+ * @param sparsity
+ */
+ private void runSparseBlockIndexRangeTest( SparseBlock.Type btype, double sparsity, UpdateType utype)
+ {
+ try
+ {
+ //data generation
+ double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 456);
+
+ //init sparse block
+ SparseBlock sblock = null;
+ MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A);
+ SparseRow[] srtmp = mbtmp.getSparseBlock();
+ switch( btype ) {
+ case MCSR: sblock = new SparseBlockMCSR(srtmp,true); break;
+ case CSR: sblock = new SparseBlockCSR(srtmp, (int)mbtmp.getNonZeros()); break;
+ case COO: sblock = new SparseBlockCOO(srtmp, (int)mbtmp.getNonZeros()); break;
+ }
+
+ //delete range per row via set
+ if( utype == UpdateType.DELETE ) {
+ for( int i=0; i<rows; i++ ) {
+ sblock.deleteIndexRange(i, cl, cu);
+ Arrays.fill(A[i], cl, cu, 0);
+ }
+ }
+ else if( utype == UpdateType.INSERT ) {
+ double[] vals = new double[cu-cl];
+ for( int j=cl; j<cu; j++ )
+ vals[j-cl] = j;
+ for( int i=0; i<rows; i++ ) {
+ sblock.setIndexRange(i, cl, cu, vals, 0, cu-cl);
+ System.arraycopy(vals, 0, A[i], cl, cu-cl);
+ }
+ }
+
+ //check for correct number of non-zeros
+ int[] rnnz = new int[rows]; int nnz = 0;
+ for( int i=0; i<rows; i++ ) {
+ for( int j=0; j<cols; j++ )
+ rnnz[i] += (A[i][j]!=0) ? 1 : 0;
+ nnz += rnnz[i];
+ }
+ if( nnz != sblock.size() )
+ Assert.fail("Wrong number of non-zeros: "+sblock.size()+", expected: "+nnz);
+
+ //check correct isEmpty return
+ for( int i=0; i<rows; i++ )
+ if( sblock.isEmpty(i) != (rnnz[i]==0) )
+ Assert.fail("Wrong isEmpty(row) result for row nnz: "+rnnz[i]);
+
+ //check correct values
+ Iterator<IJV> iter = sblock.getIterator();
+ int count = 0;
+ while( iter.hasNext() ) {
+ IJV cell = iter.next();
+ if( cell.v != A[cell.i][cell.j] )
+ Assert.fail("Wrong value returned by iterator: "+cell.v+", expected: "+A[cell.i][cell.j]);
+ count++;
+ }
+ if( count != nnz )
+ Assert.fail("Wrong number of values returned by iterator: "+count+", expected: "+nnz);
+ }
+ catch(Exception ex) {
+ ex.printStackTrace();
+ throw new RuntimeException(ex);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/82d5c5f1/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockIterator.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockIterator.java b/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockIterator.java
new file mode 100644
index 0000000..db388c2
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockIterator.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.sparse;
+
+import java.util.Iterator;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.sysml.runtime.matrix.data.IJV;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.SparseBlock;
+import org.apache.sysml.runtime.matrix.data.SparseBlockCOO;
+import org.apache.sysml.runtime.matrix.data.SparseBlockCSR;
+import org.apache.sysml.runtime.matrix.data.SparseBlockMCSR;
+import org.apache.sysml.runtime.matrix.data.SparseRow;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
+
+/**
+ * This is a sparse matrix block component test for sparse block iterator
+ * functionality. In order to achieve broad coverage, we test against
+ * full and partial iterators as well as different sparsity values.
+ *
+ */
+public class SparseBlockIterator extends AutomatedTestBase
+{
+ private final static int rows = 772;
+ private final static int cols = 394;
+ private final static int rlPartial = 134;
+ private final static double sparsity1 = 0.1;
+ private final static double sparsity2 = 0.2;
+ private final static double sparsity3 = 0.3;
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ }
+
+ @Test
+ public void testSparseBlockMCSR1Full() {
+ runSparseBlockIteratorTest(SparseBlock.Type.MCSR, sparsity1, false);
+ }
+
+ @Test
+ public void testSparseBlockMCSR2Full() {
+ runSparseBlockIteratorTest(SparseBlock.Type.MCSR, sparsity2, false);
+ }
+
+ @Test
+ public void testSparseBlockMCSR3Full() {
+ runSparseBlockIteratorTest(SparseBlock.Type.MCSR, sparsity3, false);
+ }
+
+ @Test
+ public void testSparseBlockMCSR1Partial() {
+ runSparseBlockIteratorTest(SparseBlock.Type.MCSR, sparsity1, true);
+ }
+
+ @Test
+ public void testSparseBlockMCSR2Partial() {
+ runSparseBlockIteratorTest(SparseBlock.Type.MCSR, sparsity2, true);
+ }
+
+ @Test
+ public void testSparseBlockMCSR3Partial() {
+ runSparseBlockIteratorTest(SparseBlock.Type.MCSR, sparsity3, true);
+ }
+
+ @Test
+ public void testSparseBlockCSR1Full() {
+ runSparseBlockIteratorTest(SparseBlock.Type.CSR, sparsity1, false);
+ }
+
+ @Test
+ public void testSparseBlockCSR2Full() {
+ runSparseBlockIteratorTest(SparseBlock.Type.CSR, sparsity2, false);
+ }
+
+ @Test
+ public void testSparseBlockCSR3Full() {
+ runSparseBlockIteratorTest(SparseBlock.Type.CSR, sparsity3, false);
+ }
+
+ @Test
+ public void testSparseBlockCSR1Partial() {
+ runSparseBlockIteratorTest(SparseBlock.Type.CSR, sparsity1, true);
+ }
+
+ @Test
+ public void testSparseBlockCSR2Partial() {
+ runSparseBlockIteratorTest(SparseBlock.Type.CSR, sparsity2, true);
+ }
+
+ @Test
+ public void testSparseBlockCSR3Partial() {
+ runSparseBlockIteratorTest(SparseBlock.Type.CSR, sparsity3, true);
+ }
+
+ @Test
+ public void testSparseBlockCOO1Full() {
+ runSparseBlockIteratorTest(SparseBlock.Type.COO, sparsity1, false);
+ }
+
+ @Test
+ public void testSparseBlockCOO2Full() {
+ runSparseBlockIteratorTest(SparseBlock.Type.COO, sparsity2, false);
+ }
+
+ @Test
+ public void testSparseBlockCOO3Full() {
+ runSparseBlockIteratorTest(SparseBlock.Type.COO, sparsity3, false);
+ }
+
+ @Test
+ public void testSparseBlockCOO1Partial() {
+ runSparseBlockIteratorTest(SparseBlock.Type.COO, sparsity1, true);
+ }
+
+ @Test
+ public void testSparseBlockCOO2Partial() {
+ runSparseBlockIteratorTest(SparseBlock.Type.COO, sparsity2, true);
+ }
+
+ @Test
+ public void testSparseBlockCOO3Partial() {
+ runSparseBlockIteratorTest(SparseBlock.Type.COO, sparsity3, true);
+ }
+
+
+ /**
+ *
+ * @param sparseM1
+ * @param sparseM2
+ * @param instType
+ */
+ private void runSparseBlockIteratorTest( SparseBlock.Type btype, double sparsity, boolean partial)
+ {
+ try
+ {
+ //data generation
+ double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 8765432);
+
+ //init sparse block
+ SparseBlock sblock = null;
+ MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A);
+ SparseRow[] srtmp = mbtmp.getSparseBlock();
+ switch( btype ) {
+ case MCSR: sblock = new SparseBlockMCSR(srtmp,true); break;
+ case CSR: sblock = new SparseBlockCSR(srtmp, (int)mbtmp.getNonZeros()); break;
+ case COO: sblock = new SparseBlockCOO(srtmp, (int)mbtmp.getNonZeros()); break;
+ }
+
+ //check for correct number of non-zeros
+ int[] rnnz = new int[rows]; int nnz = 0;
+ int rl = partial ? rlPartial : 0;
+ for( int i=rl; i<rows; i++ ) {
+ for( int j=0; j<cols; j++ )
+ rnnz[i] += (A[i][j]!=0) ? 1 : 0;
+ nnz += rnnz[i];
+ }
+ if( !partial && nnz != sblock.size() )
+ Assert.fail("Wrong number of non-zeros: "+sblock.size()+", expected: "+nnz);
+
+ //check correct isEmpty return
+ for( int i=rl; i<rows; i++ )
+ if( sblock.isEmpty(i) != (rnnz[i]==0) )
+ Assert.fail("Wrong isEmpty(row) result for row nnz: "+rnnz[i]);
+
+ //check correct values
+ Iterator<IJV> iter = !partial ? sblock.getIterator() :
+ sblock.getIterator(rl, rows);
+ int count = 0;
+ while( iter.hasNext() ) {
+ IJV cell = iter.next();
+ if( cell.v != A[cell.i][cell.j] )
+ Assert.fail("Wrong value returned by iterator: "+cell.v+", expected: "+A[cell.i][cell.j]);
+ count++;
+ }
+ if( count != nnz )
+ Assert.fail("Wrong number of values returned by iterator: "+count+", expected: "+nnz);
+ }
+ catch(Exception ex) {
+ ex.printStackTrace();
+ throw new RuntimeException(ex);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/82d5c5f1/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockScan.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockScan.java b/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockScan.java
new file mode 100644
index 0000000..4f89a17
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/sparse/SparseBlockScan.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.sparse;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.data.SparseBlock;
+import org.apache.sysml.runtime.matrix.data.SparseBlockCOO;
+import org.apache.sysml.runtime.matrix.data.SparseBlockCSR;
+import org.apache.sysml.runtime.matrix.data.SparseBlockMCSR;
+import org.apache.sysml.runtime.matrix.data.SparseRow;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
+
+/**
+ * This is a sparse matrix block component test for sparse block scan
+ * functionality. In order to achieve broad coverage, we test against
+ * different sparsity values.
+ *
+ */
+public class SparseBlockScan extends AutomatedTestBase
+{
+ private final static int rows = 871;
+ private final static int cols = 295;
+ private final static double sparsity1 = 0.09;
+ private final static double sparsity2 = 0.19;
+ private final static double sparsity3 = 0.29;
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ }
+
+ @Test
+ public void testSparseBlockMCSR1Full() {
+ runSparseBlockScanTest(SparseBlock.Type.MCSR, sparsity1);
+ }
+
+ @Test
+ public void testSparseBlockMCSR2Full() {
+ runSparseBlockScanTest(SparseBlock.Type.MCSR, sparsity2);
+ }
+
+ @Test
+ public void testSparseBlockMCSR3Full() {
+ runSparseBlockScanTest(SparseBlock.Type.MCSR, sparsity3);
+ }
+
+ @Test
+ public void testSparseBlockCSR1Full() {
+ runSparseBlockScanTest(SparseBlock.Type.CSR, sparsity1);
+ }
+
+ @Test
+ public void testSparseBlockCSR2Full() {
+ runSparseBlockScanTest(SparseBlock.Type.CSR, sparsity2);
+ }
+
+ @Test
+ public void testSparseBlockCSR3Full() {
+ runSparseBlockScanTest(SparseBlock.Type.CSR, sparsity3);
+ }
+
+ @Test
+ public void testSparseBlockCOO1Full() {
+ runSparseBlockScanTest(SparseBlock.Type.COO, sparsity1);
+ }
+
+ @Test
+ public void testSparseBlockCOO2Full() {
+ runSparseBlockScanTest(SparseBlock.Type.COO, sparsity2);
+ }
+
+ @Test
+ public void testSparseBlockCOO3Full() {
+ runSparseBlockScanTest(SparseBlock.Type.COO, sparsity3);
+ }
+
+ /**
+ *
+ * @param sparseM1
+ * @param sparseM2
+ * @param instType
+ */
+ private void runSparseBlockScanTest( SparseBlock.Type btype, double sparsity)
+ {
+ try
+ {
+ //data generation
+ double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 1234);
+
+ //init sparse block
+ SparseBlock sblock = null;
+ MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A);
+ SparseRow[] srtmp = mbtmp.getSparseBlock();
+ switch( btype ) {
+ case MCSR: sblock = new SparseBlockMCSR(srtmp,true); break;
+ case CSR: sblock = new SparseBlockCSR(srtmp, (int)mbtmp.getNonZeros()); break;
+ case COO: sblock = new SparseBlockCOO(srtmp, (int)mbtmp.getNonZeros()); break;
+ }
+
+ //check for correct number of non-zeros
+ int[] rnnz = new int[rows]; int nnz = 0;
+ for( int i=0; i<rows; i++ ) {
+ for( int j=0; j<cols; j++ )
+ rnnz[i] += (A[i][j]!=0) ? 1 : 0;
+ nnz += rnnz[i];
+ }
+ if( nnz != sblock.size() )
+ Assert.fail("Wrong number of non-zeros: "+sblock.size()+", expected: "+nnz);
+
+ //check correct isEmpty return
+ for( int i=0; i<rows; i++ )
+ if( sblock.isEmpty(i) != (rnnz[i]==0) )
+ Assert.fail("Wrong isEmpty(row) result for row nnz: "+rnnz[i]);
+
+ //check correct values
+ int count = 0;
+ for( int i=0; i<rows; i++) {
+ int alen = sblock.size(i);
+ int apos = sblock.pos(i);
+ int[] aix = sblock.indexes(i);
+ double[] avals = sblock.values(i);
+ for( int j=0; j<alen; j++ ) {
+ if( avals[apos+j] != A[i][aix[apos+j]] )
+ Assert.fail("Wrong value returned by scan: "+avals[apos+j]+", expected: "+A[i][apos+aix[j]]);
+ count++;
+ }
+ }
+ if( count != nnz )
+ Assert.fail("Wrong number of values returned by scan: "+count+", expected: "+nnz);
+ }
+ catch(Exception ex) {
+ ex.printStackTrace();
+ throw new RuntimeException(ex);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/82d5c5f1/src/test_suites/java/org/apache/sysml/test/integration/functions/sparse/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/sparse/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/sparse/ZPackageSuite.java
new file mode 100644
index 0000000..f4d1411
--- /dev/null
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/sparse/ZPackageSuite.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.sparse;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+/** Group together the tests in this package into a single suite so that the Maven build
+ * won't run two of them at once. */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ SparseBlockAppendSort.class,
+ SparseBlockDelete.class,
+ SparseBlockGetFirstIndex.class,
+ SparseBlockGetSet.class,
+ SparseBlockIndexRange.class,
+ SparseBlockIterator.class,
+ SparseBlockScan.class,
+})
+
+
+/** This class is just a holder for the above JUnit annotations. */
+public class ZPackageSuite {
+
+}
[5/7] incubator-systemml git commit: [SYSTEMML-380] New CSR sparse
matrix block
Posted by mb...@apache.org.
[SYSTEMML-380] New CSR sparse matrix block
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/6aa34379
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/6aa34379
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/6aa34379
Branch: refs/heads/master
Commit: 6aa34379fb1ed94db740c1997d3b6afe8e335451
Parents: 34e8221
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Jan 17 23:05:08 2016 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sun Jan 17 23:12:34 2016 -0800
----------------------------------------------------------------------
.../runtime/matrix/data/SparseBlockCSR.java | 439 +++++++++++++++++++
1 file changed, 439 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6aa34379/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockCSR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockCSR.java b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockCSR.java
new file mode 100644
index 0000000..1f7a890
--- /dev/null
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockCSR.java
@@ -0,0 +1,439 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.apache.sysml.runtime.matrix.data;
+
+import java.util.Arrays;
+
+import org.apache.sysml.runtime.util.SortUtils;
+
+/**
+ * SparseBlock implementation that realizes a traditional 'compressed sparse row'
+ * representation, where the entire sparse block is stored as three arrays: ptr
+ * of length rlen+1 to store offsets per row, and indexes/values of length nnz
+ * to store column indexes and values of non-zero entries. This format is very
+ * memory efficient for sparse (but not ultra-sparse) matrices and provides very
+ * good performance for common operations, partially due to lower memory bandwidth
+ * requirements. However, this format is slow on incremental construction (because
+ * it does not allow append/sort per row) without reshifting. Finally, the total
+ * nnz is limited to INTEGER_MAX, whereas for SparseBlockMCSR only the nnz per
+ * row are limited to INTEGER_MAX.
+ *
+ * TODO: extensions for faster incremental construction (e.g., max row)
+ * TODO more efficient fused setIndexRange impl to avoid repeated copies and updates
+ *
+ */
+public class SparseBlockCSR extends SparseBlock
+{
+ private static final long serialVersionUID = 1922673868466164244L;
+
+ private int[] _ptr = null; //row pointer array (size: rlen+1)
+ private int[] _indexes = null; //column index array (size: >=nnz)
+ private double[] _values = null; //value array (size: >=nnz)
+ private int _size = 0; //actual number of nnz
+
+ public SparseBlockCSR(int rlen) {
+ this(rlen, INIT_CAPACITY);
+ }
+
+ public SparseBlockCSR(int rlen, int capacity) {
+ _ptr = new int[rlen+1]; //ix0=0
+ _indexes = new int[capacity];
+ _values = new double[capacity];
+ _size = 0;
+ }
+
+ /**
+ * Copy constructor old sparse row representation.
+ */
+ public SparseBlockCSR(SparseRow[] rows, int nnz)
+ {
+ int rlen = rows.length;
+
+ _ptr = new int[rlen+1]; //ix0=0
+ _indexes = new int[nnz];
+ _values = new double[nnz];
+ _size = nnz;
+
+ for( int i=0, pos=0; i<rlen; i++ ) {
+ int alen = rows[i].size();
+ int[] aix = rows[i].getIndexContainer();
+ double[] avals = rows[i].getValueContainer();
+ for( int j=0; j<alen; j++ ) {
+ _indexes[pos] = aix[j];
+ _values[pos] = avals[j];
+ pos++;
+ }
+ _ptr[i+1]=pos;
+ }
+ }
+
+ @Override
+ public void allocate(int r) {
+ //do nothing everything preallocated
+ }
+
+ @Override
+ public int numRows() {
+ return _ptr.length-1;
+ }
+
+ @Override
+ public boolean isThreadSafe() {
+ return false;
+ }
+
+ @Override
+ public long size() {
+ return _size;
+ }
+
+ @Override
+ public int size(int r) {
+ return _ptr[r+1] - _ptr[r];
+ }
+
+ @Override
+ public boolean isEmpty(int r) {
+ return (_ptr[r+1] - _ptr[r] == 0);
+ }
+
+ @Override
+ public int[] indexes(int r) {
+ return _indexes;
+ }
+
+ @Override
+ public double[] values(int r) {
+ return _values;
+ }
+
+ @Override
+ public int pos(int r) {
+ return _ptr[r];
+ }
+
+ @Override
+ public boolean set(int r, int c, double v) {
+ int pos = pos(r);
+ int len = size(r);
+
+ //search for existing col index
+ int index = Arrays.binarySearch(_indexes, pos, pos+len, c);
+ if( index >= 0 ) {
+ //delete/overwrite existing value (on value delete, we shift
+ //left for (1) correct nnz maintenance, and (2) smaller size)
+ if( v == 0 ) {
+ shiftLeftAndDelete(index);
+ decrPtr(r+1);
+ return true; // nnz--
+ }
+ else {
+ _values[index] = v;
+ return false;
+ }
+ }
+
+ //early abort on zero (if no overwrite)
+ if( v==0 ) return false;
+
+ //insert new index-value pair
+ index = Math.abs( index+1 );
+ if( _size==_values.length )
+ resizeAndInsert(index, c, v);
+ else
+ shiftRightAndInsert(index, c, v);
+ incrPtr(r+1);
+ return true; // nnz++
+ }
+
+ @Override
+ public void append(int r, int c, double v) {
+ //early abort on zero
+ if( v==0 ) return;
+
+ int pos = pos(r);
+ int len = size(r);
+ if( pos+len == _size ) {
+ //resize and append
+ if( _size==_values.length )
+ resize();
+ insert(_size, c, v);
+ }
+ else {
+ //resize, shift and insert
+ if( _size==_values.length )
+ resizeAndInsert(pos+len, c, v);
+ else
+ shiftRightAndInsert(pos+len, c, v);
+ }
+ incrPtr(r+1);
+ }
+
+ @Override
+ public void setIndexRange(int r, int cl, int cu, double[] v, int vix, int vlen) {
+ //delete existing values in range if necessary
+ deleteIndexRange(r, cl, cu);
+
+ //determine input nnz
+ int lnnz = 0;
+ for( int i=vix; i<vix+vlen; i++ )
+ lnnz += ( v[i] != 0 ) ? 1 : 0;
+
+ //prepare free space (allocate and shift)
+ int lsize = _size+lnnz;
+ if( _values.length < lsize )
+ resize(lsize);
+ int index = posFIndexGT(r, cl);
+ shiftRightByN((index>0)?index:pos(r+1), lnnz);
+
+ //insert values
+ for( int i=vix; i<vix+vlen; i++ )
+ if( v[i] != 0 ) {
+ _indexes[ index ] = cl+i-vix;
+ _values[ index ] = v[i];
+ index++;
+ }
+ incrPtr(r+1, lnnz);
+ }
+
+ @Override
+ public void deleteIndexRange(int r, int cl, int cu) {
+ int start = posFIndexGTE(r,cl);
+ if( start < 0 ) //nothing to delete
+ return;
+
+ int len = size(r);
+ int end = posFIndexGTE(r, cu);
+ if( end < 0 ) //delete all remaining
+ end = start+len;
+
+ //overlapping array copy (shift rhs values left)
+ System.arraycopy(_indexes, end, _indexes, start, _size-end);
+ System.arraycopy(_values, end, _values, start, _size-end);
+ _size -= (end-start);
+
+ decrPtr(r+1, end-start);
+ }
+
+ @Override
+ public void sort() {
+ int rlen = numRows();
+ for( int i=0; i<rlen && pos(i)<_size; i++ )
+ sort(i);
+ }
+
+ @Override
+ public void sort(int r) {
+ int pos = pos(r);
+ int len = size(r);
+
+ if( len<=100 || !SortUtils.isSorted(pos, pos+len, _indexes) )
+ SortUtils.sortByIndex(pos, pos+len, _indexes, _values);
+ }
+
+ @Override
+ public double get(int r, int c) {
+ int pos = pos(r);
+ int len = size(r);
+
+ //search for existing col index in [pos,pos+len)
+ int index = Arrays.binarySearch(_indexes, pos, pos+len, c);
+ return (index >= 0) ? _values[index] : 0;
+ }
+
+ @Override
+ public int posFIndexLTE(int r, int c) {
+ int pos = pos(r);
+ int len = size(r);
+
+ //search for existing col index in [pos,pos+len)
+ int index = Arrays.binarySearch(_indexes, pos, pos+len, c);
+ if( index >= 0 )
+ return (index < pos+len) ? index : -1;
+
+ //search lt col index (see binary search)
+ index = Math.abs( index+1 );
+ return (index-1 >= pos) ? index-1 : -1;
+ }
+
+ @Override
+ public int posFIndexGTE(int r, int c) {
+ int pos = pos(r);
+ int len = size(r);
+
+ //search for existing col index
+ int index = Arrays.binarySearch(_indexes, pos, pos+len, c);
+ if( index >= 0 )
+ return (index < pos+len) ? index : -1;
+
+ //search gt col index (see binary search)
+ index = Math.abs( index+1 );
+ return (index < pos+len) ? index : -1;
+ }
+
+ @Override
+ public int posFIndexGT(int r, int c) {
+ int pos = pos(r);
+ int len = size(r);
+
+ //search for existing col index
+ int index = Arrays.binarySearch(_indexes, pos, pos+len, c);
+ if( index >= 0 )
+ return (index+1 < pos+len) ? index+1 : -1;
+
+ //search gt col index (see binary search)
+ index = Math.abs( index+1 );
+ return (index < pos+len) ? index : -1;
+ }
+
+ ///////////////////////////
+ // private helper methods
+
+ /**
+ *
+ */
+ private void resize() {
+ //compute new size
+ double tmpCap = _values.length * RESIZE_FACTOR1;
+ int newCap = (int)Math.min(tmpCap, Integer.MAX_VALUE);
+
+ resize(newCap);
+ }
+
+ private void resize(int capacity) {
+ //reallocate arrays and copy old values
+ _indexes = Arrays.copyOf(_indexes, capacity);
+ _values = Arrays.copyOf(_values, capacity);
+ }
+
+ /**
+ *
+ * @param ix
+ * @param c
+ * @param v
+ */
+ private void resizeAndInsert(int ix, int c, double v) {
+ //compute new size
+ double tmpCap = _values.length * RESIZE_FACTOR1;
+ int newCap = (int)Math.min(tmpCap, Integer.MAX_VALUE);
+
+ int[] oldindexes = _indexes;
+ double[] oldvalues = _values;
+ _indexes = new int[newCap];
+ _values = new double[newCap];
+
+ //copy lhs values to new array
+ System.arraycopy(oldindexes, 0, _indexes, 0, ix);
+ System.arraycopy(oldvalues, 0, _values, 0, ix);
+
+ //copy rhs values to new array
+ System.arraycopy(oldindexes, ix, _indexes, ix+1, _size-ix);
+ System.arraycopy(oldvalues, ix, _values, ix+1, _size-ix);
+
+ //insert new value
+ insert(ix, c, v);
+ }
+
+ /**
+ *
+ * @param ix
+ * @param c
+ * @param v
+ */
+ private void shiftRightAndInsert(int ix, int c, double v) {
+ //overlapping array copy (shift rhs values right by 1)
+ System.arraycopy(_indexes, ix, _indexes, ix+1, _size-ix);
+ System.arraycopy(_values, ix, _values, ix+1, _size-ix);
+
+ //insert new value
+ insert(ix, c, v);
+ }
+
+ /**
+ *
+ * @param index
+ */
+ private void shiftLeftAndDelete(int ix)
+ {
+ //overlapping array copy (shift rhs values left by 1)
+ System.arraycopy(_indexes, ix+1, _indexes, ix, _size-ix-1);
+ System.arraycopy(_values, ix+1, _values, ix, _size-ix-1);
+ _size--;
+ }
+
+ private void shiftRightByN(int ix, int n)
+ {
+ //overlapping array copy (shift rhs values right by 1)
+ System.arraycopy(_indexes, ix, _indexes, ix+n, _size-ix);
+ System.arraycopy(_values, ix, _values, ix+n, _size-ix);
+ _size += n;
+ }
+
+ /**
+ *
+ * @param ix
+ * @param c
+ * @param v
+ */
+ private void insert(int ix, int c, double v) {
+ _indexes[ix] = c;
+ _values[ix] = v;
+ _size++;
+ }
+
+ /**
+ *
+ * @param rl
+ */
+ private void incrPtr(int rl) {
+ incrPtr(rl, 1);
+ }
+
+ /**
+ *
+ * @param rl
+ * @param cnt
+ */
+ private void incrPtr(int rl, int cnt) {
+ int rlen = numRows();
+ for( int i=rl; i<rlen+1; i++ )
+ _ptr[i]+=cnt;
+ }
+
+ /**
+ *
+ * @param rl
+ */
+ private void decrPtr(int rl) {
+ decrPtr(rl, 1);
+ }
+
+ /**
+ *
+ * @param rl
+ * @param cnt
+ */
+ private void decrPtr(int rl, int cnt) {
+ int rlen = numRows();
+ for( int i=rl; i<rlen+1; i++ )
+ _ptr[i]-=cnt;
+ }
+}