You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/03/27 22:41:22 UTC
[1/4] incubator-systemml git commit: Cache-conscious sparse-dense
core block matrix multiplication
Repository: incubator-systemml
Updated Branches:
refs/heads/master 02aadb80a -> 42e471079
Cache-conscious sparse-dense core block matrix multiplication
This patch makes the sparse-dense block matrix multiply cache-conscious
by leveraging the same blocking scheme as dense-dense matrix multiply in
combination with a tiny temporary array of positions in A (which memory
overhead is bounded by 128B per thread). On a variety of testcases with
varying sparsity and matrix shapes, we observed improvements of up to
3.5x without degradation for small or very sparse matrices.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/67a2954b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/67a2954b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/67a2954b
Branch: refs/heads/master
Commit: 67a2954b7af6a6d2bd1370109994d79818854dde
Parents: 02aadb8
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sat Mar 26 00:19:59 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sat Mar 26 21:57:23 2016 -0700
----------------------------------------------------------------------
.../runtime/matrix/data/LibMatrixMult.java | 71 +++++++++++---------
1 file changed, 40 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/67a2954b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
index e47ae03..5f8f649 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
@@ -1323,39 +1323,48 @@ public class LibMatrixMult
}
}
else //MATRIX-MATRIX
- {
- for( int i=rl, cix=rl*n; i<ru; i++, cix+=n )
- {
- if( !a.isEmpty(i) )
- {
- int apos = a.pos(i);
- int alen = a.size(i);
- int[] aix = a.indexes(i);
- double[] avals = a.values(i);
-
- if( alen==1 && avals[0]==1 ) //ROW SELECTION
- {
- //plain memcopy for permutation matrices
- System.arraycopy(b, aix[0]*n, c, cix, n);
- }
- else //GENERAL CASE
- {
- //rest not aligned to blocks of 4 rows
- final int bn = alen % 4;
- switch( bn ){
- case 1: vectMultiplyAdd(avals[apos], b, c, aix[apos]*n, cix, n); break;
- case 2: vectMultiplyAdd2(avals[apos],avals[apos+1], b, c, aix[apos]*n, aix[apos+1]*n, cix, n); break;
- case 3: vectMultiplyAdd3(avals[apos],avals[apos+1],avals[apos+2], b, c, aix[apos]*n, aix[apos+1]*n, aix[apos+2]*n, cix, n); break;
- }
-
- //compute blocks of 4 rows (core inner loop)
- for( int k = apos+bn; k<apos+alen; k+=4 ) {
- vectMultiplyAdd4( avals[k], avals[k+1], avals[k+2], avals[k+3], b, c,
- aix[k]*n, aix[k+1]*n, aix[k+2]*n, aix[k+3]*n, cix, n );
- }
+ {
+ //blocksizes to fit blocks of B (dense) and several rows of A/C in common L2 cache size,
+ //while blocking A/C for L1/L2 yet allowing long scans (2 pages) in the inner loop over j
+ final int blocksizeI = 32;
+ final int blocksizeK = 24;
+ final int blocksizeJ = 1024;
+
+ //temporary array of current sparse positions
+ int[] curk = new int[blocksizeI];
+
+ //blocked execution over IKJ
+ for( int bi = rl; bi < ru; bi+=blocksizeI ) {
+ Arrays.fill(curk, 0); //reset positions
+ for( int bk = 0, bimin = Math.min(ru, bi+blocksizeI); bk < cd; bk+=blocksizeK ) {
+ for( int bj = 0, bkmin = Math.min(cd, bk+blocksizeK); bj < n; bj+=blocksizeJ ) {
+ int bjlen = Math.min(n, bj+blocksizeJ)-bj;
+
+ //core sub block matrix multiplication
+ for( int i=bi, cix=bi*n+bj; i<bimin; i++, cix+=n ) {
+ if( !a.isEmpty(i) ) {
+ int apos = a.pos(i);
+ int alen = a.size(i);
+ int[] aix = a.indexes(i);
+ double[] avals = a.values(i);
+
+ int k = curk[i-bi] + apos;
+ //rest not aligned to blocks of 4 rows
+ int bn = alen%4;
+ for( ; k<apos+bn && aix[k]<bkmin; k++ )
+ vectMultiplyAdd(avals[k], b, c, aix[k]*n+bj, cix, bjlen);
+ //compute blocks of 4 rows (core inner loop), allowed to exceed bkmin
+ for( ; k<apos+alen && aix[k]<bkmin; k+=4 )
+ vectMultiplyAdd4( avals[k], avals[k+1], avals[k+2], avals[k+3], b, c,
+ aix[k]*n+bj, aix[k+1]*n+bj, aix[k+2]*n+bj, aix[k+3]*n+bj, cix, bjlen );
+ //update positions on last bj block
+ if( bj+bjlen==n )
+ curk[i-bi] = k - apos;
+ }
+ }
}
}
- }
+ }
}
}
else
[2/4] incubator-systemml git commit: [SYSTEMML-533] Fix variable name
placeholder for instruction patching
Posted by mb...@apache.org.
[SYSTEMML-533] Fix variable name placeholder for instruction patching
Whenever instructions expect constant value arguments but we have
variables during compilation, we generate placeholders and rely on
instruction patching during runtime. So far we used ## which runs into
collisions if used as literals. This patch changes this placeholder to
the extended ASCII character \u00b6 to avoid collisions and cleans up
remaining code that directly referred to the placeholder instead of the
global compiler constant.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/e0356496
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/e0356496
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/e0356496
Branch: refs/heads/master
Commit: e0356496adc2070c2d532fd242c7dbf263ab73e4
Parents: 67a2954
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sat Mar 26 01:07:44 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sat Mar 26 21:57:30 2016 -0700
----------------------------------------------------------------------
.../apache/sysml/hops/cost/CostEstimator.java | 12 ++--
.../java/org/apache/sysml/lops/LeftIndex.java | 35 -----------
src/main/java/org/apache/sysml/lops/Lop.java | 14 +++--
.../java/org/apache/sysml/lops/Ternary.java | 8 ---
.../ExternalFunctionProgramBlock.java | 6 --
.../org/apache/sysml/test/utils/TestUtils.java | 64 --------------------
6 files changed, 14 insertions(+), 125 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e0356496/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
index 00be756..00790a4 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
@@ -419,8 +419,8 @@ public abstract class CostEstimator
{
String[] parts = InstructionUtils.getInstructionParts(st.nextToken());
byte outIndex = Byte.parseByte(parts[2]);
- long rlen = parts[3].contains("##")?-1:UtilFunctions.parseToLong(parts[3]);
- long clen = parts[4].contains("##")?-1:UtilFunctions.parseToLong(parts[4]);
+ long rlen = parts[3].contains(Lop.VARIABLE_NAME_PLACEHOLDER)?-1:UtilFunctions.parseToLong(parts[3]);
+ long clen = parts[4].contains(Lop.VARIABLE_NAME_PLACEHOLDER)?-1:UtilFunctions.parseToLong(parts[4]);
long brlen = Long.parseLong(parts[5]);
long bclen = Long.parseLong(parts[6]);
long nnz = (long) (Double.parseDouble(parts[9]) * rlen * clen);
@@ -501,9 +501,9 @@ public abstract class CostEstimator
protected String replaceInstructionPatch( String inst )
{
String ret = inst;
- while( ret.contains("##") ) {
- int index1 = ret.indexOf("##");
- int index2 = ret.indexOf("##", index1+3);
+ while( ret.contains(Lop.VARIABLE_NAME_PLACEHOLDER) ) {
+ int index1 = ret.indexOf(Lop.VARIABLE_NAME_PLACEHOLDER);
+ int index2 = ret.indexOf(Lop.VARIABLE_NAME_PLACEHOLDER, index1+3);
String replace = ret.substring(index1,index2+2);
ret = ret.replaceAll(replace, "1");
}
@@ -622,7 +622,7 @@ public abstract class CostEstimator
attr = new String[]{String.valueOf(paramsMap.get("margin").equals("rows")?0:1)};
}
- vs[0] = stats.get( parts[1].substring(7).replaceAll("##", "") );
+ vs[0] = stats.get( parts[1].substring(7).replaceAll(Lop.VARIABLE_NAME_PLACEHOLDER, "") );
vs[1] = _unknownStats; //TODO
vs[2] = stats.get( parts[parts.length-1] );
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e0356496/src/main/java/org/apache/sysml/lops/LeftIndex.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/LeftIndex.java b/src/main/java/org/apache/sysml/lops/LeftIndex.java
index 5168421..0adf769 100644
--- a/src/main/java/org/apache/sysml/lops/LeftIndex.java
+++ b/src/main/java/org/apache/sysml/lops/LeftIndex.java
@@ -137,41 +137,6 @@ public class LeftIndex extends Lop
return sb.toString();
}
-/* @Override
- public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int input_index5, int output_index)
- throws LopsException {
-
- * Example: B = A[row_l:row_u, col_l:col_u]
- * A - input matrix (input_index1)
- * row_l - lower bound in row dimension
- * row_u - upper bound in row dimension
- * col_l - lower bound in column dimension
- * col_u - upper bound in column dimension
- *
- * Since row_l,row_u,col_l,col_u are scalars, values for input_index(2,3,4,5)
- * will be equal to -1. They should be ignored and the scalar value labels must
- * be derived from input lops.
-
- String rowl = this.getInputs().get(1).getOutputParameters().getLabel();
- if (this.getInputs().get(1).getExecLocation() != ExecLocation.Data
- || !((Data) this.getInputs().get(1)).isLiteral())
- rowl = "##" + rowl + "##";
- String rowu = this.getInputs().get(2).getOutputParameters().getLabel();
- if (this.getInputs().get(2).getExecLocation() != ExecLocation.Data
- || !((Data) this.getInputs().get(2)).isLiteral())
- rowu = "##" + rowu + "##";
- String coll = this.getInputs().get(3).getOutputParameters().getLabel();
- if (this.getInputs().get(3).getExecLocation() != ExecLocation.Data
- || !((Data) this.getInputs().get(3)).isLiteral())
- coll = "##" + coll + "##";
- String colu = this.getInputs().get(4).getOutputParameters().getLabel();
- if (this.getInputs().get(4).getExecLocation() != ExecLocation.Data
- || !((Data) this.getInputs().get(4)).isLiteral())
- colu = "##" + colu + "##";
-
- return getInstructions(Integer.toString(input_index1), rowl, rowu, coll, colu, Integer.toString(output_index));
- }
-*/
@Override
public String toString() {
return "leftIndex";
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e0356496/src/main/java/org/apache/sysml/lops/Lop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Lop.java b/src/main/java/org/apache/sysml/lops/Lop.java
index bed9d10..1c2bd5b 100644
--- a/src/main/java/org/apache/sysml/lops/Lop.java
+++ b/src/main/java/org/apache/sysml/lops/Lop.java
@@ -77,13 +77,15 @@ public abstract class Lop
public static final String FILE_SEPARATOR = "/";
public static final String PROCESS_PREFIX = "_p";
- public static final String INSTRUCTION_DELIMITOR = "\u2021"; // "\u002c"; //",";
- public static final String OPERAND_DELIMITOR = "\u00b0"; //\u2021"; //00ea"; //"::#::";
- public static final String VALUETYPE_PREFIX = "\u00b7" ; //":#:";
- public static final String DATATYPE_PREFIX = VALUETYPE_PREFIX; //":#:";
- public static final String LITERAL_PREFIX = VALUETYPE_PREFIX; //":#:";
+ //special delimiters w/ extended ASCII characters to avoid collisions
+ public static final String INSTRUCTION_DELIMITOR = "\u2021";
+ public static final String OPERAND_DELIMITOR = "\u00b0";
+ public static final String VALUETYPE_PREFIX = "\u00b7" ;
+ public static final String DATATYPE_PREFIX = VALUETYPE_PREFIX;
+ public static final String LITERAL_PREFIX = VALUETYPE_PREFIX;
+ public static final String VARIABLE_NAME_PLACEHOLDER = "\u00b6";
+
public static final String NAME_VALUE_SEPARATOR = "="; // e.g., used in parameterized builtins
- public static final String VARIABLE_NAME_PLACEHOLDER = "##"; //TODO: use in LOPs
public static final String MATRIX_VAR_NAME_PREFIX = "_mVar";
public static final String SCALAR_VAR_NAME_PREFIX = "_Var";
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e0356496/src/main/java/org/apache/sysml/lops/Ternary.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/Ternary.java b/src/main/java/org/apache/sysml/lops/Ternary.java
index 43f19bf..e697cc5 100644
--- a/src/main/java/org/apache/sysml/lops/Ternary.java
+++ b/src/main/java/org/apache/sysml/lops/Ternary.java
@@ -221,10 +221,6 @@ public class Ternary extends Lop
throw new LopsException(this.printErrorLocation() + "In Tertiary Lop, Unexpected input while computing the instructions for op: " + operation + " \n");
}
- // parse the third input (scalar)
- // if it is a literal, copy val, else surround with the label with
- // ## symbols. these will be replaced at runtime.
-
int scalarIndex = 2; // index of the scalar input
sb.append( "ctabletransformscalarweight" );
@@ -248,10 +244,6 @@ public class Ternary extends Lop
throw new LopsException(this.printErrorLocation() + "In Tertiary Lop, Unexpected input while computing the instructions for op: " + operation + " \n");
}
- // parse the third input (scalar)
- // if it is a literal, copy val, else surround with the label with
- // ## symbols. these will be replaced at runtime.
-
int scalarIndex2 = 1; // index of the scalar input
int scalarIndex3 = 2; // index of the scalar input
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e0356496/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
index b36bd59..af1671a 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
@@ -492,12 +492,6 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
// create a GMR job that transforms each of these matrices from block to cell
for (int i = 0; i < matrices.size(); i++) {
- //inputs[i] = "##" + matrices.get(i).getName() + "##";
- //inputInfo[i] = binBlockInputInfo;
- //outputInfo[i] = textCellOutputInfo;
- //numRows[i] = numCols[i] = numRowsPerBlock[i] = numColsPerBlock[i] = -1;
- //resultDimsUnknown[i] = 1;
-
inLabels.add(matrices.get(i).getName());
outLabels.add(matrices.get(i).getName()+"_extFnInput");
resultIndex[i] = (byte) i; //(matrices.size()+i);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e0356496/src/test/java/org/apache/sysml/test/utils/TestUtils.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/utils/TestUtils.java b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
index e6e7c12..74a5667 100644
--- a/src/test/java/org/apache/sysml/test/utils/TestUtils.java
+++ b/src/test/java/org/apache/sysml/test/utils/TestUtils.java
@@ -44,8 +44,6 @@ import java.util.Iterator;
import java.util.Locale;
import java.util.Random;
import java.util.StringTokenizer;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
@@ -88,9 +86,6 @@ public class TestUtils
private static ArrayList<String> _AssertInfos = new ArrayList<String>();
private static boolean _AssertOccured = false;
- /** String used to replace variables in scripts */
- private static String _RS = "\\$\\$";
-
/**
* <p>
* Compares to arrays for equality. The elements in the array can be in
@@ -1635,65 +1630,6 @@ public class TestUtils
/**
* <p>
- * Replaces variables in a DML or R script with the specified values. A
- * variable of format ##name## will be replaced where the name is used to
- * identify the variable in the hashmap containing the belonging value.
- * </p>
- *
- * @param strScriptDirectory
- * directory which contains the DML script
- * @param strScriptFile
- * filename of the DML script
- * @param variables
- * hashmap containing all the variables and their replacements
- * @deprecated Use ParameterBuilder.setVariablesInScript instead
- */
- public static void setVariablesInScript(String strScriptDirectory, String strScriptFile,
- HashMap<String, String> variables) {
- try {
- String strScript = strScriptDirectory + strScriptFile;
- String strTmpScript = strScript + "t";
-
- BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(strScript)));
- FileOutputStream out = new FileOutputStream(strTmpScript);
- PrintWriter pw = new PrintWriter(out);
- String content;
- Pattern unresolvedVars = Pattern.compile(_RS + ".*" + _RS);
- /**
- * sothat variables, which were not assigned, are replaced by an
- * empty string
- */
- while ((content = in.readLine()) != null) {
- for (String variable : variables.keySet()) {
- Pattern pattern = Pattern.compile(_RS + variable + _RS);
- Matcher matcher = pattern.matcher(content);
- while (matcher.find()) {
- content = content.replaceFirst(matcher.group().replace("$", "\\$"), variables.get(variable));
- }
- }
- Matcher matcher = unresolvedVars.matcher(content);
- content = matcher.replaceAll("");
- pw.println(content);
- }
- pw.close();
- out.close();
- in.close();
-
- /*
- * // remove checksum files if created Path crcFile = new
- * Path(dmlScriptDirectory + "." + dmlScriptFile + ".crc"); if
- * (fs.exists(crcFile)) fs.delete(crcFile, false); crcFile = new
- * Path(dmlScriptDirectory + "." + dmlScriptFile + "t.crc"); if
- * (fs.exists(crcFile)) fs.delete(crcFile, false);
- */
- } catch (IOException e) {
- e.printStackTrace();
- fail("unable to set variables in dml script: " + e.getMessage());
- }
- }
-
- /**
- * <p>
* Prints out a DML script.
* </p>
*
[3/4] incubator-systemml git commit: Cache-conscious sparse-sparse
core block matrix multiplication
Posted by mb...@apache.org.
Cache-conscious sparse-sparse core block matrix multiplication
In a similar spirit as the new sparse-dense cache-conscious matrix
multiply, this patch also makes sparse-sparse matrix multiply
cache-conscious. However, due to the sparse inner loop, we use a best
effort cache blocking with dynamic block sizes computed from the
sparsity of A to reflect average row reuse. Overall, we've seen
performance improvements of up to 2x without degradation over a variety
of usecases with different sparsity and shapes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/5e9782ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/5e9782ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/5e9782ff
Branch: refs/heads/master
Commit: 5e9782ffcb46f7718c57cdce5b924fa6c85efeff
Parents: e035649
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sat Mar 26 21:32:39 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sat Mar 26 21:57:37 2016 -0700
----------------------------------------------------------------------
.../runtime/matrix/data/LibMatrixMult.java | 52 +++++++++++---------
.../sysml/runtime/util/UtilFunctions.java | 9 +++-
2 files changed, 37 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5e9782ff/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
index 5f8f649..698149c 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
@@ -1403,10 +1403,9 @@ public class LibMatrixMult
SparseBlock b = m2.sparseBlock;
double[] c = ret.denseBlock;
int m = m1.rlen;
+ int cd = m1.clen;
int n = m2.clen;
- //TODO perf sparse block
-
// MATRIX-MATRIX (VV, MV not applicable here because V always dense)
if(LOW_LEVEL_OPTIMIZATION)
{
@@ -1433,28 +1432,37 @@ public class LibMatrixMult
}
else //MATRIX-MATRIX
{
- for( int i=rl, cix=rl*n; i<ru; i++, cix+=n )
- {
- if( !a.isEmpty(i) )
- {
- int apos = a.pos(i);
- int alen = a.size(i);
- int[] aix = a.indexes(i);
- double[] avals = a.values(i);
-
- for(int k = apos; k < apos+alen; k++)
- {
- double val = avals[k];
- if( !b.isEmpty(aix[k]) )
- {
- int bpos = b.pos(aix[k]);
- int blen = b.size(aix[k]);
- int[] bix = b.indexes(aix[k]);
- double[] bvals = b.values(aix[k]);
+ //block sizes for best-effort blocking w/ sufficient row reuse in B yet small overhead
+ final int blocksizeI = 32;
+ final int blocksizeK = (int)Math.max(32, UtilFunctions.nextIntPow2(
+ (int)Math.pow((double)m*cd/m1.nonZeros,2)));
+
+ //temporary array of current sparse positions
+ int[] curk = new int[blocksizeI];
+
+ //blocked execution over IK
+ for( int bi = rl; bi < ru; bi+=blocksizeI ) {
+ Arrays.fill(curk, 0); //reset positions
+ for( int bk = 0, bimin = Math.min(ru, bi+blocksizeI); bk < cd; bk+=blocksizeK ) {
+ final int bkmin = Math.min(cd, bk+blocksizeK);
+
+ //core sub block matrix multiplication
+ for( int i=bi, cix=bi*n; i<bimin; i++, cix+=n ) {
+ if( !a.isEmpty(i) ) {
+ final int apos = a.pos(i);
+ final int alen = a.size(i);
+ int[] aix = a.indexes(i);
+ double[] avals = a.values(i);
- vectMultiplyAdd(val, bvals, c, bix, bpos, cix, blen);
+ int k = curk[i-bi] + apos;
+ for(; k < apos+alen && aix[k]<bkmin; k++) {
+ if( !b.isEmpty(aix[k]) )
+ vectMultiplyAdd(avals[k], b.values(aix[k]), c,
+ b.indexes(aix[k]), b.pos(aix[k]), cix, b.size(aix[k]));
+ }
+ curk[i-bi] = k - apos;
}
- }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5e9782ff/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
index fe66e55..c1d83e7 100644
--- a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
+++ b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
@@ -32,11 +32,16 @@ public class UtilFunctions
public static double DOUBLE_EPS = Math.pow(2, -53);
- public static int longHashFunc(long v)
- {
+ public static int longHashFunc(long v) {
return (int)(v^(v>>>32));
}
+ public static int nextIntPow2( int in ) {
+ int expon = (in==0) ? 0 : 32-Integer.numberOfLeadingZeros(in-1);
+ long pow2 = (long) Math.pow(2, expon);
+ return (int)((pow2>Integer.MAX_VALUE)?Integer.MAX_VALUE : pow2);
+ }
+
//return one block index given the index in cell format and block size
//TODO to be deleted
public static long blockIndexCalculation(long cellIndex, int blockSize)
[4/4] incubator-systemml git commit: [SYSTEMML-573] Various code
cleanups (frame writers, utils, costs estim)
Posted by mb...@apache.org.
[SYSTEMML-573] Various code cleanups (frame writers, utils, costs estim)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/42e47107
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/42e47107
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/42e47107
Branch: refs/heads/master
Commit: 42e471079b296529c2e54a72961e11d28ab70dda
Parents: 5e9782f
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Mar 27 13:08:55 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sun Mar 27 13:08:55 2016 -0700
----------------------------------------------------------------------
.../apache/sysml/hops/cost/CostEstimator.java | 4 +-
.../spark/MatrixIndexingSPInstruction.java | 32 +++++------
.../spark/QuantilePickSPInstruction.java | 16 +++---
.../spark/TernarySPInstruction.java | 2 +-
.../ConvertTextLineToBinaryCellFunction.java | 8 +--
.../instructions/spark/utils/RDDSortUtils.java | 16 +++---
.../instructions/spark/utils/SparkUtils.java | 2 +-
.../sysml/runtime/io/FrameWriterTextCSV.java | 29 ++++------
.../sysml/runtime/io/FrameWriterTextCell.java | 26 ++++-----
.../apache/sysml/runtime/matrix/CombineMR.java | 8 +--
.../data/BinaryBlockToBinaryCellConverter.java | 4 +-
.../data/BinaryBlockToRowBlockConverter.java | 2 +-
.../data/BinaryBlockToTextCellConverter.java | 4 +-
.../sysml/runtime/matrix/data/LibMatrixAgg.java | 2 +-
.../sysml/runtime/matrix/data/MatrixBlock.java | 2 +-
.../matrix/data/OperationsOnMatrixValues.java | 32 +++++------
.../runtime/matrix/mapred/CSVReblockMapper.java | 4 +-
.../sysml/runtime/util/UtilFunctions.java | 58 ++++----------------
18 files changed, 104 insertions(+), 147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
index 00790a4..0e2a353 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
@@ -503,8 +503,8 @@ public abstract class CostEstimator
String ret = inst;
while( ret.contains(Lop.VARIABLE_NAME_PLACEHOLDER) ) {
int index1 = ret.indexOf(Lop.VARIABLE_NAME_PLACEHOLDER);
- int index2 = ret.indexOf(Lop.VARIABLE_NAME_PLACEHOLDER, index1+3);
- String replace = ret.substring(index1,index2+2);
+ int index2 = ret.indexOf(Lop.VARIABLE_NAME_PLACEHOLDER, index1+1);
+ String replace = ret.substring(index1,index2+1);
ret = ret.replaceAll(replace, "1");
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
index 89992b8..3bf4d98 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java
@@ -294,10 +294,10 @@ public class MatrixIndexingSPInstruction extends UnarySPInstruction
long end_lhs_globalRowIndex = start_lhs_globalRowIndex + rightKV._2.getNumRows() - 1;
long end_lhs_globalColIndex = start_lhs_globalColIndex + rightKV._2.getNumColumns() - 1;
- long start_lhs_rowIndex = UtilFunctions.blockIndexCalculation(start_lhs_globalRowIndex, brlen);
- long end_lhs_rowIndex = UtilFunctions.blockIndexCalculation(end_lhs_globalRowIndex, brlen);
- long start_lhs_colIndex = UtilFunctions.blockIndexCalculation(start_lhs_globalColIndex, bclen);
- long end_lhs_colIndex = UtilFunctions.blockIndexCalculation(end_lhs_globalColIndex, bclen);
+ long start_lhs_rowIndex = UtilFunctions.computeBlockIndex(start_lhs_globalRowIndex, brlen);
+ long end_lhs_rowIndex = UtilFunctions.computeBlockIndex(end_lhs_globalRowIndex, brlen);
+ long start_lhs_colIndex = UtilFunctions.computeBlockIndex(start_lhs_globalColIndex, bclen);
+ long end_lhs_colIndex = UtilFunctions.computeBlockIndex(end_lhs_globalColIndex, bclen);
for(long leftRowIndex = start_lhs_rowIndex; leftRowIndex <= end_lhs_rowIndex; leftRowIndex++) {
for(long leftColIndex = start_lhs_colIndex; leftColIndex <= end_lhs_colIndex; leftColIndex++) {
@@ -308,20 +308,20 @@ public class MatrixIndexingSPInstruction extends UnarySPInstruction
long lhs_cl = Math.max((leftColIndex-1)*bclen+1, start_lhs_globalColIndex);
long lhs_cu = Math.min(leftColIndex*bclen, end_lhs_globalColIndex);
- int lhs_lrl = UtilFunctions.cellInBlockCalculation(lhs_rl, brlen);
- int lhs_lru = UtilFunctions.cellInBlockCalculation(lhs_ru, brlen);
- int lhs_lcl = UtilFunctions.cellInBlockCalculation(lhs_cl, bclen);
- int lhs_lcu = UtilFunctions.cellInBlockCalculation(lhs_cu, bclen);
+ int lhs_lrl = UtilFunctions.computeCellInBlock(lhs_rl, brlen);
+ int lhs_lru = UtilFunctions.computeCellInBlock(lhs_ru, brlen);
+ int lhs_lcl = UtilFunctions.computeCellInBlock(lhs_cl, bclen);
+ int lhs_lcu = UtilFunctions.computeCellInBlock(lhs_cu, bclen);
long rhs_rl = lhs_rl - rl + 1;
long rhs_ru = rhs_rl + (lhs_ru - lhs_rl);
long rhs_cl = lhs_cl - cl + 1;
long rhs_cu = rhs_cl + (lhs_cu - lhs_cl);
- int rhs_lrl = UtilFunctions.cellInBlockCalculation(rhs_rl, brlen);
- int rhs_lru = UtilFunctions.cellInBlockCalculation(rhs_ru, brlen);
- int rhs_lcl = UtilFunctions.cellInBlockCalculation(rhs_cl, bclen);
- int rhs_lcu = UtilFunctions.cellInBlockCalculation(rhs_cu, bclen);
+ int rhs_lrl = UtilFunctions.computeCellInBlock(rhs_rl, brlen);
+ int rhs_lru = UtilFunctions.computeCellInBlock(rhs_ru, brlen);
+ int rhs_lcl = UtilFunctions.computeCellInBlock(rhs_cl, bclen);
+ int rhs_lcu = UtilFunctions.computeCellInBlock(rhs_cu, bclen);
MatrixBlock slicedRHSBlk = rightKV._2.sliceOperations(rhs_lrl, rhs_lru, rhs_lcl, rhs_lcu, new MatrixBlock());
@@ -440,10 +440,10 @@ public class MatrixIndexingSPInstruction extends UnarySPInstruction
MatrixBlock slicedRHSMatBlock = _binput.sliceOperations(rhs_rl, rhs_ru, rhs_cl, rhs_cu, new MatrixBlock());
// Provide local zero-based index to leftIndexingOperations
- int lhs_lrl = UtilFunctions.cellInBlockCalculation(lhs_rl, _brlen);
- int lhs_lru = UtilFunctions.cellInBlockCalculation(lhs_ru, _brlen);
- int lhs_lcl = UtilFunctions.cellInBlockCalculation(lhs_cl, _bclen);
- int lhs_lcu = UtilFunctions.cellInBlockCalculation(lhs_cu, _bclen);
+ int lhs_lrl = UtilFunctions.computeCellInBlock(lhs_rl, _brlen);
+ int lhs_lru = UtilFunctions.computeCellInBlock(lhs_ru, _brlen);
+ int lhs_lcl = UtilFunctions.computeCellInBlock(lhs_cl, _bclen);
+ int lhs_lcu = UtilFunctions.computeCellInBlock(lhs_cu, _bclen);
MatrixBlock ret = arg._2.leftIndexingOperations(slicedRHSMatBlock, lhs_lrl, lhs_lru, lhs_lcl, lhs_lcu, new MatrixBlock(), false);
return new Tuple2<MatrixIndexes, MatrixBlock>(arg._1, ret);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java
index 5cea24f..55b9584 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java
@@ -177,8 +177,8 @@ public class QuantilePickSPInstruction extends BinarySPInstruction
*/
private double lookupKey(JavaPairRDD<MatrixIndexes,MatrixBlock> in, long key, int brlen)
{
- long rix = UtilFunctions.blockIndexCalculation(key, brlen);
- long pos = UtilFunctions.cellInBlockCalculation(key, brlen);
+ long rix = UtilFunctions.computeBlockIndex(key, brlen);
+ long pos = UtilFunctions.computeCellInBlock(key, brlen);
List<MatrixBlock> val = in.lookup(new MatrixIndexes(rix,1));
return val.get(0).quickGetValue((int)pos, 0);
@@ -211,8 +211,8 @@ public class QuantilePickSPInstruction extends BinarySPInstruction
public FilterFunction(long key25, long key75, int brlen)
{
- _minRowIndex = UtilFunctions.blockIndexCalculation(key25, brlen);
- _maxRowIndex = UtilFunctions.blockIndexCalculation(key75, brlen);
+ _minRowIndex = UtilFunctions.computeBlockIndex(key25, brlen);
+ _maxRowIndex = UtilFunctions.computeBlockIndex(key75, brlen);
}
@Override
@@ -239,10 +239,10 @@ public class QuantilePickSPInstruction extends BinarySPInstruction
public ExtractAndSumFunction(long key25, long key75, int brlen)
{
- _minRowIndex = UtilFunctions.blockIndexCalculation(key25, brlen);
- _maxRowIndex = UtilFunctions.blockIndexCalculation(key75, brlen);
- _minPos = UtilFunctions.cellInBlockCalculation(key25, brlen);
- _maxPos = UtilFunctions.cellInBlockCalculation(key75, brlen);
+ _minRowIndex = UtilFunctions.computeBlockIndex(key25, brlen);
+ _maxRowIndex = UtilFunctions.computeBlockIndex(key75, brlen);
+ _minPos = UtilFunctions.computeCellInBlock(key25, brlen);
+ _maxPos = UtilFunctions.computeCellInBlock(key75, brlen);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java
index ec2d4b0..fa00eb0 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java
@@ -280,7 +280,7 @@ public class TernarySPInstruction extends ComputationSPInstruction
for( int i=0; i<mb.getNumRows(); i++ )
{
//compute global target indexes (via ctable obj for error handling consistency)
- long row = UtilFunctions.cellIndexCalculation(ix.getRowIndex(), _brlen, i);
+ long row = UtilFunctions.computeCellIndex(ix.getRowIndex(), _brlen, i);
double v2 = mb.quickGetValue(i, 0);
Pair<MatrixIndexes,Double> p = ctab.execute(row, v2, 1.0);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java
index cde5d4e..b135497 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertTextLineToBinaryCellFunction.java
@@ -69,10 +69,10 @@ public class ConvertTextLineToBinaryCellFunction implements PairFunction<String,
// Get appropriate indexes for blockIndexes and cell
// For line: 1020 704 2.362153706180234 (assuming default block size: 1000 X 1000),
// blockRowIndex = 2, blockColIndex = 1, rowIndexInBlock = 19, colIndexInBlock = 703 ... TODO: double check this !!!
- long blockRowIndex = UtilFunctions.blockIndexCalculation(retVal.getKey().getRowIndex(), (int) brlen);
- long blockColIndex = UtilFunctions.blockIndexCalculation(retVal.getKey().getColumnIndex(), (int) bclen);
- long rowIndexInBlock = UtilFunctions.cellInBlockCalculation(retVal.getKey().getRowIndex(), brlen);
- long colIndexInBlock = UtilFunctions.cellInBlockCalculation(retVal.getKey().getColumnIndex(), bclen);
+ long blockRowIndex = UtilFunctions.computeBlockIndex(retVal.getKey().getRowIndex(), (int) brlen);
+ long blockColIndex = UtilFunctions.computeBlockIndex(retVal.getKey().getColumnIndex(), (int) bclen);
+ long rowIndexInBlock = UtilFunctions.computeCellInBlock(retVal.getKey().getRowIndex(), brlen);
+ long colIndexInBlock = UtilFunctions.computeCellInBlock(retVal.getKey().getColumnIndex(), bclen);
// Perform sanity check
if(blockRowIndex <= 0 || blockColIndex <= 0 || rowIndexInBlock < 0 || colIndexInBlock < 0) {
throw new Exception("Error computing indexes for the line:" + line);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java
index e3c3779..34c1358 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java
@@ -432,8 +432,8 @@ public class RDDSortUtils
{
Tuple2<Double,Long> val = arg0.next();
long valix = val._2 + 1;
- long rix = UtilFunctions.blockIndexCalculation(valix, _brlen);
- int pos = UtilFunctions.cellInBlockCalculation(valix, _brlen);
+ long rix = UtilFunctions.computeBlockIndex(valix, _brlen);
+ int pos = UtilFunctions.computeCellInBlock(valix, _brlen);
if( ix == null || ix.getRowIndex() != rix )
{
@@ -483,8 +483,8 @@ public class RDDSortUtils
{
Tuple2<DoublePair,Long> val = arg0.next();
long valix = val._2 + 1;
- long rix = UtilFunctions.blockIndexCalculation(valix, _brlen);
- int pos = UtilFunctions.cellInBlockCalculation(valix, _brlen);
+ long rix = UtilFunctions.computeBlockIndex(valix, _brlen);
+ int pos = UtilFunctions.computeCellInBlock(valix, _brlen);
if( ix == null || ix.getRowIndex() != rix )
{
@@ -535,8 +535,8 @@ public class RDDSortUtils
{
Tuple2<ValueIndexPair,Long> val = arg0.next();
long valix = val._2 + 1;
- long rix = UtilFunctions.blockIndexCalculation(valix, _brlen);
- int pos = UtilFunctions.cellInBlockCalculation(valix, _brlen);
+ long rix = UtilFunctions.computeBlockIndex(valix, _brlen);
+ int pos = UtilFunctions.computeCellInBlock(valix, _brlen);
if( ix == null || ix.getRowIndex() != rix )
{
@@ -586,8 +586,8 @@ public class RDDSortUtils
{
Tuple2<Long,Long> val = arg0.next();
long valix = val._1;
- long rix = UtilFunctions.blockIndexCalculation(valix, _brlen);
- int pos = UtilFunctions.cellInBlockCalculation(valix, _brlen);
+ long rix = UtilFunctions.computeBlockIndex(valix, _brlen);
+ int pos = UtilFunctions.computeCellInBlock(valix, _brlen);
if( ix == null || ix.getRowIndex() != rix )
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
index f3232f7..6b9037a 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java
@@ -167,7 +167,7 @@ public class SparkUtils
// len = {clen or rlen}, blen = {brlen or bclen}
public static long getStartGlobalIndex(long blockIndex, int blen, long len) {
- return UtilFunctions.cellIndexCalculation(blockIndex, blen, 0);
+ return UtilFunctions.computeCellIndex(blockIndex, blen, 0);
}
public static JavaPairRDD<MatrixIndexes, MatrixBlock> getRDDWithEmptyBlocks(JavaSparkContext sc,
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
index 650778d..10d957f 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java
@@ -22,6 +22,7 @@ package org.apache.sysml.runtime.io;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
+import java.util.Iterator;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -107,10 +108,8 @@ public class FrameWriterTextCSV extends FrameWriter
if( props.hasHeader() )
{
//write row chunk-wise to prevent OOM on large number of columns
- for( int bj=0; bj<clen; bj+=BLOCKSIZE_J )
- {
- for( int j=bj; j < Math.min(clen,bj+BLOCKSIZE_J); j++)
- {
+ for( int bj=0; bj<clen; bj+=BLOCKSIZE_J ) {
+ for( int j=bj; j < Math.min(clen,bj+BLOCKSIZE_J); j++) {
sb.append("C"+ (j+1));
if ( j < clen-1 )
sb.append(delim);
@@ -124,16 +123,14 @@ public class FrameWriterTextCSV extends FrameWriter
}
// Write data lines
- for( int i=0; i<rlen; i++ )
- {
+ Iterator<String[]> iter = src.getStringRowIterator();
+ while( iter.hasNext() ) {
//write row chunk-wise to prevent OOM on large number of columns
- for( int bj=0; bj<clen; bj+=BLOCKSIZE_J )
- {
- for( int j=bj; j<Math.min(clen,bj+BLOCKSIZE_J); j++ )
- {
- if(src.get(i, j) != null)
- sb.append(src.get(i, j).toString());
-
+ String[] row = iter.next();
+ for( int bj=0; bj<clen; bj+=BLOCKSIZE_J ) {
+ for( int j=bj; j<Math.min(clen,bj+BLOCKSIZE_J); j++ ) {
+ if(row[j] != null)
+ sb.append(row[j]);
if( j != clen-1 )
sb.append(delim);
}
@@ -142,14 +139,12 @@ public class FrameWriterTextCSV extends FrameWriter
}
sb.append('\n');
- br.write( sb.toString() ); //same as append
+ br.write( sb.toString() );
sb.setLength(0);
}
}
- finally
- {
+ finally {
IOUtilFunctions.closeSilently(br);
}
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
index 2af9a60..c595448 100644
--- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
+++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java
@@ -22,6 +22,7 @@ package org.apache.sysml.runtime.io;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
+import java.util.Iterator;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -84,8 +85,7 @@ public class FrameWriterTextCell extends FrameWriter
int cols = src.getNumColumns();
//bound check per block
- if( rows > rlen || cols > clen )
- {
+ if( rows > rlen || cols > clen ) {
throw new IOException("Frame block [1:"+rows+",1:"+cols+"] " +
"out of overall frame range [1:"+rlen+",1:"+clen+"].");
}
@@ -95,21 +95,19 @@ public class FrameWriterTextCell extends FrameWriter
//for obj reuse and preventing repeated buffer re-allocations
StringBuilder sb = new StringBuilder();
- for( int i=0; i<rows; i++ )
- {
- String rowIndex = Integer.toString(i+1);
- for( int j=0; j<cols; j++ )
- {
- if(src.get(i, j) != null)
- {
- String lvalue = src.get(i, j).toString();
+ Iterator<String[]> iter = src.getStringRowIterator();
+ for( int i=0; iter.hasNext(); i++ ) { //for all rows
+ String rowIndex = Integer.toString(i+1);
+ String[] row = iter.next();
+ for( int j=0; j<cols; j++ ) {
+ if( row[j] != null ) {
sb.append(rowIndex);
sb.append(' ');
sb.append( j+1 );
sb.append(' ');
- sb.append( lvalue );
+ sb.append( row[j] );
sb.append('\n');
- br.write( sb.toString() ); //same as append
+ br.write( sb.toString() );
sb.setLength(0);
entriesWritten = true;
}
@@ -121,10 +119,8 @@ public class FrameWriterTextCell extends FrameWriter
br.write("1 1 0\n");
}
}
- finally
- {
+ finally {
IOUtilFunctions.closeSilently(br);
}
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java b/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
index e48c868..d40857f 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
@@ -196,8 +196,8 @@ public class CombineMR
{
Pair<Integer, Integer> blockSize=outputBlockSizes.get(ins.output);
keyBuff.setIndexes(
- UtilFunctions.cellIndexCalculation(in1.getIndexes().getRowIndex(), blockSize.getKey(), r),
- UtilFunctions.cellIndexCalculation(in1.getIndexes().getColumnIndex(), blockSize.getValue(), c)
+ UtilFunctions.computeCellIndex(in1.getIndexes().getRowIndex(), blockSize.getKey(), r),
+ UtilFunctions.computeCellIndex(in1.getIndexes().getColumnIndex(), blockSize.getValue(), c)
);
valueBuff.setValue(in1.getValue().getValue(r, c));
valueBuff.setOtherValue(in2.getValue().getValue(r, c));
@@ -259,8 +259,8 @@ public class CombineMR
{
Pair<Integer, Integer> blockSize=outputBlockSizes.get(ins.output);
keyBuff.setIndexes(
- UtilFunctions.cellIndexCalculation(indexes.getRowIndex(), blockSize.getKey(), r),
- UtilFunctions.cellIndexCalculation(indexes.getColumnIndex(), blockSize.getValue(), c)
+ UtilFunctions.computeCellIndex(indexes.getRowIndex(), blockSize.getKey(), r),
+ UtilFunctions.computeCellIndex(indexes.getColumnIndex(), blockSize.getValue(), c)
);
valueBuff.setValue(in1.getValue().getValue(r, c));
double temp=in2.getValue().getValue(r, c);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java
index e4fefe4..42c2095 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java
@@ -58,8 +58,8 @@ Converter<MatrixIndexes, MatrixBlock, MatrixIndexes, MatrixCell>
@Override
public void convert(MatrixIndexes k1, MatrixBlock v1) {
reset();
- startIndexes.setIndexes(UtilFunctions.cellIndexCalculation(k1.getRowIndex(), brow,0),
- UtilFunctions.cellIndexCalculation(k1.getColumnIndex(),bcolumn,0));
+ startIndexes.setIndexes(UtilFunctions.computeCellIndex(k1.getRowIndex(), brow,0),
+ UtilFunctions.computeCellIndex(k1.getColumnIndex(),bcolumn,0));
sparse=v1.isInSparseFormat();
thisBlockWidth=v1.getNumColumns();
if(sparse)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java
index 26ea776..bfbd5bd 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java
@@ -58,7 +58,7 @@ public class BinaryBlockToRowBlockConverter implements Converter<MatrixIndexes,
@Override
public void convert(MatrixIndexes k1, MatrixBlock v1) {
reset();
- _startRowID = UtilFunctions.cellIndexCalculation(k1.getRowIndex(), _srcBrlen, 0);
+ _startRowID = UtilFunctions.computeCellIndex(k1.getRowIndex(), _srcBrlen, 0);
_colID = k1.getColumnIndex();
_destBlock.reset(1, v1.getNumColumns());
_srcRlen = v1.getNumRows();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java
index 0004adc..95cb5ec 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java
@@ -62,8 +62,8 @@ Converter<MatrixIndexes, MatrixBlock, NullWritable, Text>
@Override
public void convert(MatrixIndexes k1, MatrixBlock v1) {
reset();
- startIndexes.setIndexes(UtilFunctions.cellIndexCalculation(k1.getRowIndex(), brow,0),
- UtilFunctions.cellIndexCalculation(k1.getColumnIndex(),bcolumn,0));
+ startIndexes.setIndexes(UtilFunctions.computeCellIndex(k1.getRowIndex(), brow,0),
+ UtilFunctions.computeCellIndex(k1.getColumnIndex(),bcolumn,0));
sparse=v1.isInSparseFormat();
thisBlockWidth=v1.getNumColumns();
if(sparse)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java
index d48a004..a5620e5 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java
@@ -610,7 +610,7 @@ public class LibMatrixAgg
int m = out.rlen;
double[] c = out.getDenseBlock();
for( int i=0, cix=0; i<m; i++, cix+=2 )
- c[cix] = UtilFunctions.cellIndexCalculation(ix.getColumnIndex(), bclen, (int)c[cix]-1);
+ c[cix] = UtilFunctions.computeCellIndex(ix.getColumnIndex(), bclen, (int)c[cix]-1);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index 06ed711..034aeba 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -4520,7 +4520,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
|| ((Builtin)(op.aggOp.increOp.fn)).bFunc == Builtin.BuiltinFunctionCode.MININDEX)
){
double currMaxValue = result.quickGetValue(i, 1);
- long newMaxIndex = UtilFunctions.cellIndexCalculation(indexesIn.getColumnIndex(), blockingFactorCol, j);
+ long newMaxIndex = UtilFunctions.computeCellIndex(indexesIn.getColumnIndex(), blockingFactorCol, j);
double newMaxValue = quickGetValue(i, j);
double update = op.aggOp.increOp.fn.execute(newMaxValue, currMaxValue);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
index d9c8853..ea33de3 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java
@@ -305,10 +305,10 @@ public class OperationsOnMatrixValues
public static void performSlice(IndexedMatrixValue in, IndexRange ixrange, int brlen, int bclen, ArrayList<IndexedMatrixValue> outlist)
throws DMLUnsupportedOperationException, DMLRuntimeException
{
- long cellIndexTopRow = UtilFunctions.cellIndexCalculation(in.getIndexes().getRowIndex(), brlen, 0);
- long cellIndexBottomRow = UtilFunctions.cellIndexCalculation(in.getIndexes().getRowIndex(), brlen, in.getValue().getNumRows()-1);
- long cellIndexLeftCol = UtilFunctions.cellIndexCalculation(in.getIndexes().getColumnIndex(), bclen, 0);
- long cellIndexRightCol = UtilFunctions.cellIndexCalculation(in.getIndexes().getColumnIndex(), bclen, in.getValue().getNumColumns()-1);
+ long cellIndexTopRow = UtilFunctions.computeCellIndex(in.getIndexes().getRowIndex(), brlen, 0);
+ long cellIndexBottomRow = UtilFunctions.computeCellIndex(in.getIndexes().getRowIndex(), brlen, in.getValue().getNumRows()-1);
+ long cellIndexLeftCol = UtilFunctions.computeCellIndex(in.getIndexes().getColumnIndex(), bclen, 0);
+ long cellIndexRightCol = UtilFunctions.computeCellIndex(in.getIndexes().getColumnIndex(), bclen, in.getValue().getNumColumns()-1);
long cellIndexOverlapTop = Math.max(cellIndexTopRow, ixrange.rowStart);
long cellIndexOverlapBottom = Math.min(cellIndexBottomRow, ixrange.rowEnd);
@@ -321,13 +321,13 @@ public class OperationsOnMatrixValues
}
IndexRange tmpRange = new IndexRange(
- UtilFunctions.cellInBlockCalculation(cellIndexOverlapTop, brlen),
- UtilFunctions.cellInBlockCalculation(cellIndexOverlapBottom, brlen),
- UtilFunctions.cellInBlockCalculation(cellIndexOverlapLeft, bclen),
- UtilFunctions.cellInBlockCalculation(cellIndexOverlapRight, bclen));
+ UtilFunctions.computeCellInBlock(cellIndexOverlapTop, brlen),
+ UtilFunctions.computeCellInBlock(cellIndexOverlapBottom, brlen),
+ UtilFunctions.computeCellInBlock(cellIndexOverlapLeft, bclen),
+ UtilFunctions.computeCellInBlock(cellIndexOverlapRight, bclen));
- int rowCut=UtilFunctions.cellInBlockCalculation(ixrange.rowStart, brlen);
- int colCut=UtilFunctions.cellInBlockCalculation(ixrange.colStart, bclen);
+ int rowCut=UtilFunctions.computeCellInBlock(ixrange.rowStart, brlen);
+ int colCut=UtilFunctions.computeCellInBlock(ixrange.colStart, bclen);
int rowsInLastBlock = (int)((ixrange.rowEnd-ixrange.rowStart+1)%brlen);
if(rowsInLastBlock==0)
@@ -336,15 +336,15 @@ public class OperationsOnMatrixValues
if(colsInLastBlock==0)
colsInLastBlock=bclen;
- long resultBlockIndexTop=UtilFunctions.blockIndexCalculation(cellIndexOverlapTop-ixrange.rowStart+1, brlen);
- long resultBlockIndexBottom=UtilFunctions.blockIndexCalculation(cellIndexOverlapBottom-ixrange.rowStart+1, brlen);
- long resultBlockIndexLeft=UtilFunctions.blockIndexCalculation(cellIndexOverlapLeft-ixrange.colStart+1, bclen);
- long resultBlockIndexRight=UtilFunctions.blockIndexCalculation(cellIndexOverlapRight-ixrange.colStart+1, bclen);
+ long resultBlockIndexTop=UtilFunctions.computeBlockIndex(cellIndexOverlapTop-ixrange.rowStart+1, brlen);
+ long resultBlockIndexBottom=UtilFunctions.computeBlockIndex(cellIndexOverlapBottom-ixrange.rowStart+1, brlen);
+ long resultBlockIndexLeft=UtilFunctions.computeBlockIndex(cellIndexOverlapLeft-ixrange.colStart+1, bclen);
+ long resultBlockIndexRight=UtilFunctions.computeBlockIndex(cellIndexOverlapRight-ixrange.colStart+1, bclen);
int boundaryRlen = brlen;
int boundaryClen = bclen;
- long finalBlockIndexBottom=UtilFunctions.blockIndexCalculation(ixrange.rowEnd-ixrange.rowStart+1, brlen);
- long finalBlockIndexRight=UtilFunctions.blockIndexCalculation(ixrange.colEnd-ixrange.colStart+1, bclen);
+ long finalBlockIndexBottom=UtilFunctions.computeBlockIndex(ixrange.rowEnd-ixrange.rowStart+1, brlen);
+ long finalBlockIndexRight=UtilFunctions.computeBlockIndex(ixrange.colEnd-ixrange.colStart+1, bclen);
if(resultBlockIndexBottom==finalBlockIndexBottom)
boundaryRlen=rowsInLastBlock;
if(resultBlockIndexRight==finalBlockIndexRight)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java
index bf00997..f371b70 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java
@@ -75,8 +75,8 @@ public class CSVReblockMapper extends MapperBase implements Mapper<LongWritable,
{
int start=0;
row.getIndexes().setTag(outTag);
- long rowIndex=UtilFunctions.blockIndexCalculation(rowOffset+num+1, brlen);
- row.getRow().indexInBlock=UtilFunctions.cellInBlockCalculation(rowOffset+num+1, brlen);
+ long rowIndex=UtilFunctions.computeBlockIndex(rowOffset+num+1, brlen);
+ row.getRow().indexInBlock=UtilFunctions.computeCellInBlock(rowOffset+num+1, brlen);
long col=0;
for(; col<cells.length/bclen; col++)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/42e47107/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
index c1d83e7..40fb329 100644
--- a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
+++ b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java
@@ -42,34 +42,6 @@ public class UtilFunctions
return (int)((pow2>Integer.MAX_VALUE)?Integer.MAX_VALUE : pow2);
}
- //return one block index given the index in cell format and block size
- //TODO to be deleted
- public static long blockIndexCalculation(long cellIndex, int blockSize)
- {
- if(cellIndex>0)
- return (cellIndex-1)/blockSize+1;
- else
- return (long)Math.floor((double)(cellIndex-1)/(double)blockSize)+1;
- }
-
- //return cell index in the block, for given index in the cell format and block size
- //TODO to be deleted
- public static int cellInBlockCalculation(long cellIndex, int blockSize)
- {
- if(cellIndex>0)
- return (int) ((cellIndex-1)%blockSize);
- else
- //return (int) Math.abs((cellIndex-1)%blockSize);
- return (int) ((cellIndex-1)%blockSize)+blockSize;
- }
-
- //given block index and block size and cells in block, return the index in cell format
- //TODO to be deleted
- public static long cellIndexCalculation(long blockIndex, int blockSize, int cellInBlock)
- {
- return (blockIndex-1)*blockSize+1+cellInBlock;
- }
-
/**
* Computes the 1-based block index based on the global cell index and block size meta
* data. See computeCellIndex for the inverse operation.
@@ -78,8 +50,7 @@ public class UtilFunctions
* @param blockSize
* @return
*/
- public static long computeBlockIndex(long cellIndex, int blockSize)
- {
+ public static long computeBlockIndex(long cellIndex, int blockSize) {
return (cellIndex-1)/blockSize + 1;
}
@@ -91,8 +62,7 @@ public class UtilFunctions
* @param blockSize
* @return
*/
- public static int computeCellInBlock(long cellIndex, int blockSize)
- {
+ public static int computeCellInBlock(long cellIndex, int blockSize) {
return (int) ((cellIndex-1)%blockSize);
}
@@ -107,9 +77,7 @@ public class UtilFunctions
* @param cellInBlock
* @return
*/
- public static long computeCellIndex( long blockIndex, int blockSize, int cellInBlock )
- {
- //
+ public static long computeCellIndex( long blockIndex, int blockSize, int cellInBlock ) {
return (blockIndex-1)*blockSize + 1 + cellInBlock;
}
@@ -123,8 +91,7 @@ public class UtilFunctions
* @param blockSize
* @return
*/
- public static int computeBlockSize( long len, long blockIndex, long blockSize )
- {
+ public static int computeBlockSize( long len, long blockIndex, long blockSize ) {
long remain = len - (blockIndex-1)*blockSize;
return (int)Math.min(blockSize, remain);
}
@@ -189,15 +156,15 @@ public class UtilFunctions
{
IndexRange tempRange = new IndexRange(-1, -1, -1, -1);
- long topBlockRowIndex=UtilFunctions.blockIndexCalculation(indexRange.rowStart, blockRowFactor);
- int topRowInTopBlock=UtilFunctions.cellInBlockCalculation(indexRange.rowStart, blockRowFactor);
- long bottomBlockRowIndex=UtilFunctions.blockIndexCalculation(indexRange.rowEnd, blockRowFactor);
- int bottomRowInBottomBlock=UtilFunctions.cellInBlockCalculation(indexRange.rowEnd, blockRowFactor);
+ long topBlockRowIndex=UtilFunctions.computeBlockIndex(indexRange.rowStart, blockRowFactor);
+ int topRowInTopBlock=UtilFunctions.computeCellInBlock(indexRange.rowStart, blockRowFactor);
+ long bottomBlockRowIndex=UtilFunctions.computeBlockIndex(indexRange.rowEnd, blockRowFactor);
+ int bottomRowInBottomBlock=UtilFunctions.computeCellInBlock(indexRange.rowEnd, blockRowFactor);
- long leftBlockColIndex=UtilFunctions.blockIndexCalculation(indexRange.colStart, blockColFactor);
- int leftColInLeftBlock=UtilFunctions.cellInBlockCalculation(indexRange.colStart, blockColFactor);
- long rightBlockColIndex=UtilFunctions.blockIndexCalculation(indexRange.colEnd, blockColFactor);
- int rightColInRightBlock=UtilFunctions.cellInBlockCalculation(indexRange.colEnd, blockColFactor);
+ long leftBlockColIndex=UtilFunctions.computeBlockIndex(indexRange.colStart, blockColFactor);
+ int leftColInLeftBlock=UtilFunctions.computeCellInBlock(indexRange.colStart, blockColFactor);
+ long rightBlockColIndex=UtilFunctions.computeBlockIndex(indexRange.colEnd, blockColFactor);
+ int rightColInRightBlock=UtilFunctions.computeCellInBlock(indexRange.colEnd, blockColFactor);
//no overlap
if(in.getIndexes().getRowIndex()<topBlockRowIndex || in.getIndexes().getRowIndex()>bottomBlockRowIndex
@@ -234,7 +201,6 @@ public class UtilFunctions
long total = UtilFunctions.getTotalLength(metadata);
long lpos=(long)Math.ceil(total*p);//lower bound is inclusive
long upos=(long)Math.ceil(total*(1-p));//upper bound is inclusive
- //System.out.println("getLengthForInterQuantile(): " + (upos-lpos+1));
return upos-lpos+1;
}