You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/10/13 19:54:11 UTC
systemml git commit: [SYSTEMML-2486] Performance matrix histogram
estimator for dense
Repository: systemml
Updated Branches:
refs/heads/master fc6f89128 -> 41de8dcdc
[SYSTEMML-2486] Performance matrix histogram estimator for dense
This patch improves the performance of matrix histogram construction via
a special case for fully dense matrices that allow for the construction
from meta data without a pass over the input.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/41de8dcd
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/41de8dcd
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/41de8dcd
Branch: refs/heads/master
Commit: 41de8dcdc621b7dc2c1557aca64095512cdd6cf6
Parents: fc6f891
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sat Oct 13 21:53:59 2018 +0200
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sat Oct 13 21:53:59 2018 +0200
----------------------------------------------------------------------
.../org/apache/sysml/hops/estim/EstimatorBitsetMM.java | 6 +++---
.../apache/sysml/hops/estim/EstimatorDensityMap.java | 2 +-
.../apache/sysml/hops/estim/EstimatorLayeredGraph.java | 2 +-
.../sysml/hops/estim/EstimatorMatrixHistogram.java | 13 ++++++++++---
4 files changed, 15 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/41de8dcd/src/main/java/org/apache/sysml/hops/estim/EstimatorBitsetMM.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/estim/EstimatorBitsetMM.java b/src/main/java/org/apache/sysml/hops/estim/EstimatorBitsetMM.java
index aacbf29..8802e9c 100644
--- a/src/main/java/org/apache/sysml/hops/estim/EstimatorBitsetMM.java
+++ b/src/main/java/org/apache/sysml/hops/estim/EstimatorBitsetMM.java
@@ -108,7 +108,7 @@ public class EstimatorBitsetMM extends SparsityEstimator
}
}
- private abstract static class BitsetMatrix {
+ public abstract static class BitsetMatrix {
protected final int _rlen;
protected final int _clen;
protected long _nonZeros;
@@ -207,7 +207,7 @@ public class EstimatorBitsetMM extends SparsityEstimator
* not allow for range ORs). However, this implies a maximum size of 16GB.
*
*/
- private static class BitsetMatrix1 extends BitsetMatrix {
+ public static class BitsetMatrix1 extends BitsetMatrix {
//linearized and padded data array in row-major order, where each long
//represents 64 boolean values, all rows are aligned at 64 for simple access
private final int _rowLen;
@@ -407,7 +407,7 @@ public class EstimatorBitsetMM extends SparsityEstimator
}
@SuppressWarnings("unused")
- private static class BitsetMatrix2 extends BitsetMatrix {
+ public static class BitsetMatrix2 extends BitsetMatrix {
private BitSet[] _data;
public BitsetMatrix2(int rlen, int clen) {
http://git-wip-us.apache.org/repos/asf/systemml/blob/41de8dcd/src/main/java/org/apache/sysml/hops/estim/EstimatorDensityMap.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/estim/EstimatorDensityMap.java b/src/main/java/org/apache/sysml/hops/estim/EstimatorDensityMap.java
index 27e6dcf..144f7a7 100644
--- a/src/main/java/org/apache/sysml/hops/estim/EstimatorDensityMap.java
+++ b/src/main/java/org/apache/sysml/hops/estim/EstimatorDensityMap.java
@@ -180,7 +180,7 @@ public class EstimatorDensityMap extends SparsityEstimator
m1Map.getNumColumnsOrig(), _b, true);
}
- private static class DensityMap {
+ public static class DensityMap {
private final MatrixBlock _map;
private final int _rlen;
private final int _clen;
http://git-wip-us.apache.org/repos/asf/systemml/blob/41de8dcd/src/main/java/org/apache/sysml/hops/estim/EstimatorLayeredGraph.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/estim/EstimatorLayeredGraph.java b/src/main/java/org/apache/sysml/hops/estim/EstimatorLayeredGraph.java
index fa8adaa..809ac8b 100644
--- a/src/main/java/org/apache/sysml/hops/estim/EstimatorLayeredGraph.java
+++ b/src/main/java/org/apache/sysml/hops/estim/EstimatorLayeredGraph.java
@@ -88,7 +88,7 @@ public class EstimatorLayeredGraph extends SparsityEstimator {
return leafs;
}
- private static class LayeredGraph {
+ public static class LayeredGraph {
private final List<Node[]> _nodes; //nodes partitioned by graph level
private final int _rounds; //length of propagated r-vectors
http://git-wip-us.apache.org/repos/asf/systemml/blob/41de8dcd/src/main/java/org/apache/sysml/hops/estim/EstimatorMatrixHistogram.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/estim/EstimatorMatrixHistogram.java b/src/main/java/org/apache/sysml/hops/estim/EstimatorMatrixHistogram.java
index 9f9075d..206c6d5 100644
--- a/src/main/java/org/apache/sysml/hops/estim/EstimatorMatrixHistogram.java
+++ b/src/main/java/org/apache/sysml/hops/estim/EstimatorMatrixHistogram.java
@@ -19,6 +19,7 @@
package org.apache.sysml.hops.estim;
+import java.util.Arrays;
import java.util.Random;
import java.util.stream.IntStream;
@@ -195,7 +196,7 @@ public class EstimatorMatrixHistogram extends SparsityEstimator
h1.getRows(), h2.getCols(), nnz);
}
- private static class MatrixHistogram {
+ public static class MatrixHistogram {
// count vectors (the histogram)
private final int[] rNnz; //nnz per row
private int[] rNnz1e = null; //nnz per row for cols w/ <= 1 non-zeros
@@ -218,7 +219,12 @@ public class EstimatorMatrixHistogram extends SparsityEstimator
&& in.getNumRows() == in.getNumColumns();
// 2) compute basic synopsis details
- if( !in.isEmpty() ) {
+ if( in.getLength() == in.getNonZeros() ) {
+ //fully dense: constant row/column counts
+ Arrays.fill(rNnz, n);
+ Arrays.fill(cNnz, m);
+ }
+ else if( !in.isEmpty() ) {
if( in.isInSparseFormat() ) {
SparseBlock a = in.getSparseBlock();
for( int i=0; i<m; i++ ) {
@@ -250,7 +256,8 @@ public class EstimatorMatrixHistogram extends SparsityEstimator
rNdiv2 = rSummary[3]; cNdiv2 = cSummary[3];
// 4) compute exception details if necessary (optional)
- if( useExcepts & !in.isEmpty() && (rMaxNnz > 1 || cMaxNnz > 1) ) {
+ if( useExcepts && !in.isEmpty() && (rMaxNnz > 1 || cMaxNnz > 1)
+ && in.getLength() != in.getNonZeros() ) { //not fully dense
rNnz1e = new int[in.getNumRows()];
cNnz1e = new int[in.getNumColumns()];