You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2018/10/21 17:23:30 UTC
[2/2] systemml git commit: [SYSTEMML-2479] Extended density map
estimator (additional operations)
[SYSTEMML-2479] Extended density map estimator (additional operations)
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/0a957e4c
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/0a957e4c
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/0a957e4c
Branch: refs/heads/master
Commit: 0a957e4c9a6aca0ef1cbf41e7dcdbdbc90ba4a04
Parents: 569806d
Author: Matthias Boehm <mb...@gmail.com>
Authored: Sun Oct 21 19:23:08 2018 +0200
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Sun Oct 21 19:23:08 2018 +0200
----------------------------------------------------------------------
.../sysml/hops/estim/EstimatorDensityMap.java | 60 +++++--
.../functions/estim/OpSingleTest.java | 159 +++----------------
2 files changed, 69 insertions(+), 150 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/0a957e4c/src/main/java/org/apache/sysml/hops/estim/EstimatorDensityMap.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/estim/EstimatorDensityMap.java b/src/main/java/org/apache/sysml/hops/estim/EstimatorDensityMap.java
index 144f7a7..e845a5b 100644
--- a/src/main/java/org/apache/sysml/hops/estim/EstimatorDensityMap.java
+++ b/src/main/java/org/apache/sysml/hops/estim/EstimatorDensityMap.java
@@ -23,6 +23,7 @@ import org.apache.commons.lang.NotImplementedException;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.DenseBlock;
+import org.apache.sysml.runtime.matrix.data.LibMatrixReorg;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.SparseBlock;
import org.apache.sysml.runtime.util.UtilFunctions;
@@ -81,10 +82,10 @@ public class EstimatorDensityMap extends SparsityEstimator
@Override
public double estim(MatrixBlock m1, MatrixBlock m2, OpCode op) {
if( isExactMetadataOp(op) )
- return estimExactMetaData(m1.getMatrixCharacteristics(),
- m2.getMatrixCharacteristics(), op).getSparsity();
+ return estimExactMetaData(m1.getMatrixCharacteristics(), m2 != null ?
+ m2.getMatrixCharacteristics() : null, op).getSparsity();
DensityMap m1Map = new DensityMap(m1, _b);
- DensityMap m2Map = (m1 == m2) ? //self product
+ DensityMap m2Map = (m1 == m2 || m2 == null) ? //self product
m1Map : new DensityMap(m2, _b);
DensityMap outMap = estimIntern(m1Map, m2Map, op);
return OptimizerUtils.getSparsity( //aggregate output histogram
@@ -105,18 +106,19 @@ public class EstimatorDensityMap extends SparsityEstimator
*/
private DensityMap estimIntern(DensityMap m1Map, DensityMap m2Map, OpCode op) {
switch(op) {
- case MM: return estimInternMM(m1Map, m2Map);
- case MULT: return estimInternMult(m1Map, m2Map);
- case PLUS: return estimInternPlus(m1Map, m2Map);
+ case MM: return estimInternMM(m1Map, m2Map);
+ case MULT: return estimInternMult(m1Map, m2Map);
+ case PLUS: return estimInternPlus(m1Map, m2Map);
+ case NEQZERO: return m1Map;
+ case EQZERO: return estimInternEqZero(m1Map);
case RBIND:
case CBIND:
//TODO simple append not possible due to partial blocks at end of m1Map
- case TRANS:
- case DIAG:
- case RESHAPE:
- //TODO add missing estimators
+ case TRANS: return estimInternTrans(m1Map);
+ case DIAG: return estimInternDiag(m1Map);
+ case RESHAPE: return estimInternReshape(m1Map);
default:
throw new NotImplementedException();
}
@@ -180,6 +182,40 @@ public class EstimatorDensityMap extends SparsityEstimator
m1Map.getNumColumnsOrig(), _b, true);
}
+ private DensityMap estimInternTrans(DensityMap m1Map) {
+ MatrixBlock out = LibMatrixReorg.transpose(m1Map.getMap(),
+ new MatrixBlock(m1Map.getNumColumns(), m1Map.getNumRows(), false));
+ return new DensityMap(out, m1Map.getNumColumnsOrig(),
+ m1Map.getNumRowsOrig(), _b, m1Map._scaled);
+ }
+
+ private DensityMap estimInternDiag(DensityMap m1Map) {
+ if( m1Map.getNumColumnsOrig() > 1 )
+ throw new NotImplementedException();
+ m1Map.toNnz();
+ MatrixBlock out = LibMatrixReorg.diag(m1Map.getMap(),
+ new MatrixBlock(m1Map.getNumRows(), m1Map.getNumRows(), false));
+ return new DensityMap(out, m1Map.getNumRowsOrig(),
+ m1Map.getNumRowsOrig(), _b, m1Map._scaled);
+ }
+
+ private DensityMap estimInternReshape(DensityMap m1Map) {
+ MatrixBlock out = new MatrixBlock(1,1,(double)m1Map.getNonZeros());
+ int b = Math.max(m1Map.getNumRowsOrig(), m1Map.getNumColumnsOrig());
+ return new DensityMap(out, m1Map.getNumRowsOrig(),
+ m1Map.getNumColumnsOrig(), b, false);
+ }
+
+ private DensityMap estimInternEqZero(DensityMap m1Map) {
+ MatrixBlock out = new MatrixBlock(m1Map.getNumRows(), m1Map.getNumColumns(), false);
+ m1Map.toSparsity();
+ for(int i=0; i<m1Map.getNumRows(); i++)
+ for(int j=0; j<m1Map.getNumColumns(); j++)
+ out.quickSetValue(i, j, 1-m1Map.get(i, j));
+ return new DensityMap(out, m1Map.getNumRowsOrig(),
+ m1Map.getNumColumnsOrig(), _b, m1Map._scaled);
+ }
+
public static class DensityMap {
private final MatrixBlock _map;
private final int _rlen;
@@ -207,6 +243,10 @@ public class EstimatorDensityMap extends SparsityEstimator
throw new RuntimeException("Invalid block size: "+_b);
}
+ public MatrixBlock getMap() {
+ return _map;
+ }
+
public int getNumRows() {
return _map.getNumRows();
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/0a957e4c/src/test/java/org/apache/sysml/test/integration/functions/estim/OpSingleTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/estim/OpSingleTest.java b/src/test/java/org/apache/sysml/test/integration/functions/estim/OpSingleTest.java
index fa567ed..72fccce 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/estim/OpSingleTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/estim/OpSingleTest.java
@@ -24,6 +24,7 @@ import org.apache.directory.api.util.exception.NotImplementedException;
import org.apache.sysml.hops.estim.EstimatorBasicAvg;
import org.apache.sysml.hops.estim.EstimatorBasicWorst;
import org.apache.sysml.hops.estim.EstimatorBitsetMM;
+import org.apache.sysml.hops.estim.EstimatorDensityMap;
import org.apache.sysml.hops.estim.SparsityEstimator;
import org.apache.sysml.hops.estim.SparsityEstimator.OpCode;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -49,17 +50,6 @@ public class OpSingleTest extends AutomatedTestBase
//do nothing
}
- //Average Case
-// @Test
-// public void testAvgEqzero() {
-// runSparsityEstimateTest(new EstimatorBasicAvg(), m, k, sparsity, eqzero);
-// }
-
-// @Test
-// public void testAvgDiag() {
-// runSparsityEstimateTest(new EstimatorBasicAvg(), m, k, sparsity, diag);
-// }
-
@Test
public void testAvgNeqzero() {
runSparsityEstimateTest(new EstimatorBasicAvg(), m, k, sparsity, neqzero);
@@ -74,95 +64,36 @@ public class OpSingleTest extends AutomatedTestBase
public void testAvgReshape() {
runSparsityEstimateTest(new EstimatorBasicAvg(), m, k, sparsity, reshape);
}
-
- //Worst Case
-// @Test
-// public void testWorstEqzero() {
-// runSparsityEstimateTest(new EstimatorBasicWorst(), m, k, sparsity, eqzero);
-// }
-
-// @Test
-// public void testWCasediag() {
-// runSparsityEstimateTest(new EstimatorBasicWorst(), m, k, sparsity, diag);
-// }
-
+
@Test
- public void testWorstNeqzero() {
+ public void testWCNeqzero() {
runSparsityEstimateTest(new EstimatorBasicWorst(), m, k, sparsity, neqzero);
}
@Test
- public void testWoestTrans() {
+ public void testWCTrans() {
runSparsityEstimateTest(new EstimatorBasicWorst(), m, k, sparsity, trans);
}
@Test
- public void testWorstReshape() {
+ public void testWCReshape() {
runSparsityEstimateTest(new EstimatorBasicWorst(), m, k, sparsity, reshape);
}
-// //DensityMap
-// @Test
-// public void testDMCaseeqzero() {
-// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, eqzero);
-// }
-//
-// @Test
-// public void testDMCasediag() {
-// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, diag);
-// }
-//
-// @Test
-// public void testDMCaseneqzero() {
-// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, neqzero);
-// }
-//
-// @Test
-// public void testDMCasetrans() {
-// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, trans);
-// }
-//
-// @Test
-// public void testDMCasereshape() {
-// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, reshape);
-// }
-//
-// //MNC
-// @Test
-// public void testMNCCaseeqzero() {
-// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, eqzero);
-// }
-//
-// @Test
-// public void testMNCCasediag() {
-// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, diag);
-// }
-//
-// @Test
-// public void testMNCCaseneqzero() {
-// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, neqzero);
-// }
-//
-// @Test
-// public void testMNCCasetrans() {
-// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, trans);
-// }
-//
-// @Test
-// public void testMNCCasereshape() {
-// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, reshape);
-// }
-//
- //Bitset
-// @Test
-// public void testBitsetCaseeqzero() {
-// runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, sparsity, eqzero);
-// }
+ @Test
+ public void testDMapNeqzero() {
+ runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, neqzero);
+ }
+
+ @Test
+ public void testDMapTrans() {
+ runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, trans);
+ }
-// @Test
-// public void testBitsetCasediag() {
-// runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, sparsity, diag);
-// }
+ @Test
+ public void testDMapReshape() {
+ runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, reshape);
+ }
@Test
public void testBitsetNeqzero() {
@@ -179,58 +110,6 @@ public class OpSingleTest extends AutomatedTestBase
runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, sparsity, reshape);
}
-// //Layered Graph
-// @Test
-// public void testLGCaseeqzero() {
-// runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, sparsity, eqzero);
-// }
-//
-// @Test
-// public void testLGCasediag() {
-// runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, sparsity, diag);
-// }
-//
-// @Test
-// public void testLGCaseneqzero() {
-// runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, sparsity, neqzero);
-// }
-//
-// @Test
-// public void testLGCasetans() {
-// runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, sparsity, trans);
-// }
-//
-// @Test
-// public void testLGCasereshape() {
-// runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, sparsity, reshape);
-// }
-//
-// //Sample
-// @Test
-// public void testSampleCaseeqzero() {
-// runSparsityEstimateTest(new EstimatorSample(), m, k, sparsity, eqzero);
-// }
-//
-// @Test
-// public void testSampleCasediag() {
-// runSparsityEstimateTest(new EstimatorSample(), m, k, sparsity, diag);
-// }
-//
-// @Test
-// public void testSampleCaseneqzero() {
-// runSparsityEstimateTest(new EstimatorSample(), m, k, sparsity, neqzero);
-// }
-//
-// @Test
-// public void testSampleCasetrans() {
-// runSparsityEstimateTest(new EstimatorSample(), m, k, sparsity, trans);
-// }
-//
-// @Test
-// public void testSampleCasereshape() {
-// runSparsityEstimateTest(new EstimatorSample(), m, k, sparsity, reshape);
-// }
-
private void runSparsityEstimateTest(SparsityEstimator estim, int m, int k, double sp, OpCode op) {
MatrixBlock m1 = MatrixBlock.randOperations(m, k, sp, 1, 1, "uniform", 3);
MatrixBlock m2 = new MatrixBlock();
@@ -255,6 +134,6 @@ public class OpSingleTest extends AutomatedTestBase
throw new NotImplementedException();
}
//compare estimated and real sparsity
- TestUtils.compareScalars(est, m2.getSparsity(), (estim instanceof EstimatorBasicWorst) ? 5e-1 : 1e-2);
+ TestUtils.compareScalars(est, m2.getSparsity(), 0);
}
}