You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/04/24 22:30:07 UTC
incubator-systemml git commit: [SYSTEMML-1558] Efficient
quantile/centralMoment on compressed matrices
Repository: incubator-systemml
Updated Branches:
refs/heads/master 3031d340d -> 30dcd64b4
[SYSTEMML-1558] Efficient quantile/centralMoment on compressed matrices
This patch adds support for selected order statistics over compressed
matrices, specifically quantiles and centralMoment. The core idea is to
count the frequency per value without accessing all data (e.g., only OLE
segment length fields), and perform the actual statistics computation
over the reduced input leveraging the distinct values and their counts
as weights.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/30dcd64b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/30dcd64b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/30dcd64b
Branch: refs/heads/master
Commit: 30dcd64b4058033a81b775836b6a915e15d47814
Parents: 3031d34
Author: Matthias Boehm <mb...@gmail.com>
Authored: Mon Apr 24 15:31:34 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Mon Apr 24 15:31:50 2017 -0700
----------------------------------------------------------------------
.../sysml/runtime/compress/ColGroupDDC1.java | 19 +-
.../sysml/runtime/compress/ColGroupDDC2.java | 19 +-
.../sysml/runtime/compress/ColGroupOLE.java | 19 ++
.../sysml/runtime/compress/ColGroupRLE.java | 21 ++
.../sysml/runtime/compress/ColGroupValue.java | 37 +++-
.../runtime/compress/CompressedMatrixBlock.java | 39 +++-
.../sysml/runtime/matrix/data/MatrixBlock.java | 1 -
.../compress/BasicMatrixCentralMomentTest.java | 191 +++++++++++++++++++
.../compress/BasicMatrixQuantileTest.java | 187 ++++++++++++++++++
.../functions/compress/ZPackageSuite.java | 2 +
10 files changed, 517 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC1.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC1.java b/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC1.java
index c9e5c9d..4cd103e 100644
--- a/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC1.java
+++ b/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC1.java
@@ -177,6 +177,19 @@ public class ColGroupDDC1 extends ColGroupDDC
c[i] = _values[(_data[i]&0xFF)*ncol+colpos];
target.recomputeNonZeros();
}
+
+ @Override
+ public int[] getCounts() {
+ final int nrow = getNumRows();
+ final int numVals = getNumValues();
+
+ int[] counts = new int[numVals];
+ for( int i=0; i<nrow; i++ ) {
+ counts[_data[i]&0xFF] ++;
+ }
+
+ return counts;
+ }
@Override
protected void countNonZerosPerRow(int[] rnnz, int rl, int ru) {
@@ -272,15 +285,11 @@ public class ColGroupDDC1 extends ColGroupDDC
@Override
protected void computeSum(MatrixBlock result, KahanFunction kplus) {
- final int nrow = getNumRows();
final int ncol = getNumCols();
final int numVals = getNumValues();
//iterative over codes and count per code (guaranteed <=255)
- int[] counts = new int[numVals];
- for( int i=0; i<nrow; i++ ) {
- counts[_data[i]&0xFF] ++;
- }
+ int[] counts = getCounts();
//post-scaling of pre-aggregate with distinct values
KahanObject kbuff = new KahanObject(result.quickGetValue(0, 0), result.quickGetValue(0, 1));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC2.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC2.java b/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC2.java
index 2584a47..f447aaf 100644
--- a/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC2.java
+++ b/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC2.java
@@ -180,6 +180,19 @@ public class ColGroupDDC2 extends ColGroupDDC
target.recomputeNonZeros();
}
+ @Override
+ public int[] getCounts() {
+ final int nrow = getNumRows();
+ final int numVals = getNumValues();
+
+ int[] counts = new int[numVals];
+ for( int i=0; i<nrow; i++ ) {
+ counts[_data[i]] ++;
+ }
+
+ return counts;
+ }
+
@Override
protected void countNonZerosPerRow(int[] rnnz, int rl, int ru) {
final int ncol = getNumCols();
@@ -264,17 +277,13 @@ public class ColGroupDDC2 extends ColGroupDDC
@Override
protected void computeSum(MatrixBlock result, KahanFunction kplus) {
- final int nrow = getNumRows();
final int ncol = getNumCols();
final int numVals = getNumValues();
if( numVals < MAX_TMP_VALS )
{
//iterative over codes and count per code
- int[] counts = new int[numVals];
- for( int i=0; i<nrow; i++ ) {
- counts[_data[i]] ++;
- }
+ int[] counts = getCounts();
//post-scaling of pre-aggregate with distinct values
KahanObject kbuff = new KahanObject(result.quickGetValue(0, 0), result.quickGetValue(0, 1));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/main/java/org/apache/sysml/runtime/compress/ColGroupOLE.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/compress/ColGroupOLE.java b/src/main/java/org/apache/sysml/runtime/compress/ColGroupOLE.java
index b4b9169..fab8917 100644
--- a/src/main/java/org/apache/sysml/runtime/compress/ColGroupOLE.java
+++ b/src/main/java/org/apache/sysml/runtime/compress/ColGroupOLE.java
@@ -229,6 +229,25 @@ public class ColGroupOLE extends ColGroupOffset
target.recomputeNonZeros();
}
+ @Override
+ public int[] getCounts() {
+ final int numVals = getNumValues();
+
+ int[] counts = new int[numVals];
+ for (int k = 0; k < numVals; k++) {
+ int boff = _ptr[k];
+ int blen = len(k);
+
+ //iterate over bitmap blocks and count partial lengths
+ int count = 0;
+ for (int bix=0; bix < blen; bix+=_data[boff+bix]+1)
+ count += _data[boff+bix];
+ counts[k] = count;
+ }
+
+ return counts;
+ }
+
@Override
public ColGroup scalarOperation(ScalarOperator op)
throws DMLRuntimeException
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/main/java/org/apache/sysml/runtime/compress/ColGroupRLE.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/compress/ColGroupRLE.java b/src/main/java/org/apache/sysml/runtime/compress/ColGroupRLE.java
index b9fe5d3..cd658b0 100644
--- a/src/main/java/org/apache/sysml/runtime/compress/ColGroupRLE.java
+++ b/src/main/java/org/apache/sysml/runtime/compress/ColGroupRLE.java
@@ -226,6 +226,27 @@ public class ColGroupRLE extends ColGroupOffset
target.recomputeNonZeros();
}
+ @Override
+ public int[] getCounts() {
+ final int numVals = getNumValues();
+
+ int[] counts = new int[numVals];
+ for (int k = 0; k < numVals; k++) {
+ int boff = _ptr[k];
+ int blen = len(k);
+ int curRunEnd = 0;
+ int count = 0;
+ for (int bix = 0; bix < blen; bix+=2) {
+ int curRunStartOff = curRunEnd + _data[boff+bix];
+ curRunEnd = curRunStartOff + _data[boff+bix+1];
+ count += curRunEnd-curRunStartOff;
+ }
+ counts[k] = count;
+ }
+
+ return counts;
+ }
+
@Override
public void rightMultByVector(MatrixBlock vector, MatrixBlock result, int rl, int ru)
throws DMLRuntimeException
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/main/java/org/apache/sysml/runtime/compress/ColGroupValue.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/compress/ColGroupValue.java b/src/main/java/org/apache/sysml/runtime/compress/ColGroupValue.java
index a374137..8bfc019 100644
--- a/src/main/java/org/apache/sysml/runtime/compress/ColGroupValue.java
+++ b/src/main/java/org/apache/sysml/runtime/compress/ColGroupValue.java
@@ -115,7 +115,7 @@ public abstract class ColGroupValue extends ColGroup
}
/**
- * Obtain number of distrinct sets of values associated with the bitmaps in this column group.
+ * Obtain number of distinct sets of values associated with the bitmaps in this column group.
*
* @return the number of distinct sets of values associated with the bitmaps
* in this column group
@@ -128,6 +128,41 @@ public abstract class ColGroupValue extends ColGroup
return _values;
}
+ public MatrixBlock getValuesAsBlock() {
+ boolean containsZeros = (this instanceof ColGroupOffset) ?
+ ((ColGroupOffset)this)._zeros : false;
+ int rlen = containsZeros ? _values.length+1 : _values.length;
+ MatrixBlock ret = new MatrixBlock(rlen, 1, false);
+ for( int i=0; i<_values.length; i++ )
+ ret.quickSetValue(i, 0, _values[i]);
+ return ret;
+ }
+
+ public abstract int[] getCounts();
+
+ public int[] getCounts(boolean inclZeros) {
+ int[] counts = getCounts();
+ if( inclZeros && this instanceof ColGroupOffset ) {
+ counts = Arrays.copyOf(counts, counts.length+1);
+ int sum = 0;
+ for( int i=0; i<counts.length; i++ )
+ sum += counts[i];
+ counts[counts.length-1] = getNumRows()-sum;
+ }
+ return counts;
+ }
+
+ public MatrixBlock getCountsAsBlock() {
+ return getCountsAsBlock(getCounts());
+ }
+
+ public static MatrixBlock getCountsAsBlock(int[] counts) {
+ MatrixBlock ret = new MatrixBlock(counts.length, 1, false);
+ for( int i=0; i<counts.length; i++ )
+ ret.quickSetValue(i, 0, counts[i]);
+ return ret;
+ }
+
protected int containsAllZeroValue() {
int numVals = getNumValues();
int numCols = getNumCols();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java
index fc59065..003f31a 100644
--- a/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java
@@ -89,6 +89,7 @@ import org.apache.sysml.runtime.matrix.operators.ReorgOperator;
import org.apache.sysml.runtime.matrix.operators.ScalarOperator;
import org.apache.sysml.runtime.matrix.operators.UnaryOperator;
import org.apache.sysml.runtime.util.IndexRange;
+import org.apache.sysml.runtime.util.SortUtils;
/**
* Experimental version of MatrixBlock that allows a compressed internal
@@ -1901,17 +1902,29 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable
@Override
public CM_COV_Object cmOperations(CMOperator op) throws DMLRuntimeException {
printDecompressWarning("cmOperations");
- MatrixBlock tmp = isCompressed() ? decompress() : this;
- return tmp.cmOperations(op);
+ if( !isCompressed() || isEmptyBlock() )
+ return super.cmOperations(op);
+ ColGroup grp = _colGroups.get(0);
+ if( grp instanceof ColGroupUncompressed )
+ return ((ColGroupUncompressed)grp).getData().cmOperations(op);
+
+ ColGroupValue grpVal = (ColGroupValue)grp;
+ MatrixBlock vals = grpVal.getValuesAsBlock();
+ MatrixBlock counts = ColGroupValue.getCountsAsBlock(grpVal.getCounts(true));
+ return vals.cmOperations(op, counts);
}
@Override
public CM_COV_Object cmOperations(CMOperator op, MatrixBlock weights)
throws DMLRuntimeException {
printDecompressWarning("cmOperations");
- MatrixBlock left = isCompressed() ? decompress() : this;
MatrixBlock right = getUncompressed(weights);
- return left.cmOperations(op, right);
+ if( !isCompressed() || isEmptyBlock() )
+ return super.cmOperations(op, right);
+ ColGroup grp = _colGroups.get(0);
+ if( grp instanceof ColGroupUncompressed )
+ return ((ColGroupUncompressed)grp).getData().cmOperations(op);
+ return decompress().cmOperations(op, right);
}
@Override
@@ -1937,9 +1950,23 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable
public MatrixValue sortOperations(MatrixValue weights, MatrixValue result)
throws DMLRuntimeException {
printDecompressWarning("sortOperations");
- MatrixBlock left = isCompressed() ? decompress() : this;
MatrixBlock right = getUncompressed(weights);
- return left.sortOperations(right, result);
+ if( !isCompressed() )
+ return super.sortOperations(right, result);
+ ColGroup grp = _colGroups.get(0);
+ if( grp instanceof ColGroupUncompressed )
+ return ((ColGroupUncompressed)grp).getData().sortOperations(right, result);
+
+ if( right == null ) {
+ ColGroupValue grpVal = (ColGroupValue)grp;
+ MatrixBlock vals = grpVal.getValuesAsBlock();
+ int[] counts = grpVal.getCounts(true);
+ SortUtils.sortByValue(0, vals.getNumRows(), vals.getDenseBlock(), counts);
+ MatrixBlock counts2 = ColGroupValue.getCountsAsBlock(counts);
+ return vals.sortOperations(counts2, result);
+ }
+ else
+ return decompress().sortOperations(right, result);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index f5d0a45..c458b07 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -4821,7 +4821,6 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
t += quickGetValue(i,1);
} while(t<pos && i < getNumRows());
- //System.out.println("values: " + quickGetValue(i,0) + "," + quickGetValue(i,1) + " -- " + quickGetValue(i+1,0) + "," + quickGetValue(i+1,1));
if ( quickGetValue(i,1) != 0 ) {
// i^th value is present in the data set, simply return it
if ( average ) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/test/java/org/apache/sysml/test/integration/functions/compress/BasicMatrixCentralMomentTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/compress/BasicMatrixCentralMomentTest.java b/src/test/java/org/apache/sysml/test/integration/functions/compress/BasicMatrixCentralMomentTest.java
new file mode 100644
index 0000000..b3dd43b
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/compress/BasicMatrixCentralMomentTest.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.compress;
+
+import org.apache.sysml.runtime.compress.CompressedMatrixBlock;
+import org.apache.sysml.runtime.functionobjects.CM;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.matrix.operators.CMOperator;
+import org.apache.sysml.runtime.matrix.operators.CMOperator.AggregateOperationTypes;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class BasicMatrixCentralMomentTest extends AutomatedTestBase
+{
+ private static final int rows = 2071;
+ private static final int cols = 1;
+ private static final double sparsity1 = 0.9;
+ private static final double sparsity2 = 0.1;
+ private static final double sparsity3 = 0.0;
+
+ public enum SparsityType {
+ DENSE,
+ SPARSE,
+ EMPTY,
+ }
+
+ public enum ValueType {
+ RAND, //UC
+ CONST, //RLE
+ RAND_ROUND_OLE, //OLE
+ RAND_ROUND_DDC, //RLE
+ }
+
+ @Override
+ public void setUp() {
+
+ }
+
+ @Test
+ public void testDenseRandDataCompression() {
+ runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND, true);
+ }
+
+ @Test
+ public void testSparseRandDataCompression() {
+ runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND, true);
+ }
+
+ @Test
+ public void testEmptyCompression() {
+ runMatrixAppendTest(SparsityType.EMPTY, ValueType.RAND, true);
+ }
+
+ @Test
+ public void testDenseRoundRandDataOLECompression() {
+ runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND_ROUND_OLE, true);
+ }
+
+ @Test
+ public void testSparseRoundRandDataOLECompression() {
+ runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND_ROUND_OLE, true);
+ }
+
+ @Test
+ public void testDenseRoundRandDataDDCCompression() {
+ runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND_ROUND_DDC, true);
+ }
+
+ @Test
+ public void testSparseRoundRandDataDDCCompression() {
+ runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND_ROUND_DDC, true);
+ }
+
+ @Test
+ public void testDenseConstantDataCompression() {
+ runMatrixAppendTest(SparsityType.DENSE, ValueType.CONST, true);
+ }
+
+ @Test
+ public void testSparseConstDataCompression() {
+ runMatrixAppendTest(SparsityType.SPARSE, ValueType.CONST, true);
+ }
+
+ @Test
+ public void testDenseRandDataNoCompression() {
+ runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND, false);
+ }
+
+ @Test
+ public void testSparseRandDataNoCompression() {
+ runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND, false);
+ }
+
+ @Test
+ public void testEmptyNoCompression() {
+ runMatrixAppendTest(SparsityType.EMPTY, ValueType.RAND, false);
+ }
+
+ @Test
+ public void testDenseRoundRandDataOLENoCompression() {
+ runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND_ROUND_OLE, false);
+ }
+
+ @Test
+ public void testSparseRoundRandDataOLENoCompression() {
+ runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND_ROUND_OLE, false);
+ }
+
+ @Test
+ public void testDenseConstDataNoCompression() {
+ runMatrixAppendTest(SparsityType.DENSE, ValueType.CONST, false);
+ }
+
+ @Test
+ public void testSparseConstDataNoCompression() {
+ runMatrixAppendTest(SparsityType.SPARSE, ValueType.CONST, false);
+ }
+
+
+ /**
+ *
+ * @param mb
+ */
+ private void runMatrixAppendTest(SparsityType sptype, ValueType vtype, boolean compress)
+ {
+ try
+ {
+ //prepare sparsity for input data
+ double sparsity = -1;
+ switch( sptype ){
+ case DENSE: sparsity = sparsity1; break;
+ case SPARSE: sparsity = sparsity2; break;
+ case EMPTY: sparsity = sparsity3; break;
+ }
+
+ //generate input data
+ double min = (vtype==ValueType.CONST)? 10 : -10;
+ double[][] input = TestUtils.generateTestMatrix(rows, cols, min, 10, sparsity, 7);
+ if( vtype==ValueType.RAND_ROUND_OLE || vtype==ValueType.RAND_ROUND_DDC ) {
+ CompressedMatrixBlock.ALLOW_DDC_ENCODING = (vtype==ValueType.RAND_ROUND_DDC);
+ input = TestUtils.round(input);
+ }
+ MatrixBlock mb = DataConverter.convertToMatrixBlock(input);
+
+ //compress given matrix block
+ CompressedMatrixBlock cmb = new CompressedMatrixBlock(mb);
+ if( compress )
+ cmb.compress();
+
+ //quantile uncompressed
+ AggregateOperationTypes opType = CMOperator.getCMAggOpType(2);
+ CMOperator cm = new CMOperator(CM.getCMFnObject(opType), opType);
+
+ double ret1 = mb.cmOperations(cm).getRequiredResult(opType);
+
+ //quantile compressed
+ double ret2 = cmb.cmOperations(cm).getRequiredResult(opType);
+
+ //compare result with input
+ TestUtils.compareScalars(ret1, ret2, 0.0000001);
+ }
+ catch(Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ finally {
+ CompressedMatrixBlock.ALLOW_DDC_ENCODING = true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/test/java/org/apache/sysml/test/integration/functions/compress/BasicMatrixQuantileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/compress/BasicMatrixQuantileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/compress/BasicMatrixQuantileTest.java
new file mode 100644
index 0000000..77fc471
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/functions/compress/BasicMatrixQuantileTest.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.compress;
+
+import org.apache.sysml.runtime.compress.CompressedMatrixBlock;
+import org.apache.sysml.runtime.matrix.data.MatrixBlock;
+import org.apache.sysml.runtime.util.DataConverter;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.utils.TestUtils;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class BasicMatrixQuantileTest extends AutomatedTestBase
+{
+ private static final int rows = 2071;
+ private static final int cols = 1;
+ private static final double sparsity1 = 0.9;
+ private static final double sparsity2 = 0.1;
+ private static final double sparsity3 = 0.0;
+
+ public enum SparsityType {
+ DENSE,
+ SPARSE,
+ EMPTY,
+ }
+
+ public enum ValueType {
+ RAND, //UC
+ CONST, //RLE
+ RAND_ROUND_OLE, //OLE
+ RAND_ROUND_DDC, //RLE
+ }
+
+ @Override
+ public void setUp() {
+
+ }
+
+ @Test
+ public void testDenseRandDataCompression() {
+ runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND, true);
+ }
+
+ @Test
+ public void testSparseRandDataCompression() {
+ runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND, true);
+ }
+
+ @Test
+ public void testEmptyCompression() {
+ runMatrixAppendTest(SparsityType.EMPTY, ValueType.RAND, true);
+ }
+
+ @Test
+ public void testDenseRoundRandDataOLECompression() {
+ runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND_ROUND_OLE, true);
+ }
+
+ @Test
+ public void testSparseRoundRandDataOLECompression() {
+ runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND_ROUND_OLE, true);
+ }
+
+ @Test
+ public void testDenseRoundRandDataDDCCompression() {
+ runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND_ROUND_DDC, true);
+ }
+
+ @Test
+ public void testSparseRoundRandDataDDCCompression() {
+ runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND_ROUND_DDC, true);
+ }
+
+ @Test
+ public void testDenseConstantDataCompression() {
+ runMatrixAppendTest(SparsityType.DENSE, ValueType.CONST, true);
+ }
+
+ @Test
+ public void testSparseConstDataCompression() {
+ runMatrixAppendTest(SparsityType.SPARSE, ValueType.CONST, true);
+ }
+
+ @Test
+ public void testDenseRandDataNoCompression() {
+ runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND, false);
+ }
+
+ @Test
+ public void testSparseRandDataNoCompression() {
+ runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND, false);
+ }
+
+ @Test
+ public void testEmptyNoCompression() {
+ runMatrixAppendTest(SparsityType.EMPTY, ValueType.RAND, false);
+ }
+
+ @Test
+ public void testDenseRoundRandDataOLENoCompression() {
+ runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND_ROUND_OLE, false);
+ }
+
+ @Test
+ public void testSparseRoundRandDataOLENoCompression() {
+ runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND_ROUND_OLE, false);
+ }
+
+ @Test
+ public void testDenseConstDataNoCompression() {
+ runMatrixAppendTest(SparsityType.DENSE, ValueType.CONST, false);
+ }
+
+ @Test
+ public void testSparseConstDataNoCompression() {
+ runMatrixAppendTest(SparsityType.SPARSE, ValueType.CONST, false);
+ }
+
+
+ /**
+ *
+ * @param mb
+ */
+ private void runMatrixAppendTest(SparsityType sptype, ValueType vtype, boolean compress)
+ {
+ try
+ {
+ //prepare sparsity for input data
+ double sparsity = -1;
+ switch( sptype ){
+ case DENSE: sparsity = sparsity1; break;
+ case SPARSE: sparsity = sparsity2; break;
+ case EMPTY: sparsity = sparsity3; break;
+ }
+
+ //generate input data
+ double min = (vtype==ValueType.CONST)? 10 : -10;
+ double[][] input = TestUtils.generateTestMatrix(rows, cols, min, 10, sparsity, 7);
+ if( vtype==ValueType.RAND_ROUND_OLE || vtype==ValueType.RAND_ROUND_DDC ) {
+ CompressedMatrixBlock.ALLOW_DDC_ENCODING = (vtype==ValueType.RAND_ROUND_DDC);
+ input = TestUtils.round(input);
+ }
+ MatrixBlock mb = DataConverter.convertToMatrixBlock(input);
+
+ //compress given matrix block
+ CompressedMatrixBlock cmb = new CompressedMatrixBlock(mb);
+ if( compress )
+ cmb.compress();
+
+ //quantile uncompressed
+ MatrixBlock tmp1 = (MatrixBlock) mb.sortOperations(null, new MatrixBlock());
+ double ret1 = tmp1.pickValue(0.95);
+
+ //quantile compressed
+ MatrixBlock tmp2 = (MatrixBlock) cmb.sortOperations(null, new MatrixBlock());
+ double ret2 = tmp2.pickValue(0.95);
+
+ //compare result with input
+ TestUtils.compareScalars(ret1, ret2, 0.0000001);
+ }
+ catch(Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ finally {
+ CompressedMatrixBlock.ALLOW_DDC_ENCODING = true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/test_suites/java/org/apache/sysml/test/integration/functions/compress/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/compress/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/compress/ZPackageSuite.java
index 52270e9..4563465 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/functions/compress/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/compress/ZPackageSuite.java
@@ -29,7 +29,9 @@ import org.junit.runners.Suite;
BasicCompressionTest.class,
BasicGetValueTest.class,
BasicMatrixAppendTest.class,
+ BasicMatrixCentralMomentTest.class,
BasicMatrixMultChainTest.class,
+ BasicMatrixQuantileTest.class,
BasicMatrixTransposeSelfMultTest.class,
BasicMatrixVectorMultTest.class,
BasicScalarOperationsSparseUnsafeTest.class,