You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2022/10/28 12:09:46 UTC
[systemds] 01/01: [MINOR] Add scheme for empty
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 11d07737a61d2142774c89857fb00d3338d6cc1d
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Tue Oct 25 19:55:18 2022 +0200
[MINOR] Add scheme for empty
Add a empty scheme for empty column groups.
Closes #1711
---
.../runtime/compress/colgroup/ColGroupEmpty.java | 3 +-
.../runtime/compress/colgroup/ColGroupSDC.java | 3 +-
.../runtime/compress/colgroup/ColGroupSDCFOR.java | 9 +-
.../compress/colgroup/ColGroupSDCSingle.java | 2 +-
.../compress/colgroup/ColGroupSDCSingleZeros.java | 3 +-
.../compress/colgroup/ColGroupSDCZeros.java | 9 +-
.../compress/colgroup/dictionary/Dictionary.java | 5 +-
.../colgroup/dictionary/MatrixBlockDictionary.java | 4 +-
.../compress/colgroup/scheme/ConstScheme.java | 18 +-
.../scheme/{ConstScheme.java => EmptyScheme.java} | 82 +++--
.../org/apache/sysds/runtime/data/SparseBlock.java | 12 +
.../component/compress/colgroup/ColGroupTest.java | 62 +++-
.../colgroup/scheme/CLAEmptySchemeTest.java | 354 +++++++++++++++++++++
13 files changed, 491 insertions(+), 75 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java
index 81dd5cb6d7..32d66fcdba 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.compress.colgroup.dictionary.Dictionary;
+import org.apache.sysds.runtime.compress.colgroup.scheme.EmptyScheme;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
import org.apache.sysds.runtime.compress.utils.Util;
@@ -330,6 +331,6 @@ public class ColGroupEmpty extends AColGroupCompressed {
@Override
public ICLAScheme getCompressionScheme() {
- return null;
+ return EmptyScheme.create(this);
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
index a665149aa5..db9cc6f1a5 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
@@ -586,7 +586,7 @@ public class ColGroupSDC extends ASDC implements AMapToDataGroup {
@Override
public AColGroup appendNInternal(AColGroup[] g) {
- int sumRows = 0;
+ int sumRows = getNumRows();
for(int i = 1; i < g.length; i++) {
if(!Arrays.equals(_colIndexes, g[i]._colIndexes)) {
LOG.warn("Not same columns therefore not appending \n" + Arrays.toString(_colIndexes) + "\n\n"
@@ -617,7 +617,6 @@ public class ColGroupSDC extends ASDC implements AMapToDataGroup {
return null;
}
-
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
index b381e81665..dd953c283a 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
@@ -57,7 +57,7 @@ import org.apache.sysds.runtime.matrix.operators.UnaryOperator;
* with no modifications.
*
*/
-public class ColGroupSDCFOR extends ASDC {
+public class ColGroupSDCFOR extends ASDC implements AMapToDataGroup {
private static final long serialVersionUID = 3883228464052204203L;
@@ -116,6 +116,11 @@ public class ColGroupSDCFOR extends ASDC {
return _data.getCounts(counts);
}
+ @Override
+ public AMapToData getMapToData() {
+ return _data;
+ }
+
@Override
protected void computeRowSums(double[] c, int rl, int ru, double[] preAgg) {
ColGroupSDC.computeRowSums(c, rl, ru, preAgg, _data, _indexes, _numRows);
@@ -447,7 +452,7 @@ public class ColGroupSDCFOR extends ASDC {
@Override
public AColGroup appendNInternal(AColGroup[] g) {
- int sumRows = 0;
+ int sumRows = getNumRows();
for(int i = 1; i < g.length; i++) {
if(!Arrays.equals(_colIndexes, g[i]._colIndexes)) {
LOG.warn("Not same columns therefore not appending \n" + Arrays.toString(_colIndexes) + "\n\n"
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java
index a66f4387de..739eb33379 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java
@@ -584,7 +584,7 @@ public class ColGroupSDCSingle extends ASDC {
@Override
public AColGroup appendNInternal(AColGroup[] g) {
- int sumRows = 0;
+ int sumRows = getNumRows();
for(int i = 1; i < g.length; i++) {
if(!Arrays.equals(_colIndexes, g[i]._colIndexes)) {
LOG.warn("Not same columns therefore not appending \n" + Arrays.toString(_colIndexes) + "\n\n"
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java
index 46657d2b7a..e4a53b3cfb 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java
@@ -817,7 +817,7 @@ public class ColGroupSDCSingleZeros extends ASDCZero {
@Override
public AColGroup appendNInternal(AColGroup[] g) {
- int sumRows = 0;
+ int sumRows = getNumRows();
for(int i = 1; i < g.length; i++) {
if(!Arrays.equals(_colIndexes, g[i]._colIndexes)) {
LOG.warn("Not same columns therefore not appending \n" + Arrays.toString(_colIndexes) + "\n\n"
@@ -839,7 +839,6 @@ public class ColGroupSDCSingleZeros extends ASDCZero {
}
AOffset no = _indexes.appendN(Arrays.copyOf(g, g.length, AOffsetsGroup[].class), getNumRows());
return create(_colIndexes, sumRows, _dict, no, null);
-
}
@Override
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java
index 3ffb070b26..f926cdac6f 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java
@@ -56,7 +56,7 @@ import org.apache.sysds.runtime.matrix.operators.UnaryOperator;
*
* This column group is handy in cases where sparse unsafe operations is executed on very sparse columns.
*/
-public class ColGroupSDCZeros extends ASDCZero {
+public class ColGroupSDCZeros extends ASDCZero implements AMapToDataGroup{
private static final long serialVersionUID = -3703199743391937991L;
/** Pointers to row indexes in the dictionary. Note the dictionary has one extra entry. */
@@ -89,6 +89,11 @@ public class ColGroupSDCZeros extends ASDCZero {
return ColGroupType.SDCZeros;
}
+ @Override
+ public AMapToData getMapToData(){
+ return _data;
+ }
+
@Override
protected void decompressToDenseBlockDenseDictionary(DenseBlock db, int rl, int ru, int offR, int offC,
double[] values) {
@@ -728,7 +733,7 @@ public class ColGroupSDCZeros extends ASDCZero {
@Override
public AColGroup appendNInternal(AColGroup[] g) {
- int sumRows = 0;
+ int sumRows = getNumRows();
for(int i = 1; i < g.length; i++) {
if(!Arrays.equals(_colIndexes, g[i]._colIndexes)) {
LOG.warn("Not same columns therefore not appending \n" + Arrays.toString(_colIndexes) + "\n\n"
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java
index 7cfc49e32b..852cc733c1 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java
@@ -26,7 +26,6 @@ import java.math.BigDecimal;
import java.math.MathContext;
import java.util.Arrays;
-import org.apache.commons.lang.NotImplementedException;
import org.apache.sysds.runtime.compress.DMLCompressionException;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.functionobjects.Builtin;
@@ -1070,10 +1069,10 @@ public class Dictionary extends ADictionary {
public boolean eq(ADictionary o) {
if(o instanceof Dictionary)
return Arrays.equals(_values, ((Dictionary) o)._values);
- else if(o instanceof MatrixBlockDictionary){
+ else if(o instanceof MatrixBlockDictionary) {
final MatrixBlock mb = ((MatrixBlockDictionary) o).getMatrixBlock();
if(mb.isInSparseFormat())
- throw new NotImplementedException();
+ return mb.getSparseBlock().equals(_values, mb.getNumColumns());
final double[] dv = mb.getDenseBlockValues();
return Arrays.equals(_values, dv);
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java
index d575f5232f..b13efbe4c1 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java
@@ -2147,10 +2147,10 @@ public class MatrixBlockDictionary extends ADictionary {
@Override
public boolean eq(ADictionary o) {
if(o instanceof MatrixBlockDictionary)
- throw new NotImplementedException("Comparison if a MatrixBlock is equivalent is not implemented yet");
+ return _data.equals(((MatrixBlockDictionary) o)._data);
else if(o instanceof Dictionary) {
if(_data.isInSparseFormat())
- throw new NotImplementedException();
+ return _data.getSparseBlock().equals(((Dictionary) o)._values, _data.getNumColumns());
final double[] dv = _data.getDenseBlockValues();
return Arrays.equals(dv, ((Dictionary) o)._values);
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/ConstScheme.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/ConstScheme.java
index 1a65f1417f..91896d5c3b 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/ConstScheme.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/ConstScheme.java
@@ -77,7 +77,7 @@ public class ConstScheme implements ICLAScheme {
if(dv[off + cols[ci]] != values[ci])
return null;
}
- return g;
+ return returnG(cols);
}
private AColGroup encodeSparse(final MatrixBlock data, final int[] cols, final double[] values, final int nRow,
@@ -92,7 +92,9 @@ public class ConstScheme implements ICLAScheme {
final double[] aval = sb.values(r);
final int[] aix = sb.indexes(r);
int p = 0; // pointer into cols;
- while(p < cols.length && values[p] == 0.0)
+ while(values[p] == 0.0)
+ // technically also check for&& p < cols.length
+ // but this verification is indirectly maintained
p++;
for(int j = apos; j < alen && p < cols.length; j++) {
if(aix[j] == cols[p]) {
@@ -106,7 +108,7 @@ public class ConstScheme implements ICLAScheme {
return null; // not matching
}
}
- return g;
+ return returnG(cols);
}
private AColGroup encodeGeneric(final MatrixBlock data, final int[] cols, final double[] values, final int nRow,
@@ -115,6 +117,14 @@ public class ConstScheme implements ICLAScheme {
for(int ci = 0; ci < cols.length; ci++)
if(data.quickGetValue(r, cols[ci]) != values[ci])
return null;
- return g;
+ return returnG(cols);
}
+
+ private AColGroup returnG(int[] columns) {
+ if(columns == g.getColIndices())
+ return g;// great!
+ else
+ return ColGroupConst.create(columns, g.getValues());
+ }
+
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/ConstScheme.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/EmptyScheme.java
similarity index 59%
copy from src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/ConstScheme.java
copy to src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/EmptyScheme.java
index 1a65f1417f..fff5b70981 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/ConstScheme.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/scheme/EmptyScheme.java
@@ -20,101 +20,95 @@
package org.apache.sysds.runtime.compress.colgroup.scheme;
import org.apache.sysds.runtime.compress.colgroup.AColGroup;
-import org.apache.sysds.runtime.compress.colgroup.ColGroupConst;
+import org.apache.sysds.runtime.compress.colgroup.ColGroupEmpty;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
-public class ConstScheme implements ICLAScheme {
+public class EmptyScheme implements ICLAScheme {
+ /** The instance of a empty column group that in all cases here would be returned to be the same */
+ final ColGroupEmpty g;
- /** The instance of a constant column group that in all cases here would be returned to be the same */
- final ColGroupConst g;
-
- protected ConstScheme(ColGroupConst g) {
+ protected EmptyScheme(ColGroupEmpty g) {
this.g = g;
}
- public static ICLAScheme create(ColGroupConst g) {
- return new ConstScheme(g);
+ public static EmptyScheme create(ColGroupEmpty g) {
+ return new EmptyScheme(g);
}
@Override
public AColGroup encode(MatrixBlock data) {
- return encode(data, g.getColIndices(), g.getValues());
+ return encode(data, g.getColIndices());
}
@Override
public AColGroup encode(MatrixBlock data, int[] columns) {
+
if(columns.length != g.getColIndices().length)
throw new IllegalArgumentException("Invalid columns to encode");
- return encode(data, columns, g.getValues());
- }
-
- private AColGroup encode(final MatrixBlock data, final int[] cols, final double[] values) {
final int nCol = data.getNumColumns();
final int nRow = data.getNumRows();
- if(nCol < cols[cols.length - 1]) {
+ if(nCol < columns[columns.length - 1]) {
LOG.warn("Invalid to encode matrix with less columns than encode scheme max column");
return null;
}
- else if(data.isEmpty()) {
- LOG.warn("Invalid to encode an empty matrix into constant column group");
- return null; // Invalid to encode this.
- }
+ else if(data.isEmpty())
+ return returnG(columns);
else if(data.isInSparseFormat())
- return encodeSparse(data, cols, values, nRow, nCol);
+ return encodeSparse(data, columns, nRow, nCol);
else if(data.getDenseBlock().isContiguous())
- return encodeDense(data, cols, values, nRow, nCol);
+ return encodeDense(data, columns, nRow, nCol);
else
- return encodeGeneric(data, cols, values, nRow, nCol);
+ return encodeGeneric(data, columns, nRow, nCol);
}
- private AColGroup encodeDense(final MatrixBlock data, final int[] cols, final double[] values, final int nRow,
- final int nCol) {
+ private AColGroup encodeDense(final MatrixBlock data, final int[] cols, final int nRow, final int nCol) {
final double[] dv = data.getDenseBlockValues();
for(int r = 0; r < nRow; r++) {
final int off = r * nCol;
for(int ci = 0; ci < cols.length; ci++)
- if(dv[off + cols[ci]] != values[ci])
+ if(dv[off + cols[ci]] != 0.0)
return null;
}
return g;
}
- private AColGroup encodeSparse(final MatrixBlock data, final int[] cols, final double[] values, final int nRow,
- final int nCol) {
+ private AColGroup encodeSparse(final MatrixBlock data, final int[] cols, final int nRow, final int nCol) {
SparseBlock sb = data.getSparseBlock();
for(int r = 0; r < nRow; r++) {
if(sb.isEmpty(r))
- return null;
+ continue; // great!
final int apos = sb.pos(r);
final int alen = apos + sb.size(r);
- final double[] aval = sb.values(r);
final int[] aix = sb.indexes(r);
int p = 0; // pointer into cols;
- while(p < cols.length && values[p] == 0.0)
- p++;
- for(int j = apos; j < alen && p < cols.length; j++) {
- if(aix[j] == cols[p]) {
- if(aval[j] != values[p])
- return null;
+ for(int j = apos; j < alen ; j++) {
+ while(p < cols.length && cols[p] < aix[j])
p++;
- while(p < cols.length && values[p] == 0.0)
- p++;
- }
- else if(aix[j] > cols[p])
- return null; // not matching
+ if(p < cols.length && aix[j] == cols[p])
+ return null;
+
+ if(p >= cols.length)
+ continue;
}
}
- return g;
+ return returnG(cols);
}
- private AColGroup encodeGeneric(final MatrixBlock data, final int[] cols, final double[] values, final int nRow,
- final int nCol) {
+ private AColGroup encodeGeneric(final MatrixBlock data, final int[] cols, final int nRow, final int nCol) {
for(int r = 0; r < nRow; r++)
for(int ci = 0; ci < cols.length; ci++)
- if(data.quickGetValue(r, cols[ci]) != values[ci])
+ if(data.quickGetValue(r, cols[ci]) != 0.0)
return null;
- return g;
+ return returnG(cols);
+ }
+
+ private AColGroup returnG(int[] columns) {
+ if(columns == g.getColIndices())
+ return g;// great!
+ else
+ return new ColGroupEmpty(columns);
}
+
}
diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlock.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlock.java
index e5310dc23c..bbddb9a178 100644
--- a/src/main/java/org/apache/sysds/runtime/data/SparseBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlock.java
@@ -573,6 +573,18 @@ public abstract class SparseBlock implements Serializable, Block
return true;
}
+
+ /**
+ * Get if the dense double array is equivalent to this sparse Block.
+ *
+ * @param denseValues row major double values same dimensions of sparse Block.
+ * @param nCol Number of columns in dense values (and hopefully in this sparse block)
+ * @return If the dense array is equivalent
+ */
+ public boolean equals(double[] denseValues, int nCol) {
+ return equals(denseValues, nCol, Double.MIN_NORMAL * 1024);
+ }
+
/**
* Get if the dense double array is equivalent to this sparse Block.
*
diff --git a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupTest.java b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupTest.java
index fa3d1d44d3..f609f0bcea 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupTest.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupTest.java
@@ -2112,18 +2112,6 @@ public class ColGroupTest extends ColGroupBase {
assertTrue(co < eo);
}
- // @Test
- // public void copyMaintainPointers() {
- // AColGroup a = base.copy();
- // AColGroup b = other.copy();
-
- // assertTrue(a.getColIndices() == base.getColIndices());
- // assertTrue(b.getColIndices() == other.getColIndices());
- // // assertFalse(a.getColIndices() == other.getColIndices());
- // assertFalse(a == base);
- // assertFalse(b == other);
- // }
-
@Test
public void sliceRowsBeforeEnd() {
if(nRow > 10)
@@ -2241,4 +2229,54 @@ public class ColGroupTest extends ColGroupBase {
fail(e.getMessage());
}
}
+
+ @Test
+ public void testAppendSelf() {
+ appendSelfVerification(base);
+ appendSelfVerification(other);
+ }
+
+ @Test
+ public void testAppendSomethingElse() {
+ // This is under the assumption that if one is appending
+ // to the other then other should append to this.
+ // If this property does not hold it is because some cases are missing in the append logic.
+ try {
+
+ AColGroup g2 = base.append(other);
+ AColGroup g2n = other.append(base);
+ // both should be null, or both should not be.
+ if(g2 == null)
+ assertTrue(g2n == null);
+ else if(g2 != null)
+ assertTrue(g2n != null);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private void appendSelfVerification(AColGroup g) {
+ try {
+
+ AColGroup g2 = g.append(g);
+ AColGroup g2n = AColGroup.appendN(new AColGroup[] {g, g});
+
+ if(g2 != null && g2n != null) {
+ double s2 = g2.getSum(nRow * 2);
+ double s = g.getSum(nRow) * 2;
+ double s2n = g2n.getSum(nRow * 2);
+ assertEquals(s2, s, 0.0001);
+ assertEquals(s2n, s, 0.0001);
+
+ UA_ROW(InstructionUtils.parseBasicAggregateUnaryOperator("uar+", 1), 0, nRow * 2, g2, g2n, nRow * 2);
+ }
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
}
diff --git a/src/test/java/org/apache/sysds/test/component/compress/colgroup/scheme/CLAEmptySchemeTest.java b/src/test/java/org/apache/sysds/test/component/compress/colgroup/scheme/CLAEmptySchemeTest.java
new file mode 100644
index 0000000000..bfcf2012b8
--- /dev/null
+++ b/src/test/java/org/apache/sysds/test/component/compress/colgroup/scheme/CLAEmptySchemeTest.java
@@ -0,0 +1,354 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.component.compress.colgroup.scheme;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.sysds.runtime.compress.colgroup.AColGroup;
+import org.apache.sysds.runtime.compress.colgroup.ColGroupEmpty;
+import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
+import org.apache.sysds.runtime.data.DenseBlockFP64;
+import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+import org.junit.Test;
+
+public class CLAEmptySchemeTest {
+
+ private final AColGroup g;
+ private final ICLAScheme sh;
+
+ public CLAEmptySchemeTest() {
+ g = new ColGroupEmpty(//
+ new int[] {1, 3, 5} // Columns
+ );
+ sh = g.getCompressionScheme();
+ }
+
+ @Test
+ public void testConstValid() {
+ assertTrue(sh != null);
+ }
+
+ @Test
+ public void testToSmallMatrix() {
+ assertTrue(sh.encode(new MatrixBlock(1, 3, new double[] {//
+ 1.1, 1.2, 1.3})) == null);
+ }
+
+ @Test
+ public void testWrongValuesSingleRow() {
+ assertTrue(sh.encode(new MatrixBlock(1, 6, new double[] {//
+ 0.0, 1.1, 0.2, 1.2, 0.2, 1.2})) == null);
+ }
+
+ @Test
+ public void testWrongValuesSingleRowV2() {
+ assertTrue(sh.encode(new MatrixBlock(1, 6, new double[] {//
+ 0.0, 1.0, 0.2, 1.2, 0.2, 1.3})) == null);
+ }
+
+ @Test
+ public void testValidEncodeSingleRow() {
+ assertTrue(sh.encode(new MatrixBlock(1, 6, new double[] {//
+ 0.1, 0.0, 0.04, 0.0, 0.03, 0.0})) != null);
+ }
+
+ @Test
+ public void testValidEncodeMultiRow() {
+ assertTrue(sh.encode(new MatrixBlock(2, 6, new double[] {//
+ 132, 0.0, 241, 0.0, 142, 0.0, //
+ 132, 0.0, 241, 0.0, 142, 0.0, //
+ })) != null);
+ }
+
+ @Test
+ public void testValidEncodeMultiRowsLarger() {
+ assertTrue(sh.encode(new MatrixBlock(2, 10, new double[] {//
+ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1, 1, 1, 1, //
+ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1, 1, 1, 1, //
+ })) != null);
+ }
+
+ @Test
+ public void testInvalidEncodeMultiRowsValue() {
+ assertTrue(sh.encode(new MatrixBlock(4, 8, new double[] {//
+ 0.0, 0.0, 0.2, 0.0, 1.2, 0.0, 0.2, 1.3, //
+ 0.0, 0.0, 0.2, 0.0, 1.2, 0.0, 0.2, 1.3, //
+ 0.0, 0.0, 0.2, 0.0, 1.2, 0.0, 0.2, 1.3, //
+ 0.0, 0.0, 0.2, 0.0, 1.2, 0.0, 0.2, 1.3, //
+ })) != null);
+ }
+
+ @Test
+ public void testValidEncodeMultiRowDifferentValuesOtherColumns() {
+ assertTrue(sh.encode(new MatrixBlock(4, 12, new double[] {//
+ 0.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.1, 0.4, 1.2, 0.3, 1.3, //
+ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.1, 0.2, 1.2, 0.2, 1.3, //
+ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.1, 0.2, 1.2, 0.1, 1.3, //
+ 0.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.1, 0.4, 1.2, 0.1, 1.3, //
+ })) != null);
+ }
+
+ @Test
+ public void testInvalidEncodeValueMultiRowMultiError() {
+ assertTrue(sh.encode(new MatrixBlock(4, 6, new double[] {//
+ 0.0, 1.1, 0.2, 1.2, 0.2, 1.3, //
+ 0.0, 1.1, 0.2, 1.2, 0.2, 1.3, //
+ 0.0, 1.1, 0.2, 1.2, 0.2, 1.4, //
+ 0.0, 1.1, 0.2, 1.2, 0.2, 1.3, //
+ })) == null);
+ }
+
+ @Test
+ public void testInvalidEncodeMultiRow() {
+ assertTrue(sh.encode(new MatrixBlock(4, 6, new double[] {//
+ 0.0, 1.3, 0.2, 1.2, 0.2, 1.3, //
+ 0.0, 1.1, 0.2, 1.2, 0.2, 1.3, //
+ 0.0, 1.1, 0.2, 1.2, 0.2, 1.4, //
+ 0.0, 1.1, 0.2, 1.2, 0.2, 1.3, //
+ })) == null);
+ }
+
+ @Test
+ public void testEncodeOtherColumns() {
+ assertTrue(sh.encode(new MatrixBlock(4, 5, new double[] {//
+ 1.1, 0.2, 1.2, 0.2, 1.3, //
+ 1.1, 0.2, 1.2, 0.2, 1.3, //
+ 1.1, 0.2, 1.2, 0.2, 1.3, //
+ 1.1, 0.2, 1.2, 0.2, 1.3, //
+ }), new int[] {0, 2, 4}// other columns
+ ) == null);
+ }
+
+ @Test
+ public void testEncodeOtherColumnsValid() {
+ assertTrue(sh.encode(new MatrixBlock(4, 8, new double[] {//
+ 0.0, 1.1, 0.0, 0.2, 0.0, 1.2, 0.2, 1.3, //
+ 0.0, 1.1, 0.0, 0.2, 0.0, 1.2, 0.2, 1.3, //
+ 0.0, 1.1, 0.0, 0.2, 0.0, 1.2, 0.2, 1.3, //
+ 0.0, 1.1, 0.0, 0.2, 0.0, 1.2, 0.2, 1.3, //
+ }), new int[] {0, 2, 4}// other columns
+ ) != null);
+ }
+
+ @Test
+ public void testEncodeOtherColumnsInvalid() {
+ assertTrue(sh.encode(new MatrixBlock(4, 5, new double[] {//
+ 1.1, 0.2, 1.2, 0.2, 1.3, //
+ 1.1, 0.2, 1.2, 0.2, 1.3, //
+ 1.1, 0.2, 1.4, 0.2, 1.3, //
+ 1.1, 0.2, 1.2, 0.2, 1.3, //
+ }), new int[] {0, 2, 4}// other columns
+ ) == null);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidArgument_1() {
+ sh.encode(null, new int[] {0, 2, 4, 5});
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidArgument_2() {
+ sh.encode(null, new int[] {0, 2});
+ }
+
+ @Test
+ public void testSparse() {
+ MatrixBlock mb = new MatrixBlock(4, 6, new double[] {//
+ 0.01, 0.0, 0.2, 0.0, 0.2, 0.0, //
+ 0.01, 0.0, 0.2, 0.0, 0.2, 0.0, //
+ 0.01, 0.0, 0.2, 0.0, 0.2, 0.0, //
+ 0.01, 0.0, 0.2, 0.0, 0.2, 0.0, //
+ });
+
+ MatrixBlock empty = new MatrixBlock(4, 1000, 0.0);
+ mb = mb.append(empty);
+
+ assertTrue(sh.encode(mb) != null);
+ }
+
+ @Test
+ public void testSpars_AllCosOver() {
+ MatrixBlock mb = new MatrixBlock(4, 6, new double[] {//
+ 0.01, 0.0, 0.2, 0.0, 0.2, 0.0, //
+ 0.01, 0.0, 0.2, 0.0, 0.2, 0.0, //
+ 0.01, 0.0, 0.2, 0.0, 0.2, 0.0, //
+ 0.01, 0.0, 0.2, 0.0, 0.2, 0.0, //
+ });
+
+ MatrixBlock empty = new MatrixBlock(4, 1000, 0.0);
+ mb = mb.append(empty);
+
+ assertTrue(sh.encode(mb, new int[] {100, 102, 999}) != null);
+ }
+
+ @Test
+ public void testSpars_InsideInvalid() {
+ MatrixBlock mb = new MatrixBlock(4, 6, new double[] {//
+ 0.01, 0.0, 0.2, 0.0, 0.2, 0.0, //
+ 0.01, 0.0, 0.2, 0.0, 0.2, 0.0, //
+ 0.01, 0.0, 0.2, 0.0, 0.2, 0.0, //
+ 0.01, 0.0, 0.2, 0.0, 0.2, 0.0, //
+ });
+
+ MatrixBlock empty = new MatrixBlock(4, 1000, 0.0);
+ mb = mb.append(empty);
+
+ assertTrue(sh.encode(mb, new int[] {1, 4, 5}) == null);
+ }
+
+ @Test
+ public void testSparse_Append() {
+ MatrixBlock mb = new MatrixBlock(4, 6, new double[] {//
+ 0.0, 0.0, 0.2, 0.0, 0.2, 1.3, //
+ 0.0, 0.0, 0.2, 0.0, 0.2, 1.3, //
+ 0.0, 0.0, 0.2, 0.0, 0.2, 1.3, //
+ 0.0, 0.0, 0.2, 0.0, 0.2, 1.3, //
+ });
+
+ MatrixBlock empty = new MatrixBlock(4, 1000, 0.0);
+ mb = empty.append(mb);
+
+ assertTrue(sh.encode(mb) != null);
+ }
+
+ @Test
+ public void testSparseValidCustom() {
+ MatrixBlock mb = new MatrixBlock(4, 9, new double[] {//
+ 0.0, 0.0, 1.1, 0.0, 0.2, 0.0, 1.2, 0.2, 1.3, //
+ 0.0, 0.0, 1.1, 0.0, 0.2, 0.0, 1.2, 0.2, 1.3, //
+ 0.0, 0.0, 1.1, 0.0, 0.2, 0.0, 1.2, 0.2, 1.3, //
+ 0.0, 0.0, 1.1, 0.0, 0.2, 0.0, 1.2, 0.2, 1.3, //
+ });
+
+ MatrixBlock empty = new MatrixBlock(4, 1000, 0.0);
+ mb = empty.append(mb);
+
+ assertTrue(sh.encode(mb, new int[] {1001, 1003, 1005}) != null);
+ }
+
+ @Test
+ public void testSparseValidCustom2() {
+ MatrixBlock mb = new MatrixBlock(4, 9, new double[] {//
+ 0.0, 0.0, 1.1, 0.0, 0.2, 0.0, 1.2, 0.2, 1.3, //
+ 0.0, 0.0, 1.1, 0.0, 0.2, 0.0, 1.2, 0.2, 1.3, //
+ 0.0, 0.0, 1.1, 0.0, 0.2, 0.0, 1.2, 0.2, 1.3, //
+ 0.0, 0.0, 1.1, 0.0, 0.2, 0.0, 1.2, 0.2, 1.3, //
+ });
+
+ MatrixBlock empty = new MatrixBlock(4, 1000, 0.0);
+ MatrixBlock comb = empty.append(mb).append(mb);
+
+ assertTrue(sh.encode(comb, new int[] {1001, 1003, 1005}) != null);
+ }
+
+ @Test
+ public void testSparseValidCustom3Valid() {
+ MatrixBlock mb = new MatrixBlock(4, 9, new double[] {//
+ 0.0, 0.0, 1.1, 0.0, 0.2, 0.0, 1.2, 0.2, 1.3, //
+ 0.0, 0.0, 1.1, 0.0, 0.2, 0.0, 1.33, 0.2, 1.3, //
+ 0.0, 0.0, 1.1, 0.0, 0.2, 0.0, 1.2, 0.2, 1.3, //
+ 0.0, 0.0, 1.1, 0.0, 0.2, 0.0, 1.2, 0.2, 1.3, //
+ });
+
+ MatrixBlock empty = new MatrixBlock(4, 1000, 0.0);
+ MatrixBlock comb = empty.append(mb).append(mb);
+
+ assertTrue(sh.encode(comb, new int[] {1001, 1003, 1005}) != null);
+ }
+
+ @Test
+ public void testSparseEmptyRow() {
+ MatrixBlock mb = new MatrixBlock(4, 6, new double[] {//
+ 0.0, 1.1, 0.2, 1.2, 0.2, 1.3, //
+ 0.0, 1.1, 0.2, 1.2, 0.2, 1.3, //
+ 0.0, 1.1, 0.2, 1.2, 0.2, 1.3, //
+ 0.0, 1.1, 0.2, 1.2, 0.2, 1.3, //
+ });
+
+ MatrixBlock empty = new MatrixBlock(4, 1000, 0.0);
+ mb = empty.append(mb);
+ MatrixBlock emptyRow = new MatrixBlock(1, 1006, 0.0);
+ mb = mb.append(emptyRow, false);
+
+ assertTrue(sh.encode(mb, new int[] {44, 45, 999}) != null);
+ }
+
+ @Test
+ public void testEmpty() {
+ MatrixBlock empty = new MatrixBlock(4, 1000, 0.0);
+ assertTrue(sh.encode(empty) != null);
+ }
+
+ @Test
+ public void testEmptyOtherColumns() {
+ MatrixBlock empty = new MatrixBlock(4, 1000, 0.0);
+ assertTrue(sh.encode(empty, new int[] {33, 34, 99}) != null);
+ }
+
+ @Test
+ public void testGenericNonContinuosBlockValid() {
+ MatrixBlock mb = new MatrixBlock(4, 6, //
+ new DenseBlockFP64Mock(new int[] {4, 9}, new double[] {//
+ 0.2, 0.0, 1.1, 0.0, 0.4, 0.0, 1.2, 0.3, 1.3, //
+ 0.0, 0.0, 1.1, 0.0, 0.2, 0.0, 1.2, 0.2, 1.3, //
+ 0.0, 0.0, 1.1, 0.0, 0.2, 0.0, 1.2, 0.1, 1.3, //
+ 0.2, 0.0, 1.1, 0.0, 0.4, 0.0, 1.2, 0.1, 1.3, //
+ }));
+ mb.recomputeNonZeros();
+ assertTrue(sh.encode(mb) != null);
+ }
+
+ @Test
+ public void testGenericNonContinuosBlockInValid() {
+ MatrixBlock mb = new MatrixBlock(4, 6, //
+ new DenseBlockFP64Mock(new int[] {4, 6}, new double[] {//
+ 0.2, 1.1, 0.4, 1.2, 0.3, 1.3, //
+ 0.0, 1.1, 0.2, 1.2, 0.2, 1.3, //
+ 0.0, 1.1, 0.2, 1.2, 0.1, 1.3, //
+ 0.2, 1.22, 0.4, 1.2, 0.1, 1.3, //
+ }));
+ mb.recomputeNonZeros();
+ assertTrue(sh.encode(mb) == null);
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testNull() {
+ sh.encode(null, null);
+ }
+
+ private class DenseBlockFP64Mock extends DenseBlockFP64 {
+ private static final long serialVersionUID = -3601232958390554672L;
+
+ public DenseBlockFP64Mock(int[] dims, double[] data) {
+ super(dims, data);
+ }
+
+ @Override
+ public boolean isContiguous() {
+ return false;
+ }
+
+ @Override
+ public int numBlocks() {
+ return 2;
+ }
+ }
+
+}