You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2023/05/29 12:22:06 UTC
[systemds] branch main updated: [SYSTEMDS-3573] Combine Column groups
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 7f333a4f65 [SYSTEMDS-3573] Combine Column groups
7f333a4f65 is described below
commit 7f333a4f6558e10a6d50cc66911b6eb285df2c13
Author: baunsgaard <ba...@tu-berlin.de>
AuthorDate: Tue May 23 14:54:48 2023 +0200
[SYSTEMDS-3573] Combine Column groups
This commit contains the code to combine basic column groups with minimal
optimizations. Cases supported include DDC, SDC(all variants), Const and
Empty but not FOR instances.
Close #1832
---
.../sysds/runtime/compress/colgroup/AColGroup.java | 49 +-
.../compress/colgroup/AColGroupCompressed.java | 2 +
.../compress/colgroup/ADictBasedColGroup.java | 17 +-
.../sysds/runtime/compress/colgroup/APreAgg.java | 2 -
.../sysds/runtime/compress/colgroup/ASDC.java | 4 +-
.../sysds/runtime/compress/colgroup/ASDCZero.java | 7 +-
.../runtime/compress/colgroup/ColGroupConst.java | 36 +-
.../runtime/compress/colgroup/ColGroupDDC.java | 16 +-
.../runtime/compress/colgroup/ColGroupDDCFOR.java | 17 +
.../runtime/compress/colgroup/ColGroupEmpty.java | 33 +-
.../colgroup/ColGroupLinearFunctional.java | 16 +-
.../runtime/compress/colgroup/ColGroupOLE.java | 8 +-
.../runtime/compress/colgroup/ColGroupRLE.java | 11 +-
.../runtime/compress/colgroup/ColGroupSDC.java | 31 +-
.../runtime/compress/colgroup/ColGroupSDCFOR.java | 26 +-
.../compress/colgroup/ColGroupSDCSingle.java | 28 ++
.../compress/colgroup/ColGroupSDCSingleZeros.java | 18 +
.../compress/colgroup/ColGroupSDCZeros.java | 21 +-
.../compress/colgroup/ColGroupUncompressed.java | 38 +-
.../runtime/compress/colgroup/ColGroupUtils.java | 16 +-
...apToDataGroup.java => IContainADictionary.java} | 6 +-
...pToDataGroup.java => IContainDefaultTuple.java} | 7 +-
.../{AMapToDataGroup.java => IMapToDataGroup.java} | 2 +-
.../compress/colgroup/dictionary/ADictionary.java | 27 +-
.../compress/colgroup/dictionary/Dictionary.java | 22 +
.../colgroup/dictionary/DictionaryFactory.java | 244 ++++++++++
.../colgroup/dictionary/IdentityDictionary.java | 10 +
.../colgroup/dictionary/MatrixBlockDictionary.java | 34 +-
.../compress/colgroup/dictionary/QDictionary.java | 22 +-
.../compress/colgroup/indexes/ArrayIndex.java | 36 ++
.../compress/colgroup/indexes/ColIndexFactory.java | 58 ++-
.../compress/colgroup/indexes/IColIndex.java | 25 +-
.../compress/colgroup/indexes/IIterate.java | 14 +
.../compress/colgroup/indexes/RangeIndex.java | 52 ++-
.../compress/colgroup/indexes/SingleIndex.java | 33 +-
.../compress/colgroup/indexes/TwoIndex.java | 31 ++
.../compress/colgroup/mapping/AMapToData.java | 4 +-
.../compress/colgroup/mapping/MapToBit.java | 6 +-
.../compress/colgroup/mapping/MapToByte.java | 6 +-
.../compress/colgroup/mapping/MapToChar.java | 6 +-
.../compress/colgroup/mapping/MapToCharPByte.java | 4 +-
.../compress/colgroup/mapping/MapToInt.java | 6 +-
.../compress/colgroup/mapping/MapToZero.java | 6 +-
.../compress/estim/encoding/ConstEncoding.java | 5 +
.../compress/estim/encoding/DenseEncoding.java | 64 ++-
.../compress/estim/encoding/EmptyEncoding.java | 5 +
.../compress/estim/encoding/EncodingFactory.java | 18 +
.../runtime/compress/estim/encoding/IEncode.java | 9 +
.../compress/estim/encoding/SparseEncoding.java | 62 ++-
.../runtime/compress/lib/CLALibCombineGroups.java | 124 +++++
.../compress/colgroup/ColGroupNegativeTests.java | 20 +-
.../compress/colgroup/NegativeConstTests.java | 2 +-
.../component/compress/dictionary/CombineTest.java | 505 +++++++++++++++++++++
.../compress/dictionary/CustomDictionaryTest.java | 1 +
.../compress/indexes/CustomIndexTest.java | 215 ++++++++-
.../component/compress/indexes/IndexesTest.java | 18 +-
.../component/compress/lib/CombineGroupsTest.java | 396 ++++++++++++++++
.../component/compress/mapping/MappingTests.java | 8 +-
.../frame/compress/FrameCompressTest.java} | 12 +-
59 files changed, 2385 insertions(+), 136 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java
index c8e25f9920..b3356db9da 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java
@@ -32,6 +32,8 @@ import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex.SliceResult;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
+import org.apache.sysds.runtime.compress.estim.encoding.IEncode;
+import org.apache.sysds.runtime.compress.lib.CLALibCombineGroups;
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.instructions.cp.CM_COV_Object;
@@ -55,6 +57,18 @@ public abstract class AColGroup implements Serializable {
/** Public super types of compression ColGroups supported */
public static enum CompressionType {
UNCOMPRESSED, RLE, OLE, DDC, CONST, EMPTY, SDC, SDCFOR, DDCFOR, DeltaDDC, LinearFunctional;
+
+ public boolean isDense() {
+ return this == DDC || this == CONST || this == DDCFOR || this == DDCFOR;
+ }
+
+ public boolean isConst() {
+ return this == CONST || this == EMPTY;
+ }
+
+ public boolean isSDC() {
+ return this == SDC;
+ }
}
/**
@@ -119,7 +133,7 @@ public abstract class AColGroup implements Serializable {
* @param colIndexes the new indexes to use in the copy
* @return a new object with pointers to underlying data.
*/
- protected abstract AColGroup copyAndSet(IColIndex colIndexes);
+ public abstract AColGroup copyAndSet(IColIndex colIndexes);
/**
* Get the upper bound estimate of in memory allocation for the column group.
@@ -631,7 +645,7 @@ public abstract class AColGroup implements Serializable {
* @param ct The compressionType that the column group should morph into
* @return A new column group
*/
- public AColGroup morph(CompressionType ct){
+ public AColGroup morph(CompressionType ct) {
throw new NotImplementedException();
}
@@ -643,6 +657,37 @@ public abstract class AColGroup implements Serializable {
*/
public abstract CompressedSizeInfoColGroup getCompressionInfo(int nRow);
+ /**
+ * Combine this column group with another
+ *
+ * @param other The other column group to combine with.
+ * @return A combined representation as a column group.
+ */
+ public AColGroup combine(AColGroup other) {
+ return CLALibCombineGroups.combine(this, other);
+ }
+
+ /**
+ * Get encoding of this column group.
+ *
+ * @return The encoding of the index structure.
+ */
+ public IEncode getEncoding() {
+ throw new NotImplementedException();
+ }
+
+ public AColGroup sortColumnIndexes() {
+ if(_colIndexes.isSorted())
+ return this;
+ else {
+ int[] reorderingIndex = _colIndexes.getReorderingIndex();
+ IColIndex ni = _colIndexes.sort();
+ return fixColIndexes(ni, reorderingIndex);
+ }
+ }
+
+ protected abstract AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering);
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupCompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupCompressed.java
index bd7367503d..00d23345f7 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupCompressed.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupCompressed.java
@@ -86,6 +86,8 @@ public abstract class AColGroupCompressed extends AColGroup {
protected abstract double[] preAggBuiltinRows(Builtin builtin);
+ public abstract boolean sameIndexStructure(AColGroupCompressed that);
+
public double[] preAggRows(ValueFunction fn) {
// final ValueFunction fn = op.aggOp.increOp.fn;
if(fn instanceof KahanPlusSq)
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictBasedColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictBasedColGroup.java
index e27ffcd9c7..d1f81a73c8 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictBasedColGroup.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictBasedColGroup.java
@@ -34,7 +34,7 @@ import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
-public abstract class ADictBasedColGroup extends AColGroupCompressed {
+public abstract class ADictBasedColGroup extends AColGroupCompressed implements IContainADictionary {
private static final long serialVersionUID = -3737025296618703668L;
/** Distinct value tuples associated with individual bitmaps. */
protected final ADictionary _dict;
@@ -59,9 +59,9 @@ public abstract class ADictBasedColGroup extends AColGroupCompressed {
@Override
public final void decompressToDenseBlock(DenseBlock db, int rl, int ru, int offR, int offC) {
- if(_dict instanceof IdentityDictionary){
+ if(_dict instanceof IdentityDictionary) {
- final MatrixBlockDictionary md = ((IdentityDictionary)_dict).getMBDict();
+ final MatrixBlockDictionary md = ((IdentityDictionary) _dict).getMBDict();
final MatrixBlock mb = md.getMatrixBlock();
// The dictionary is never empty.
if(mb.isInSparseFormat())
@@ -84,9 +84,9 @@ public abstract class ADictBasedColGroup extends AColGroupCompressed {
@Override
public final void decompressToSparseBlock(SparseBlock sb, int rl, int ru, int offR, int offC) {
- if(_dict instanceof IdentityDictionary){
+ if(_dict instanceof IdentityDictionary) {
- final MatrixBlockDictionary md = ((IdentityDictionary)_dict).getMBDict();
+ final MatrixBlockDictionary md = ((IdentityDictionary) _dict).getMBDict();
final MatrixBlock mb = md.getMatrixBlock();
// The dictionary is never empty.
if(mb.isInSparseFormat())
@@ -203,7 +203,8 @@ public abstract class ADictBasedColGroup extends AColGroupCompressed {
return allocateRightMultiplication(right, agCols, preAgg);
}
- protected abstract AColGroup allocateRightMultiplication(MatrixBlock right, IColIndex colIndexes, ADictionary preAgg);
+ protected abstract AColGroup allocateRightMultiplication(MatrixBlock right, IColIndex colIndexes,
+ ADictionary preAgg);
/**
* Find the minimum number of columns that are effected by the right multiplication
@@ -295,10 +296,10 @@ public abstract class ADictBasedColGroup extends AColGroupCompressed {
}
@Override
- protected final AColGroup copyAndSet(IColIndex colIndexes){
+ public final AColGroup copyAndSet(IColIndex colIndexes) {
return copyAndSet(colIndexes, _dict);
}
-
+
protected final AColGroup copyAndSet(ADictionary newDictionary) {
return copyAndSet(_colIndexes, newDictionary);
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/APreAgg.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/APreAgg.java
index c968fc0f14..60308bf564 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/APreAgg.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/APreAgg.java
@@ -140,8 +140,6 @@ public abstract class APreAgg extends AColGroupValue {
protected abstract void preAggregateThatRLEStructure(ColGroupRLE that, Dictionary ret);
- protected abstract boolean sameIndexStructure(AColGroupCompressed that);
-
public int getPreAggregateSize() {
return getNumValues();
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDC.java
index 1865d4721e..24e187f8f4 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDC.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDC.java
@@ -30,7 +30,7 @@ import org.apache.sysds.runtime.compress.colgroup.offset.AOffset;
* This column group is handy in cases where sparse unsafe operations is executed on very sparse columns. Then the zeros
* would be materialized in the group without any overhead.
*/
-public abstract class ASDC extends AMorphingMMColGroup implements AOffsetsGroup {
+public abstract class ASDC extends AMorphingMMColGroup implements AOffsetsGroup , IContainDefaultTuple {
private static final long serialVersionUID = 769993538831949086L;
/** Sparse row indexes for the data */
@@ -48,8 +48,6 @@ public abstract class ASDC extends AMorphingMMColGroup implements AOffsetsGroup
return _numRows;
}
- public abstract double[] getDefaultTuple();
-
@Override
public AOffset getOffsets() {
return _indexes;
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java
index 3b7592dfd6..6e63e3bbbd 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java
@@ -28,7 +28,7 @@ import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
-public abstract class ASDCZero extends APreAgg implements AOffsetsGroup {
+public abstract class ASDCZero extends APreAgg implements AOffsetsGroup, IContainDefaultTuple {
private static final long serialVersionUID = -69266306137398807L;
/** Sparse row indexes for the data */
@@ -215,4 +215,9 @@ public abstract class ASDCZero extends APreAgg implements AOffsetsGroup {
public AOffset getOffsets() {
return _indexes;
}
+
+ @Override
+ public double[] getDefaultTuple() {
+ return new double[_colIndexes.size()];
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java
index c053fa9d6d..87044290db 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java
@@ -34,6 +34,8 @@ import org.apache.sysds.runtime.compress.colgroup.scheme.ConstScheme;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
+import org.apache.sysds.runtime.compress.estim.encoding.EncodingFactory;
+import org.apache.sysds.runtime.compress.estim.encoding.IEncode;
import org.apache.sysds.runtime.compress.lib.CLALibLeftMultBy;
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
@@ -45,7 +47,7 @@ import org.apache.sysds.runtime.matrix.operators.CMOperator;
import org.apache.sysds.runtime.matrix.operators.ScalarOperator;
import org.apache.sysds.runtime.matrix.operators.UnaryOperator;
-public class ColGroupConst extends ADictBasedColGroup {
+public class ColGroupConst extends ADictBasedColGroup implements IContainDefaultTuple {
private static final long serialVersionUID = -7387793538322386611L;
@@ -67,7 +69,7 @@ public class ColGroupConst extends ADictBasedColGroup {
* @param dict The dictionary to use
* @return A Colgroup either const or empty.
*/
- protected static AColGroup create(IColIndex colIndices, ADictionary dict) {
+ public static AColGroup create(IColIndex colIndices, ADictionary dict) {
if(dict == null)
return new ColGroupEmpty(colIndices);
else
@@ -256,14 +258,14 @@ public class ColGroupConst extends ADictBasedColGroup {
ret.append(offT, _colIndexes.get(j) + offC, _dict.getValue(j));
}
- private final void decompressToDenseBlockAllColumnsContiguous(final DenseBlock db, final int rl, final int ru) {
+ private final void decompressToDenseBlockAllColumnsContiguous(final DenseBlock db, final int rl, final int ru) {
final double[] c = db.values(0);
final int nCol = _colIndexes.size();
final double[] values = _dict.getValues();
final int start = rl * nCol;
final int end = ru * nCol;
for(int i = start; i < end; i++)
- c[i] += values[i % nCol];
+ c[i] += values[i % nCol];
}
private void decompressToDenseBlockGeneric(DenseBlock db, int rl, int ru, int offR, int offC) {
@@ -307,7 +309,7 @@ public class ColGroupConst extends ADictBasedColGroup {
* @param constV The output columns.
*/
public final void addToCommon(double[] constV) {
- if(_dict instanceof IdentityDictionary){
+ if(_dict instanceof IdentityDictionary) {
MatrixBlock mb = ((IdentityDictionary) _dict).getMBDict().getMatrixBlock();
if(mb.isInSparseFormat())
addToCommonSparse(constV, mb.getSparseBlock());
@@ -568,15 +570,35 @@ public class ColGroupConst extends ADictBasedColGroup {
}
@Override
- public AColGroup recompress(){
+ public AColGroup recompress() {
return this;
}
@Override
- public CompressedSizeInfoColGroup getCompressionInfo(int nRow){
+ public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
return new CompressedSizeInfoColGroup(_colIndexes, 1, nRow, CompressionType.CONST);
}
+ @Override
+ public IEncode getEncoding() {
+ return EncodingFactory.create(this);
+ }
+
+ @Override
+ public boolean sameIndexStructure(AColGroupCompressed that) {
+ return that instanceof ColGroupEmpty || that instanceof ColGroupConst;
+ }
+
+ @Override
+ public double[] getDefaultTuple() {
+ return _dict.getValues();
+ }
+
+ @Override
+ protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering){
+ return ColGroupConst.create(newColIndex, _dict.reorder(reordering));
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
index 06b10c1126..544e61b6ba 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
@@ -38,6 +38,8 @@ import org.apache.sysds.runtime.compress.colgroup.scheme.DDCScheme;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
+import org.apache.sysds.runtime.compress.estim.encoding.EncodingFactory;
+import org.apache.sysds.runtime.compress.estim.encoding.IEncode;
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.functionobjects.Builtin;
@@ -51,7 +53,7 @@ import org.apache.sysds.runtime.matrix.operators.UnaryOperator;
/**
* Class to encapsulate information about a column group that is encoded with dense dictionary encoding (DDC).
*/
-public class ColGroupDDC extends APreAgg implements AMapToDataGroup {
+public class ColGroupDDC extends APreAgg implements IMapToDataGroup {
private static final long serialVersionUID = -5769772089913918987L;
protected final AMapToData _data;
@@ -536,7 +538,7 @@ public class ColGroupDDC extends APreAgg implements AMapToDataGroup {
return null;
}
}
- AMapToData nd = _data.appendN(Arrays.copyOf(g, g.length, AMapToDataGroup[].class));
+ AMapToData nd = _data.appendN(Arrays.copyOf(g, g.length, IMapToDataGroup[].class));
return create(_colIndexes, _dict, nd, null);
}
@@ -555,6 +557,16 @@ public class ColGroupDDC extends APreAgg implements AMapToDataGroup {
throw new NotImplementedException();
}
+ @Override
+ public IEncode getEncoding() {
+ return EncodingFactory.create(_data);
+ }
+
+ @Override
+ protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) {
+ return ColGroupDDC.create(newColIndex, _dict.reorder(reordering), _data, getCachedCounts());
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java
index d05b663f85..bb0abe8b2f 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java
@@ -36,6 +36,8 @@ import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
+import org.apache.sysds.runtime.compress.estim.encoding.EncodingFactory;
+import org.apache.sysds.runtime.compress.estim.encoding.IEncode;
import org.apache.sysds.runtime.functionobjects.Builtin;
import org.apache.sysds.runtime.functionobjects.Divide;
import org.apache.sysds.runtime.functionobjects.Minus;
@@ -469,6 +471,21 @@ public class ColGroupDDCFOR extends AMorphingMMColGroup {
throw new NotImplementedException();
}
+ @Override
+ public IEncode getEncoding() {
+ return EncodingFactory.create(_data);
+ }
+
+ @Override
+ public boolean sameIndexStructure(AColGroupCompressed that) {
+ return that instanceof ColGroupDDCFOR && ((ColGroupDDCFOR) that)._data == _data;
+ }
+
+ @Override
+ protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) {
+ throw new NotImplementedException();
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java
index cd788a1318..1e1b847782 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.util.Arrays;
import org.apache.sysds.runtime.DMLRuntimeException;
+import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
import org.apache.sysds.runtime.compress.colgroup.dictionary.Dictionary;
import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory;
import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
@@ -32,6 +33,8 @@ import org.apache.sysds.runtime.compress.colgroup.scheme.EmptyScheme;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
+import org.apache.sysds.runtime.compress.estim.encoding.EncodingFactory;
+import org.apache.sysds.runtime.compress.estim.encoding.IEncode;
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.functionobjects.Builtin;
@@ -43,7 +46,7 @@ import org.apache.sysds.runtime.matrix.operators.CMOperator;
import org.apache.sysds.runtime.matrix.operators.ScalarOperator;
import org.apache.sysds.runtime.matrix.operators.UnaryOperator;
-public class ColGroupEmpty extends AColGroupCompressed {
+public class ColGroupEmpty extends AColGroupCompressed implements IContainADictionary, IContainDefaultTuple {
private static final long serialVersionUID = -2307677253622099958L;
/**
@@ -316,7 +319,7 @@ public class ColGroupEmpty extends AColGroupCompressed {
}
@Override
- protected AColGroup copyAndSet(IColIndex colIndexes) {
+ public AColGroup copyAndSet(IColIndex colIndexes) {
return new ColGroupEmpty(colIndexes);
}
@@ -349,4 +352,30 @@ public class ColGroupEmpty extends AColGroupCompressed {
public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
return new CompressedSizeInfoColGroup(_colIndexes, 0, nRow, CompressionType.CONST);
}
+
+ @Override
+ public IEncode getEncoding() {
+ return EncodingFactory.create(this);
+ }
+
+ @Override
+ public ADictionary getDictionary() {
+ return null;
+ }
+
+ @Override
+ public double[] getDefaultTuple() {
+ return new double[getNumCols()];
+ }
+
+ @Override
+ public boolean sameIndexStructure(AColGroupCompressed that) {
+ return that instanceof ColGroupEmpty || that instanceof ColGroupConst;
+ }
+
+ @Override
+ protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) {
+ return new ColGroupEmpty(newColIndex);
+ }
+
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java
index 452d5e1f07..6e24eec583 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java
@@ -664,7 +664,7 @@ public class ColGroupLinearFunctional extends AColGroupCompressed {
}
@Override
- protected AColGroup copyAndSet(IColIndex colIndexes) {
+ public AColGroup copyAndSet(IColIndex colIndexes) {
return ColGroupLinearFunctional.create(colIndexes, _coefficents, _numRows);
}
@@ -684,13 +684,23 @@ public class ColGroupLinearFunctional extends AColGroupCompressed {
}
@Override
- public AColGroup recompress(){
+ public AColGroup recompress() {
return this;
}
@Override
- public CompressedSizeInfoColGroup getCompressionInfo(int nRow){
+ public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
throw new NotImplementedException("Not Implemented Compressed SizeInfo for Linear col group");
}
+ @Override
+ public boolean sameIndexStructure(AColGroupCompressed that) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) {
+ throw new NotImplementedException();
+ }
+
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java
index 961782522c..cf958fd273 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java
@@ -621,7 +621,7 @@ public class ColGroupOLE extends AColGroupOffset {
}
@Override
- protected boolean sameIndexStructure(AColGroupCompressed that) {
+ public boolean sameIndexStructure(AColGroupCompressed that) {
throw new NotImplementedException();
}
@@ -683,4 +683,10 @@ public class ColGroupOLE extends AColGroupOffset {
public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
throw new NotImplementedException();
}
+
+ @Override
+ protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) {
+ throw new NotImplementedException();
+ }
+
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java
index 909808ee89..ea08baacfd 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java
@@ -49,8 +49,8 @@ import org.apache.sysds.runtime.matrix.operators.UnaryOperator;
public class ColGroupRLE extends AColGroupOffset {
private static final long serialVersionUID = -1560710477952862791L;
- private ColGroupRLE(IColIndex colIndexes, int numRows, boolean zeros, ADictionary dict, char[] bitmaps, int[] bitmapOffs,
- int[] cachedCounts) {
+ private ColGroupRLE(IColIndex colIndexes, int numRows, boolean zeros, ADictionary dict, char[] bitmaps,
+ int[] bitmapOffs, int[] cachedCounts) {
super(colIndexes, numRows, zeros, dict, bitmapOffs, bitmaps, cachedCounts);
}
@@ -886,7 +886,7 @@ public class ColGroupRLE extends AColGroupOffset {
}
@Override
- protected boolean sameIndexStructure(AColGroupCompressed that) {
+ public boolean sameIndexStructure(AColGroupCompressed that) {
if(that.getCompType() == this.getCompType()) {
final ColGroupRLE rle = (ColGroupRLE) that;
return rle._ptr == this._ptr && rle._data == this._data;
@@ -996,6 +996,11 @@ public class ColGroupRLE extends AColGroupOffset {
throw new NotImplementedException();
}
+ @Override
+ protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) {
+ throw new NotImplementedException();
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
index 500986bd23..25e7d5be42 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
@@ -42,6 +42,8 @@ import org.apache.sysds.runtime.compress.colgroup.offset.OffsetFactory;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
+import org.apache.sysds.runtime.compress.estim.encoding.EncodingFactory;
+import org.apache.sysds.runtime.compress.estim.encoding.IEncode;
import org.apache.sysds.runtime.functionobjects.Builtin;
import org.apache.sysds.runtime.instructions.cp.CM_COV_Object;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
@@ -57,7 +59,7 @@ import org.apache.sysds.runtime.matrix.operators.UnaryOperator;
* This column group is handy in cases where sparse unsafe operations is executed on very sparse columns. Then the zeros
* would be materialized in the group without any overhead.
*/
-public class ColGroupSDC extends ASDC implements AMapToDataGroup {
+public class ColGroupSDC extends ASDC implements IMapToDataGroup {
private static final long serialVersionUID = 769993538831949086L;
/** Pointers to row indexes in the dictionary. */
@@ -617,7 +619,7 @@ public class ColGroupSDC extends ASDC implements AMapToDataGroup {
}
sumRows += gc.getNumRows();
}
- AMapToData nd = _data.appendN(Arrays.copyOf(g, g.length, AMapToDataGroup[].class));
+ AMapToData nd = _data.appendN(Arrays.copyOf(g, g.length, IMapToDataGroup[].class));
AOffset no = _indexes.appendN(Arrays.copyOf(g, g.length, AOffsetsGroup[].class), getNumRows());
return create(_colIndexes, sumRows, _dict, _defaultTuple, no, nd, null);
@@ -638,6 +640,31 @@ public class ColGroupSDC extends ASDC implements AMapToDataGroup {
throw new NotImplementedException();
}
+ @Override
+ public IEncode getEncoding() {
+ return EncodingFactory.create(_data, _indexes, _numRows);
+ }
+
+ @Override
+ protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) {
+ return ColGroupSDC.create(newColIndex, getNumRows(), _dict.reorder(reordering),
+ ColGroupUtils.reorderDefault(_defaultTuple, reordering), _indexes, _data, getCachedCounts());
+ }
+
+ @Override
+ public boolean sameIndexStructure(AColGroupCompressed that) {
+ if(that instanceof ColGroupSDCZeros) {
+ ColGroupSDCZeros th = (ColGroupSDCZeros) that;
+ return th._indexes == _indexes && th._data == _data;
+ }
+ else if(that instanceof ColGroupSDC) {
+ ColGroupSDC th = (ColGroupSDC) that;
+ return th._indexes == _indexes && th._data == _data;
+ }
+ else
+ return false;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
index 2969d9fc04..edca22ad2b 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
@@ -40,6 +40,8 @@ import org.apache.sysds.runtime.compress.colgroup.offset.OffsetFactory;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
+import org.apache.sysds.runtime.compress.estim.encoding.EncodingFactory;
+import org.apache.sysds.runtime.compress.estim.encoding.IEncode;
import org.apache.sysds.runtime.functionobjects.Builtin;
import org.apache.sysds.runtime.functionobjects.Divide;
import org.apache.sysds.runtime.functionobjects.Minus;
@@ -61,7 +63,7 @@ import org.apache.sysds.runtime.matrix.operators.UnaryOperator;
* with no modifications.
*
*/
-public class ColGroupSDCFOR extends ASDC implements AMapToDataGroup {
+public class ColGroupSDCFOR extends ASDC implements IMapToDataGroup {
private static final long serialVersionUID = 3883228464052204203L;
@@ -480,7 +482,7 @@ public class ColGroupSDCFOR extends ASDC implements AMapToDataGroup {
}
sumRows += gc.getNumRows();
}
- AMapToData nd = _data.appendN(Arrays.copyOf(g, g.length, AMapToDataGroup[].class));
+ AMapToData nd = _data.appendN(Arrays.copyOf(g, g.length, IMapToDataGroup[].class));
AOffset no = _indexes.appendN(Arrays.copyOf(g, g.length, AOffsetsGroup[].class), getNumRows());
return create(_colIndexes, sumRows, _dict, no, nd, null, _reference);
}
@@ -500,6 +502,26 @@ public class ColGroupSDCFOR extends ASDC implements AMapToDataGroup {
throw new NotImplementedException();
}
+ @Override
+ public IEncode getEncoding() {
+ return EncodingFactory.create(_data, _indexes, _numRows);
+ }
+
+ @Override
+ public boolean sameIndexStructure(AColGroupCompressed that) {
+ if(that instanceof ColGroupSDCFOR) {
+ ColGroupSDCFOR th = (ColGroupSDCFOR) that;
+ return th._indexes == _indexes && th._data == _data;
+ }
+ else
+ return false;
+ }
+
+ @Override
+ protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) {
+ throw new NotImplementedException();
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java
index 79e70ad52e..d05adbc6f9 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java
@@ -31,6 +31,7 @@ import org.apache.sysds.runtime.compress.colgroup.dictionary.Dictionary;
import org.apache.sysds.runtime.compress.colgroup.dictionary.DictionaryFactory;
import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory;
import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
+import org.apache.sysds.runtime.compress.colgroup.mapping.MapToZero;
import org.apache.sysds.runtime.compress.colgroup.offset.AIterator;
import org.apache.sysds.runtime.compress.colgroup.offset.AOffset;
import org.apache.sysds.runtime.compress.colgroup.offset.AOffset.OffsetSliceInfo;
@@ -39,6 +40,8 @@ import org.apache.sysds.runtime.compress.colgroup.offset.OffsetFactory;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
+import org.apache.sysds.runtime.compress.estim.encoding.EncodingFactory;
+import org.apache.sysds.runtime.compress.estim.encoding.IEncode;
import org.apache.sysds.runtime.functionobjects.Builtin;
import org.apache.sysds.runtime.instructions.cp.CM_COV_Object;
import org.apache.sysds.runtime.matrix.operators.BinaryOperator;
@@ -600,6 +603,31 @@ public class ColGroupSDCSingle extends ASDC {
throw new NotImplementedException();
}
+ @Override
+ public IEncode getEncoding() {
+ return EncodingFactory.create(new MapToZero(getCounts()[0]), _indexes, _numRows);
+ }
+
+ @Override
+ public boolean sameIndexStructure(AColGroupCompressed that) {
+ if(that instanceof ColGroupSDCSingleZeros) {
+ ColGroupSDCSingleZeros th = (ColGroupSDCSingleZeros) that;
+ return th._indexes == _indexes;
+ }
+ else if(that instanceof ColGroupSDCSingle) {
+ ColGroupSDCSingle th = (ColGroupSDCSingle) that;
+ return th._indexes == _indexes;
+ }
+ else
+ return false;
+ }
+
+ @Override
+ protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) {
+ return ColGroupSDCSingle.create(newColIndex, getNumRows(), _dict.reorder(reordering),
+ ColGroupUtils.reorderDefault(_defaultTuple, reordering), _indexes, getCachedCounts());
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java
index 565874d83a..d882fba14e 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java
@@ -32,6 +32,7 @@ import org.apache.sysds.runtime.compress.colgroup.dictionary.DictionaryFactory;
import org.apache.sysds.runtime.compress.colgroup.dictionary.MatrixBlockDictionary;
import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory;
import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
+import org.apache.sysds.runtime.compress.colgroup.mapping.MapToZero;
import org.apache.sysds.runtime.compress.colgroup.offset.AIterator;
import org.apache.sysds.runtime.compress.colgroup.offset.AOffset;
import org.apache.sysds.runtime.compress.colgroup.offset.AOffset.OffsetSliceInfo;
@@ -40,6 +41,8 @@ import org.apache.sysds.runtime.compress.colgroup.offset.OffsetFactory;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
+import org.apache.sysds.runtime.compress.estim.encoding.EncodingFactory;
+import org.apache.sysds.runtime.compress.estim.encoding.IEncode;
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.functionobjects.Builtin;
@@ -587,10 +590,20 @@ public class ColGroupSDCSingleZeros extends ASDCZero {
ColGroupSDCSingleZeros th = (ColGroupSDCSingleZeros) that;
return th._indexes == _indexes;
}
+ else if(that instanceof ColGroupSDCSingle) {
+ ColGroupSDCSingle th = (ColGroupSDCSingle) that;
+ return th._indexes == _indexes;
+ }
else
return false;
}
+ @Override
+ protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) {
+ return ColGroupSDCSingleZeros.create(newColIndex, getNumRows(), _dict.reorder(reordering), _indexes,
+ getCachedCounts());
+ }
+
@Override
public void preAggregateThatDDCStructure(ColGroupDDC that, Dictionary ret) {
final AOffsetIterator itThis = _indexes.getOffsetIterator();
@@ -865,6 +878,11 @@ public class ColGroupSDCSingleZeros extends ASDCZero {
throw new NotImplementedException();
}
+ @Override
+ public IEncode getEncoding() {
+ return EncodingFactory.create(new MapToZero(getCounts()[0]), _indexes, _numRows);
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java
index 0931526051..457926aeb6 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java
@@ -42,6 +42,8 @@ import org.apache.sysds.runtime.compress.colgroup.offset.OffsetFactory;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
+import org.apache.sysds.runtime.compress.estim.encoding.EncodingFactory;
+import org.apache.sysds.runtime.compress.estim.encoding.IEncode;
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.functionobjects.Builtin;
@@ -62,7 +64,7 @@ import org.apache.sysds.runtime.matrix.operators.UnaryOperator;
*
* This column group is handy in cases where sparse unsafe operations is executed on very sparse columns.
*/
-public class ColGroupSDCZeros extends ASDCZero implements AMapToDataGroup {
+public class ColGroupSDCZeros extends ASDCZero implements IMapToDataGroup {
private static final long serialVersionUID = -3703199743391937991L;
/** Pointers to row indexes in the dictionary. Note the dictionary has one extra entry. */
@@ -599,6 +601,10 @@ public class ColGroupSDCZeros extends ASDCZero implements AMapToDataGroup {
ColGroupSDCZeros th = (ColGroupSDCZeros) that;
return th._indexes == _indexes && th._data == _data;
}
+ else if(that instanceof ColGroupSDC) {
+ ColGroupSDC th = (ColGroupSDC) that;
+ return th._indexes == _indexes && th._data == _data;
+ }
else
return false;
}
@@ -783,7 +789,7 @@ public class ColGroupSDCZeros extends ASDCZero implements AMapToDataGroup {
}
sumRows += gc.getNumRows();
}
- AMapToData nd = _data.appendN(Arrays.copyOf(g, g.length, AMapToDataGroup[].class));
+ AMapToData nd = _data.appendN(Arrays.copyOf(g, g.length, IMapToDataGroup[].class));
AOffset no = _indexes.appendN(Arrays.copyOf(g, g.length, AOffsetsGroup[].class), getNumRows());
return create(_colIndexes, sumRows, _dict, no, nd, null);
@@ -804,6 +810,17 @@ public class ColGroupSDCZeros extends ASDCZero implements AMapToDataGroup {
throw new NotImplementedException();
}
+ @Override
+ public IEncode getEncoding() {
+ return EncodingFactory.create(_data, _indexes, _numRows);
+ }
+
+ @Override
+ protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) {
+ return ColGroupSDCZeros.create(newColIndex, getNumRows(), _dict.reorder(reordering), _indexes, _data,
+ getCachedCounts());
+ }
+
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(super.toString());
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
index 1ce1e1963b..bb2633a9fb 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
@@ -23,9 +23,12 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
+import java.util.List;
import org.apache.commons.lang.NotImplementedException;
import org.apache.sysds.runtime.DMLRuntimeException;
+import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
+import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory;
import org.apache.sysds.runtime.compress.DMLCompressionException;
import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
import org.apache.sysds.runtime.compress.colgroup.dictionary.DictLibMatrixMult;
@@ -807,7 +810,20 @@ public class ColGroupUncompressed extends AColGroup {
@Override
public AColGroup recompress() {
- return this;
+ MatrixBlock mb = CompressedMatrixBlockFactory.compress(_data).getLeft();
+ if(mb instanceof CompressedMatrixBlock) {
+ CompressedMatrixBlock cmb = (CompressedMatrixBlock) mb;
+ List<AColGroup> gs = cmb.getColGroups();
+ if(gs.size() > 1) {
+ LOG.error("The uncompressed column group did compress into multiple groups");
+ return this;
+ }
+ else {
+ return gs.get(0).copyAndSet(_colIndexes);
+ }
+ }
+ else
+ return this;
}
@Override
@@ -815,6 +831,21 @@ public class ColGroupUncompressed extends AColGroup {
throw new NotImplementedException();
}
+ @Override
+ public AColGroup copyAndSet(IColIndex colIndexes) {
+ return ColGroupUncompressed.create(_data, colIndexes);
+ }
+
+ @Override
+ protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) {
+ MatrixBlock ret = new MatrixBlock(_data.getNumRows(), _data.getNumColumns(), _data.getNonZeros());
+ // TODO add sparse optmization
+ for(int r = 0; r < _data.getNumRows(); r++)
+ for(int c = 0; c < _data.getNumColumns(); c++)
+ ret.quickSetValue(r, c, _data.quickGetValue(r, reordering[c]));
+ return create(newColIndex, ret, false);
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
@@ -833,9 +864,4 @@ public class ColGroupUncompressed extends AColGroup {
return sb.toString();
}
-
- @Override
- protected AColGroup copyAndSet(IColIndex colIndexes) {
- return ColGroupUncompressed.create(_data, colIndexes);
- }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUtils.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUtils.java
index 5961414b77..df6d2f1165 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUtils.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUtils.java
@@ -206,8 +206,8 @@ public interface ColGroupUtils {
}
}
- public static void outerProduct(final double[] leftRowSum, final double[] rightColumnSum, final IColIndex colIdxRight,
- final double[] result, final int nColR, final int rl, final int ru) {
+ public static void outerProduct(final double[] leftRowSum, final double[] rightColumnSum,
+ final IColIndex colIdxRight, final double[] result, final int nColR, final int rl, final int ru) {
for(int row = rl; row < ru; row++) {
final int offOut = nColR * row;
final double vLeft = leftRowSum[row];
@@ -216,8 +216,8 @@ public interface ColGroupUtils {
}
}
- public static void outerProduct(final double[] leftRowSum, final SparseBlock rightColSum, final IColIndex colIdxRight,
- final double[] result, final int nColR, final int rl, final int ru) {
+ public static void outerProduct(final double[] leftRowSum, final SparseBlock rightColSum,
+ final IColIndex colIdxRight, final double[] result, final int nColR, final int rl, final int ru) {
final int alen = rightColSum.size(0);
final int[] aix = rightColSum.indexes(0);
final double[] aval = rightColSum.values(0);
@@ -275,7 +275,6 @@ public interface ColGroupUtils {
ref[i] = counters[i].getMostFrequent();
}
-
public static void addMatrixToResult(MatrixBlock tmp, MatrixBlock result, IColIndex colIndexes, int rl, int ru) {
if(tmp.isEmpty())
return;
@@ -304,4 +303,11 @@ public interface ColGroupUtils {
}
}
+ public static double[] reorderDefault(double[] vals, int[] reordering){
+ double[] ret = new double[vals.length];
+ for(int i = 0; i < vals.length; i++)
+ ret[i] = vals[reordering[i]];
+ return ret;
+ }
+
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AMapToDataGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/IContainADictionary.java
similarity index 85%
copy from src/main/java/org/apache/sysds/runtime/compress/colgroup/AMapToDataGroup.java
copy to src/main/java/org/apache/sysds/runtime/compress/colgroup/IContainADictionary.java
index 50b65898a1..966233f6ad 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AMapToDataGroup.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/IContainADictionary.java
@@ -19,8 +19,8 @@
package org.apache.sysds.runtime.compress.colgroup;
-import org.apache.sysds.runtime.compress.colgroup.mapping.AMapToData;
+import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
-public interface AMapToDataGroup {
- public AMapToData getMapToData();
+public interface IContainADictionary {
+ public ADictionary getDictionary();
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AMapToDataGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/IContainDefaultTuple.java
similarity index 86%
copy from src/main/java/org/apache/sysds/runtime/compress/colgroup/AMapToDataGroup.java
copy to src/main/java/org/apache/sysds/runtime/compress/colgroup/IContainDefaultTuple.java
index 50b65898a1..ced00275d1 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AMapToDataGroup.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/IContainDefaultTuple.java
@@ -17,10 +17,9 @@
* under the License.
*/
-package org.apache.sysds.runtime.compress.colgroup;
-import org.apache.sysds.runtime.compress.colgroup.mapping.AMapToData;
+package org.apache.sysds.runtime.compress.colgroup;
-public interface AMapToDataGroup {
- public AMapToData getMapToData();
+public interface IContainDefaultTuple {
+ public double[] getDefaultTuple();
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AMapToDataGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/IMapToDataGroup.java
similarity index 96%
copy from src/main/java/org/apache/sysds/runtime/compress/colgroup/AMapToDataGroup.java
copy to src/main/java/org/apache/sysds/runtime/compress/colgroup/IMapToDataGroup.java
index 50b65898a1..6bdf12e772 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AMapToDataGroup.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/IMapToDataGroup.java
@@ -21,6 +21,6 @@ package org.apache.sysds.runtime.compress.colgroup;
import org.apache.sysds.runtime.compress.colgroup.mapping.AMapToData;
-public interface AMapToDataGroup {
+public interface IMapToDataGroup {
public AMapToData getMapToData();
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/ADictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/ADictionary.java
index 20ccd2b4a1..973a737351 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/ADictionary.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/ADictionary.java
@@ -836,7 +836,8 @@ public abstract class ADictionary implements Serializable {
* @param colsRight Offset cols on the right
* @param result The output matrix block
*/
- protected abstract void TSMMToUpperTriangle(ADictionary right, IColIndex rowsLeft, IColIndex colsRight, MatrixBlock result);
+ protected abstract void TSMMToUpperTriangle(ADictionary right, IColIndex rowsLeft, IColIndex colsRight,
+ MatrixBlock result);
/**
* Matrix multiplication but allocate output in upper triangle and twice if on diagonal, note this is right
@@ -846,7 +847,8 @@ public abstract class ADictionary implements Serializable {
* @param colsRight Offset cols on the right
* @param result The output matrix block
*/
- protected abstract void TSMMToUpperTriangleDense(double[] left, IColIndex rowsLeft, IColIndex colsRight, MatrixBlock result);
+ protected abstract void TSMMToUpperTriangleDense(double[] left, IColIndex rowsLeft, IColIndex colsRight,
+ MatrixBlock result);
/**
* Matrix multiplication but allocate output in upper triangle and twice if on diagonal, note this is right
@@ -868,8 +870,8 @@ public abstract class ADictionary implements Serializable {
* @param scale Scale factor
* @param result The output matrix block
*/
- protected abstract void TSMMToUpperTriangleScaling(ADictionary right, IColIndex rowsLeft, IColIndex colsRight, int[] scale,
- MatrixBlock result);
+ protected abstract void TSMMToUpperTriangleScaling(ADictionary right, IColIndex rowsLeft, IColIndex colsRight,
+ int[] scale, MatrixBlock result);
/**
* Matrix multiplication but allocate output in upper triangle and twice if on diagonal, note this is right
@@ -880,8 +882,8 @@ public abstract class ADictionary implements Serializable {
* @param scale Scale factor
* @param result The output matrix block
*/
- protected abstract void TSMMToUpperTriangleDenseScaling(double[] left, IColIndex rowsLeft, IColIndex colsRight, int[] scale,
- MatrixBlock result);
+ protected abstract void TSMMToUpperTriangleDenseScaling(double[] left, IColIndex rowsLeft, IColIndex colsRight,
+ int[] scale, MatrixBlock result);
/**
* Matrix multiplication but allocate output in upper triangle and twice if on diagonal, note this is right
@@ -895,6 +897,15 @@ public abstract class ADictionary implements Serializable {
protected abstract void TSMMToUpperTriangleSparseScaling(SparseBlock left, IColIndex rowsLeft, IColIndex colsRight,
int[] scale, MatrixBlock result);
+ /**
+ * Cbind this dictionary with that dictionary
+ *
+ * @param that the right hand side dictionary to cbind
+ * @param nCol the right hand side number of columns
+ * @return The combined dictionary
+ */
+ public abstract ADictionary cbind(ADictionary that, int nCol);
+
protected static String doubleToString(double v) {
if(v == (long) v)
return Long.toString(((long) v));
@@ -905,7 +916,7 @@ public abstract class ADictionary implements Serializable {
protected static void correctNan(double[] res, IColIndex colIndexes) {
// since there is no nan values every in a dictionary, we exploit that
// nan oly occur if we multiplied infinity with 0.
- for(int j = 0; j < colIndexes.size(); j++){
+ for(int j = 0; j < colIndexes.size(); j++) {
final int cix = colIndexes.get(j);
res[cix] = Double.isNaN(res[cix]) ? 0 : res[cix];
}
@@ -918,5 +929,7 @@ public abstract class ADictionary implements Serializable {
return false;
}
+ public abstract ADictionary reorder(int[] reorder);
+
public abstract boolean equals(ADictionary o);
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java
index ab5ae5ca00..632db578e6 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java
@@ -1098,4 +1098,26 @@ public class Dictionary extends ADictionary {
}
return false;
}
+
+ @Override
+ public ADictionary cbind(ADictionary that, int nCol){
+ int nRowThat = that.getNumberOfValues(nCol);
+ int nColThis = _values.length / nRowThat;
+ MatrixBlockDictionary mbd = getMBDict(nColThis);
+ return mbd.cbind(that, nCol);
+ }
+
+ @Override
+ public ADictionary reorder(int[] reorder){
+ double[] retV = new double[_values.length];
+ Dictionary ret = new Dictionary(retV);
+ int nRows = _values.length / reorder.length;
+
+ for(int r = 0; r < nRows; r++){
+ int off = r * reorder.length;
+ for(int c = 0; c < reorder.length; c++)
+ retV[off + c] = _values[off + reorder[c]];
+ }
+ return ret;
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DictionaryFactory.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DictionaryFactory.java
index bb42122f5e..da5c317874 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DictionaryFactory.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/DictionaryFactory.java
@@ -30,6 +30,12 @@ import org.apache.sysds.runtime.compress.DMLCompressionException;
import org.apache.sysds.runtime.compress.bitmap.ABitmap;
import org.apache.sysds.runtime.compress.bitmap.Bitmap;
import org.apache.sysds.runtime.compress.bitmap.MultiColBitmap;
+import org.apache.sysds.runtime.compress.colgroup.AColGroup.CompressionType;
+import org.apache.sysds.runtime.compress.colgroup.AColGroupCompressed;
+import org.apache.sysds.runtime.compress.colgroup.ColGroupEmpty;
+import org.apache.sysds.runtime.compress.colgroup.IContainADictionary;
+import org.apache.sysds.runtime.compress.colgroup.IContainDefaultTuple;
+import org.apache.sysds.runtime.compress.lib.CLALibCombineGroups;
import org.apache.sysds.runtime.compress.utils.ACount.DArrCounts;
import org.apache.sysds.runtime.compress.utils.DblArrayCountHashMap;
import org.apache.sysds.runtime.compress.utils.DoubleCountHashMap;
@@ -230,4 +236,242 @@ public interface DictionaryFactory {
final double[] resValues = map.getDictionary();
return Dictionary.create(resValues);
}
+
+ public static ADictionary combineDictionaries(AColGroupCompressed a, AColGroupCompressed b) {
+ if(a instanceof ColGroupEmpty && b instanceof ColGroupEmpty)
+ return null; // null return is handled elsewhere.
+
+ CompressionType ac = a.getCompType();
+ CompressionType bc = b.getCompType();
+
+ boolean ae = a instanceof IContainADictionary;
+ boolean be = b instanceof IContainADictionary;
+
+ if(ae && be) {
+
+ ADictionary ad = ((IContainADictionary) a).getDictionary();
+ ADictionary bd = ((IContainADictionary) b).getDictionary();
+ if(ac.isConst()) {
+ if(bc.isConst()) {
+ return new Dictionary(CLALibCombineGroups.constructDefaultTuple(a, b));
+ }
+ else if(bc.isDense()) {
+ final double[] at = ((IContainDefaultTuple) a).getDefaultTuple();
+ return combineConstSparseSparseRet(at, bd, b.getNumCols());
+ }
+ }
+ else if(ac.isDense()) {
+ if(bc.isConst()) {
+ final double[] bt = ((IContainDefaultTuple) b).getDefaultTuple();
+ return combineSparseConstSparseRet(ad, a.getNumCols(), bt);
+ }
+ else if(bc.isDense())
+ return combineFullDictionaries(ad, a.getNumCols(), bd, b.getNumCols());
+ else if(bc.isSDC()) {
+ double[] tuple = ((IContainDefaultTuple) b).getDefaultTuple();
+ return combineSDCRight(ad, a.getNumCols(), bd, tuple);
+ }
+ }
+ else if(ac.isSDC()) {
+ if(bc.isSDC()) {
+ final double[] at = ((IContainDefaultTuple) a).getDefaultTuple();
+ final double[] bt = ((IContainDefaultTuple) b).getDefaultTuple();
+ return combineSDC(ad, at, bd, bt);
+ }
+ }
+ }
+ throw new NotImplementedException("Not supporting combining dense: " + a + " " + b);
+ }
+
+ public static ADictionary combineDictionariesSparse(AColGroupCompressed a, AColGroupCompressed b) {
+ CompressionType ac = a.getCompType();
+ CompressionType bc = b.getCompType();
+
+ if(ac.isSDC()) {
+ ADictionary ad = ((IContainADictionary) a).getDictionary();
+ if(bc.isConst()) {
+ double[] bt = ((IContainDefaultTuple) b).getDefaultTuple();
+ return combineSparseConstSparseRet(ad, a.getNumCols(), bt);
+ }
+ else if(bc.isSDC()) {
+ ADictionary bd = ((IContainADictionary) b).getDictionary();
+ if(a.sameIndexStructure(b)) {
+ return ad.cbind(bd, b.getNumCols());
+ }
+
+ // real combine extract default and combine like dense but with default before.
+
+ }
+ }
+ else if(ac.isConst()) {
+ double[] at = ((IContainDefaultTuple) a).getDefaultTuple();
+ if(bc.isSDC()) {
+ ADictionary bd = ((IContainADictionary) b).getDictionary();
+ return combineConstSparseSparseRet(at, bd, b.getNumCols());
+ }
+ }
+
+ throw new NotImplementedException("Not supporting combining dense: " + a + " " + b);
+ }
+
+ /**
+ * Combine the dictionaries as if the dictionaries contain the full spectrum of the data contained.
+ *
+ * @param a Left side dictionary
+ * @param nca Number of columns left dictionary
+ * @param b Right side dictionary
+ * @param ncb Number of columns right dictionary
+ * @return A combined dictionary
+ */
+ public static ADictionary combineFullDictionaries(ADictionary a, int nca, ADictionary b, int ncb) {
+ final int ra = a.getNumberOfValues(nca);
+ final int rb = b.getNumberOfValues(ncb);
+
+ MatrixBlock ma = a.getMBDict(nca).getMatrixBlock();
+ MatrixBlock mb = b.getMBDict(ncb).getMatrixBlock();
+
+ if(ra == 1 && rb == 1)
+ return new MatrixBlockDictionary(ma.append(mb));
+
+ MatrixBlock out = new MatrixBlock(ra * rb, nca + ncb, false);
+
+ out.allocateBlock();
+
+ for(int r = 0; r < out.getNumRows(); r++) {
+ int ia = r % ra;
+ int ib = r / ra;
+ for(int c = 0; c < nca; c++)
+ out.quickSetValue(r, c, ma.quickGetValue(ia, c));
+
+ for(int c = 0; c < ncb; c++)
+ out.quickSetValue(r, c + nca, mb.quickGetValue(ib, c));
+
+ }
+ return new MatrixBlockDictionary(out);
+ }
+
+ public static ADictionary combineSDCRight(ADictionary a, int nca, ADictionary b, double[] tub) {
+ final int ncb = tub.length;
+ final int ra = a.getNumberOfValues(nca);
+ final int rb = b.getNumberOfValues(ncb);
+
+ MatrixBlock ma = a.getMBDict(nca).getMatrixBlock();
+ MatrixBlock mb = b.getMBDict(ncb).getMatrixBlock();
+
+ MatrixBlock out = new MatrixBlock(ra * (rb + 1), nca + ncb, false);
+
+ out.allocateBlock();
+
+ for(int r = 0; r < ra; r++) {
+
+ for(int c = 0; c < nca; c++)
+ out.quickSetValue(r, c, ma.quickGetValue(r, c));
+ for(int c = 0; c < ncb; c++)
+ out.quickSetValue(0, c + nca, tub[c]);
+ }
+
+ for(int r = ra; r < out.getNumRows(); r++) {
+ int ia = r % ra;
+ int ib = r / ra - 1;
+ for(int c = 0; c < nca; c++) // all good.
+ out.quickSetValue(r, c, ma.quickGetValue(ia, c));
+
+ for(int c = 0; c < ncb; c++)
+ out.quickSetValue(r, c + nca, mb.quickGetValue(ib, c));
+
+ }
+ return new MatrixBlockDictionary(out);
+ }
+
+ public static ADictionary combineSDC(ADictionary a, double[] tua, ADictionary b, double[] tub) {
+ final int nca = tua.length;
+ final int ncb = tub.length;
+ final int ra = a.getNumberOfValues(nca);
+ final int rb = b.getNumberOfValues(ncb);
+
+ MatrixBlock ma = a.getMBDict(nca).getMatrixBlock();
+ MatrixBlock mb = b.getMBDict(ncb).getMatrixBlock();
+
+ MatrixBlock out = new MatrixBlock((ra + 1) * (rb + 1), nca + ncb, false);
+
+ out.allocateBlock();
+
+ // 0 row both default tuples
+
+ for(int c = 0; c < nca; c++)
+ out.quickSetValue(0, c, tua[c]);
+
+ for(int c = 0; c < ncb; c++)
+ out.quickSetValue(0, c + nca, tub[c]);
+
+ // default case for b and all cases for a.
+ for(int r = 1; r < ra + 1; r++) {
+ for(int c = 0; c < nca; c++)
+ out.quickSetValue(r, c, ma.quickGetValue(r - 1, c));
+ for(int c = 0; c < ncb; c++)
+ out.quickSetValue(r, c + nca, tub[c]);
+ }
+
+ for(int r = ra + 1; r < out.getNumRows(); r++) {
+ int ia = r % (ra + 1) - 1;
+ int ib = r / (ra + 1) - 1;
+
+ if(ia == -1)
+ for(int c = 0; c < nca; c++)
+ out.quickSetValue(r, c, tua[c]);
+ else
+ for(int c = 0; c < nca; c++)
+ out.quickSetValue(r, c, ma.quickGetValue(ia, c));
+
+ for(int c = 0; c < ncb; c++) // all good here.
+ out.quickSetValue(r, c + nca, mb.quickGetValue(ib, c));
+
+ }
+
+ return new MatrixBlockDictionary(out);
+ }
+
+ public static ADictionary combineSparseConstSparseRet(ADictionary a, int nca, double[] tub) {
+ final int ncb = tub.length;
+ final int ra = a.getNumberOfValues(nca);
+
+ MatrixBlock ma = a.getMBDict(nca).getMatrixBlock();
+
+ MatrixBlock out = new MatrixBlock(ra, nca + ncb, false);
+
+ out.allocateBlock();
+
+ // default case for b and all cases for a.
+ for(int r = 0; r < ra; r++) {
+ for(int c = 0; c < nca; c++)
+ out.quickSetValue(r, c, ma.quickGetValue(r, c));
+ for(int c = 0; c < ncb; c++)
+ out.quickSetValue(r, c + nca, tub[c]);
+ }
+
+ return new MatrixBlockDictionary(out);
+
+ }
+
+ public static ADictionary combineConstSparseSparseRet(double[] tua, ADictionary b, int ncb) {
+ final int nca = tua.length;
+ final int rb = b.getNumberOfValues(ncb);
+
+ MatrixBlock mb = b.getMBDict(ncb).getMatrixBlock();
+
+ MatrixBlock out = new MatrixBlock(rb, nca + ncb, false);
+
+ out.allocateBlock();
+
+ // default case for b and all cases for a.
+ for(int r = 0; r < rb; r++) {
+ for(int c = 0; c < nca; c++)
+ out.quickSetValue(r, c, tua[c]);
+ for(int c = 0; c < ncb; c++)
+ out.quickSetValue(r, c + nca, mb.quickGetValue(r, c));
+ }
+
+ return new MatrixBlockDictionary(out);
+
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java
index 996d1f1b4d..b46db73c1c 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java
@@ -612,4 +612,14 @@ public class IdentityDictionary extends ADictionary {
return false;
}
+ @Override
+ public ADictionary cbind(ADictionary that, int nCol) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public ADictionary reorder(int[] reorder) {
+ return getMBDict().reorder(reorder);
+ }
+
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java
index a1b11eaf29..fb851905e8 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java
@@ -318,10 +318,10 @@ public class MatrixBlockDictionary extends ADictionary {
public void aggregateCols(double[] c, Builtin fn, IColIndex colIndexes) {
if(_data.isInSparseFormat()) {
MatrixBlock t = LibMatrixReorg.transpose(_data);
- if(!t.isInSparseFormat()){
+ if(!t.isInSparseFormat()) {
LOG.warn("Transpose for aggregating of columns");
t.denseToSparse(true);
- }
+ }
SparseBlock sbt = t.getSparseBlock();
@@ -353,7 +353,8 @@ public class MatrixBlockDictionary extends ADictionary {
}
@Override
- public void aggregateColsWithReference(double[] c, Builtin fn, IColIndex colIndexes, double[] reference, boolean def) {
+ public void aggregateColsWithReference(double[] c, Builtin fn, IColIndex colIndexes, double[] reference,
+ boolean def) {
final int nCol = _data.getNumColumns();
final int nRow = _data.getNumRows();
@@ -2063,7 +2064,8 @@ public class MatrixBlockDictionary extends ADictionary {
}
@Override
- protected void TSMMToUpperTriangleSparse(SparseBlock left, IColIndex rowsLeft, IColIndex colsRight, MatrixBlock result) {
+ protected void TSMMToUpperTriangleSparse(SparseBlock left, IColIndex rowsLeft, IColIndex colsRight,
+ MatrixBlock result) {
if(_data.isInSparseFormat())
DictLibMatrixMult.MMToUpperTriangleSparseSparse(left, _data.getSparseBlock(), rowsLeft, colsRight, result);
else
@@ -2091,8 +2093,8 @@ public class MatrixBlockDictionary extends ADictionary {
}
@Override
- protected void TSMMToUpperTriangleSparseScaling(SparseBlock left, IColIndex rowsLeft, IColIndex colsRight, int[] scale,
- MatrixBlock result) {
+ protected void TSMMToUpperTriangleSparseScaling(SparseBlock left, IColIndex rowsLeft, IColIndex colsRight,
+ int[] scale, MatrixBlock result) {
if(_data.isInSparseFormat())
DictLibMatrixMult.TSMMToUpperTriangleSparseSparseScaling(left, _data.getSparseBlock(), rowsLeft, colsRight,
scale, result);
@@ -2115,4 +2117,24 @@ public class MatrixBlockDictionary extends ADictionary {
return false;
}
+ @Override
+ public ADictionary cbind(ADictionary that, int nCol) {
+ return cbind(that.getMBDict(nCol).getMatrixBlock());
+ }
+
+ private ADictionary cbind(MatrixBlock that) {
+ return new MatrixBlockDictionary(_data.append(that));
+ }
+
+ @Override
+ public ADictionary reorder(int[] reorder) {
+ MatrixBlock ret = new MatrixBlock(_data.getNumRows(), _data.getNumColumns(), _data.getNonZeros());
+
+ // TODO add sparse exploitation.
+ for(int r = 0; r < _data.getNumRows(); r++)
+ for(int c = 0; c < _data.getNumColumns(); c++)
+ ret.quickSetValue(r, c, _data.quickGetValue(r, reorder[c]));
+
+ return create(ret, false);
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/QDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/QDictionary.java
index a7bc27af31..ad91e1403e 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/QDictionary.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/QDictionary.java
@@ -94,7 +94,7 @@ public class QDictionary extends ADictionary {
public static long getInMemorySize(int valuesCount) {
// object + values array + double
- return 16 + (long)MemoryEstimates.byteArrayCost(valuesCount) + 8;
+ return 16 + (long) MemoryEstimates.byteArrayCost(valuesCount) + 8;
}
@Override
@@ -423,7 +423,8 @@ public class QDictionary extends ADictionary {
}
@Override
- public void aggregateColsWithReference(double[] c, Builtin fn, IColIndex colIndexes, double[] reference, boolean def) {
+ public void aggregateColsWithReference(double[] c, Builtin fn, IColIndex colIndexes, double[] reference,
+ boolean def) {
throw new NotImplementedException();
}
@@ -575,7 +576,8 @@ public class QDictionary extends ADictionary {
}
@Override
- protected void TSMMToUpperTriangleSparse(SparseBlock left, IColIndex rowsLeft, IColIndex colsRight, MatrixBlock result) {
+ protected void TSMMToUpperTriangleSparse(SparseBlock left, IColIndex rowsLeft, IColIndex colsRight,
+ MatrixBlock result) {
throw new NotImplementedException();
}
@@ -592,8 +594,8 @@ public class QDictionary extends ADictionary {
}
@Override
- protected void TSMMToUpperTriangleSparseScaling(SparseBlock left, IColIndex rowsLeft, IColIndex colsRight, int[] scale,
- MatrixBlock result) {
+ protected void TSMMToUpperTriangleSparseScaling(SparseBlock left, IColIndex rowsLeft, IColIndex colsRight,
+ int[] scale, MatrixBlock result) {
throw new NotImplementedException();
}
@@ -601,4 +603,14 @@ public class QDictionary extends ADictionary {
public boolean equals(ADictionary o) {
throw new NotImplementedException();
}
+
+ @Override
+ public ADictionary cbind(ADictionary that, int nCol) {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public ADictionary reorder(int[] reorder) {
+ throw new NotImplementedException();
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/ArrayIndex.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/ArrayIndex.java
index d3f3608be0..c2acadcb8e 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/ArrayIndex.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/ArrayIndex.java
@@ -23,6 +23,7 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
+import java.util.stream.IntStream;
import org.apache.sysds.utils.MemoryEstimates;
@@ -177,6 +178,31 @@ public class ArrayIndex extends AColIndex {
return sb.toString();
}
+ @Override
+ public int[] getReorderingIndex() {
+ int[] sortedIndices = IntStream.range(0, cols.length).boxed()//
+ .sorted((i, j) -> Integer.valueOf(cols[i]).compareTo(cols[j]))//
+ .mapToInt(ele -> ele).toArray();
+ return sortedIndices;
+ }
+
+ @Override
+ public boolean isSorted() {
+ for(int i = 1; i < cols.length; i++)
+ if(cols[i - 1] > cols[i])
+ return false;
+
+ return true;
+ }
+
+ @Override
+ public IColIndex sort() {
+ int[] ret = new int[cols.length];
+ System.arraycopy(cols, 0, ret, 0, cols.length);
+ Arrays.sort(ret);
+ return ColIndexFactory.create(ret);
+ }
+
protected class ArrayIterator implements IIterate {
int id = 0;
@@ -189,5 +215,15 @@ public class ArrayIndex extends AColIndex {
public boolean hasNext() {
return id < cols.length;
}
+
+ @Override
+ public int v() {
+ return cols[id];
+ }
+
+ @Override
+ public int i() {
+ return id;
+ }
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/ColIndexFactory.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/ColIndexFactory.java
index d44eccaefb..4ecde22ff4 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/ColIndexFactory.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/ColIndexFactory.java
@@ -24,6 +24,8 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.compress.DMLCompressionException;
import org.apache.sysds.runtime.compress.colgroup.AColGroup;
@@ -31,6 +33,7 @@ import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex.ColIndexType
import org.apache.sysds.runtime.compress.utils.IntArrayList;
public interface ColIndexFactory {
+ public static final Log LOG = LogFactory.getLog(ColIndexFactory.class.getName());
public static IColIndex read(DataInput in) throws IOException {
final ColIndexType t = ColIndexType.values()[in.readByte()];
@@ -48,6 +51,10 @@ public interface ColIndexFactory {
}
}
+ public static IColIndex createI(int... indexes) {
+ return create(indexes);
+ }
+
public static IColIndex create(int[] indexes) {
if(indexes.length <= 0)
throw new DMLRuntimeException("Invalid length to create index from : " + indexes.length);
@@ -97,10 +104,10 @@ public interface ColIndexFactory {
return new RangeIndex(nCol);
}
- public static long estimateMemoryCost(int nCol, boolean contiguous){
+ public static long estimateMemoryCost(int nCol, boolean contiguous) {
if(nCol == 1)
return SingleIndex.estimateInMemorySizeStatic();
- else if (nCol == 2)
+ else if(nCol == 2)
return TwoIndex.estimateInMemorySizeStatic();
else if(contiguous)
return RangeIndex.estimateInMemorySizeStatic();
@@ -126,4 +133,51 @@ public interface ColIndexFactory {
return create(resCols);
}
+ public static IColIndex combine(AColGroup a, AColGroup b) {
+ return combine(a.getColIndices(), b.getColIndices());
+ }
+
+ public static IColIndex combine(IColIndex a, IColIndex b) {
+ final int numCols = a.size() + b.size();
+ final int[] resCols = new int[numCols];
+ int index = 0;
+ final IIterate ita = a.iterator();
+ while(ita.hasNext())
+ resCols[index++] = ita.next();
+ final IIterate itb = b.iterator();
+ while(itb.hasNext())
+ resCols[index++] = itb.next();
+ Arrays.sort(resCols);
+ return create(resCols);
+ }
+
+ /**
+ * Provide a mapping from a to the combined columns shifted over to column positions in the combined.
+ *
+ * It is assumed that the caller always input an a that is contained in comb. it is not verified in the call that it
+ * is correct.
+ *
+ * @param comb The combined indexes
+ * @param a The indexes to look up
+ * @return A column index mapping.
+ */
+ public static IColIndex getColumnMapping(IColIndex comb, IColIndex a) {
+
+ // naive scan entire a, there are options that make this faster depending on what comb look like
+ // but this is most commonly not relevant.
+ final int numCols = a.size();
+ final int[] ret = new int[numCols];
+ final IIterate itc = comb.iterator();
+ final IIterate ita = a.iterator();
+ int index = 0;
+ while(ita.hasNext()) {
+ while(itc.v() < ita.v())
+ itc.next();
+ ret[index++] = itc.i();
+ ita.next();
+ }
+
+ return create(ret);
+ }
+
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/IColIndex.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/IColIndex.java
index 246241ab56..ff0acabf3f 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/IColIndex.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/IColIndex.java
@@ -142,11 +142,34 @@ public interface IColIndex {
*
* 1,3,4 is not.
*
- *
* @return If the Columns are contiguous.
*/
public boolean isContiguous();
+ /**
+ * If the columns are not in sorted incrementing order this method can be called to get the sorting index for this
+ * set of column indexes.
+ *
+ * The returned list should be the mapping of elements for each column to where it should be after sorting.
+ *
+ * @return A Reordered index.
+ */
+ public int[] getReorderingIndex();
+
+ /**
+ * Get if the Index is sorted.
+ *
+ * @return If the index is sorted
+ */
+ public boolean isSorted();
+
+ /**
+ * Sort the index and return a new object if there are modifications otherwise return this.
+ *
+ * @return The sorted instance of this column index.
+ */
+ public IColIndex sort();
+
/** A Class for slice results containing indexes for the slicing of dictionaries, and the resulting column index */
public static class SliceResult {
/** Start index to slice inside the dictionary */
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/IIterate.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/IIterate.java
index fae615c9ec..c63833eae8 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/IIterate.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/IIterate.java
@@ -38,4 +38,18 @@ public interface IIterate {
* @return the next index.
*/
public boolean hasNext();
+
+ /**
+ * Get current value
+ *
+ * @return the value pointing at.
+ */
+ public int v();
+
+ /**
+ * Get current index
+ *
+ * @return The index currently pointed at
+ */
+ public int i();
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/RangeIndex.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/RangeIndex.java
index e6cfef1485..717c7b8d8f 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/RangeIndex.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/RangeIndex.java
@@ -23,6 +23,7 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import org.apache.sysds.runtime.compress.DMLCompressionException;
import org.apache.sysds.runtime.compress.utils.IntArrayList;
public class RangeIndex extends AColIndex {
@@ -159,7 +160,7 @@ public class RangeIndex extends AColIndex {
}
@Override
- public boolean isContiguous(){
+ public boolean isContiguous() {
return true;
}
@@ -176,17 +177,40 @@ public class RangeIndex extends AColIndex {
}
protected static boolean isValidRange(int[] indexes) {
- int len = indexes.length;
- int first = indexes[0];
- int last = indexes[indexes.length - 1];
- return last - first + 1 == len;
+ return isValidRange(indexes, indexes.length);
}
protected static boolean isValidRange(IntArrayList indexes) {
- int len = indexes.size();
- int first = indexes.get(0);
- int last = indexes.get(indexes.size() - 1);
- return last - first + 1 == len;
+ return isValidRange(indexes.extractValues(), indexes.size());
+ }
+
+ private static boolean isValidRange(final int[] indexes, final int length) {
+ int len = length;
+ int first = indexes[0];
+ int last = indexes[length - 1];
+ if(last - first + 1 == len) {
+ for(int i = 1; i < length; i++)
+ if(indexes[i - 1] > indexes[i])
+ return false;
+ return true;
+ }
+ else
+ return false;
+ }
+
+ @Override
+ public int[] getReorderingIndex() {
+ throw new DMLCompressionException("not valid to get reordering Index for range");
+ }
+
+ @Override
+ public boolean isSorted() {
+ return true;
+ }
+
+ @Override
+ public IColIndex sort() {
+ throw new DMLCompressionException("range is always sorted");
}
protected class RangeIterator implements IIterate {
@@ -201,5 +225,15 @@ public class RangeIndex extends AColIndex {
public boolean hasNext() {
return cl < u;
}
+
+ @Override
+ public int v() {
+ return cl;
+ }
+
+ @Override
+ public int i() {
+ return cl - l;
+ }
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/SingleIndex.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/SingleIndex.java
index 224cec1180..b9bef4f9b6 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/SingleIndex.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/SingleIndex.java
@@ -22,6 +22,8 @@ package org.apache.sysds.runtime.compress.colgroup.indexes;
import java.io.DataOutput;
import java.io.IOException;
+import org.apache.sysds.runtime.compress.DMLCompressionException;
+
public class SingleIndex extends AColIndex {
private final int idx;
@@ -61,10 +63,10 @@ public class SingleIndex extends AColIndex {
@Override
public long estimateInMemorySize() {
- return estimateInMemorySizeStatic();
+ return estimateInMemorySizeStatic();
}
- public static long estimateInMemorySizeStatic(){
+ public static long estimateInMemorySizeStatic() {
return 16 + 4 + 4; // object, int, and padding
}
@@ -104,10 +106,25 @@ public class SingleIndex extends AColIndex {
}
@Override
- public boolean isContiguous(){
+ public boolean isContiguous() {
return true;
}
+ @Override
+ public int[] getReorderingIndex() {
+ throw new DMLCompressionException("not valid to get reordering Index for range");
+ }
+
+ @Override
+ public boolean isSorted() {
+ return true;
+ }
+
+ @Override
+ public IColIndex sort() {
+ throw new DMLCompressionException("range is always sorted");
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
@@ -131,6 +148,16 @@ public class SingleIndex extends AColIndex {
public boolean hasNext() {
return !taken;
}
+
+ @Override
+ public int v() {
+ return idx;
+ }
+
+ @Override
+ public int i() {
+ return 0;
+ }
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/TwoIndex.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/TwoIndex.java
index 8cbdd63883..fae8dc7464 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/TwoIndex.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/TwoIndex.java
@@ -133,6 +133,27 @@ public class TwoIndex extends AColIndex {
return id1 + 1 == id2;
}
+ @Override
+ public int[] getReorderingIndex() {
+ if(id2 < id1)
+ return new int[] {1, 0};
+ else
+ return new int[] {0, 1};
+ }
+
+ @Override
+ public boolean isSorted() {
+ return id2 > id1;
+ }
+
+ @Override
+ public IColIndex sort() {
+ if(id2 < id1)
+ return new TwoIndex(id2, id1);
+ else
+ return this;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
@@ -160,6 +181,16 @@ public class TwoIndex extends AColIndex {
public boolean hasNext() {
return id < 2;
}
+
+ @Override
+ public int v() {
+ return id == 0 ? id1 : id2;
+ }
+
+ @Override
+ public int i() {
+ return id;
+ }
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java
index 7076f971de..21be58dae5 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java
@@ -27,7 +27,7 @@ import java.util.BitSet;
import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.sysds.runtime.compress.colgroup.AMapToDataGroup;
+import org.apache.sysds.runtime.compress.colgroup.IMapToDataGroup;
import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
import org.apache.sysds.runtime.compress.colgroup.dictionary.Dictionary;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
@@ -818,7 +818,7 @@ public abstract class AMapToData implements Serializable {
public abstract AMapToData append(AMapToData t);
- public abstract AMapToData appendN(AMapToDataGroup[] d);
+ public abstract AMapToData appendN(IMapToDataGroup[] d);
@Override
public String toString() {
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToBit.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToBit.java
index 01ab2a4bb1..77084ff0b7 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToBit.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToBit.java
@@ -26,7 +26,7 @@ import java.util.BitSet;
import org.apache.commons.lang.NotImplementedException;
import org.apache.sysds.runtime.DMLRuntimeException;
-import org.apache.sysds.runtime.compress.colgroup.AMapToDataGroup;
+import org.apache.sysds.runtime.compress.colgroup.IMapToDataGroup;
import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
import org.apache.sysds.utils.MemoryEstimates;
@@ -352,9 +352,9 @@ public class MapToBit extends AMapToData {
}
@Override
- public AMapToData appendN(AMapToDataGroup[] d) {
+ public AMapToData appendN(IMapToDataGroup[] d) {
int p = 0; // pointer
- for(AMapToDataGroup gd : d)
+ for(IMapToDataGroup gd : d)
p += gd.getMapToData().size();
final long[] ret = new long[(p - 1) / 64 + 1];
long[] or = _data.toLongArray();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToByte.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToByte.java
index 81eb974490..184f7746d7 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToByte.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToByte.java
@@ -26,7 +26,7 @@ import java.util.Arrays;
import java.util.BitSet;
import org.apache.commons.lang.NotImplementedException;
-import org.apache.sysds.runtime.compress.colgroup.AMapToDataGroup;
+import org.apache.sysds.runtime.compress.colgroup.IMapToDataGroup;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
import org.apache.sysds.utils.MemoryEstimates;
@@ -236,9 +236,9 @@ public class MapToByte extends AMapToData {
}
@Override
- public AMapToData appendN(AMapToDataGroup[] d) {
+ public AMapToData appendN(IMapToDataGroup[] d) {
int p = 0; // pointer
- for(AMapToDataGroup gd : d)
+ for(IMapToDataGroup gd : d)
p += gd.getMapToData().size();
final byte[] ret = Arrays.copyOf(_data, p);
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToChar.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToChar.java
index f24c8a72e8..6c1fc6d030 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToChar.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToChar.java
@@ -26,7 +26,7 @@ import java.util.Arrays;
import java.util.BitSet;
import org.apache.commons.lang.NotImplementedException;
-import org.apache.sysds.runtime.compress.colgroup.AMapToDataGroup;
+import org.apache.sysds.runtime.compress.colgroup.IMapToDataGroup;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
import org.apache.sysds.utils.MemoryEstimates;
@@ -254,9 +254,9 @@ public class MapToChar extends AMapToData {
}
@Override
- public AMapToData appendN(AMapToDataGroup[] d) {
+ public AMapToData appendN(IMapToDataGroup[] d) {
int p = 0; // pointer
- for(AMapToDataGroup gd : d)
+ for(IMapToDataGroup gd : d)
p += gd.getMapToData().size();
final char[] ret = Arrays.copyOf(_data, p);
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToCharPByte.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToCharPByte.java
index f1483a2bcd..f78a21149f 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToCharPByte.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToCharPByte.java
@@ -26,7 +26,7 @@ import java.util.Arrays;
import java.util.BitSet;
import org.apache.commons.lang.NotImplementedException;
-import org.apache.sysds.runtime.compress.colgroup.AMapToDataGroup;
+import org.apache.sysds.runtime.compress.colgroup.IMapToDataGroup;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
import org.apache.sysds.utils.MemoryEstimates;
@@ -244,7 +244,7 @@ public class MapToCharPByte extends AMapToData {
}
@Override
- public AMapToData appendN(AMapToDataGroup[] d){
+ public AMapToData appendN(IMapToDataGroup[] d){
throw new NotImplementedException();
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToInt.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToInt.java
index 2324b2bac0..88e46af3fa 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToInt.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToInt.java
@@ -26,7 +26,7 @@ import java.util.Arrays;
import java.util.BitSet;
import org.apache.commons.lang.NotImplementedException;
-import org.apache.sysds.runtime.compress.colgroup.AMapToDataGroup;
+import org.apache.sysds.runtime.compress.colgroup.IMapToDataGroup;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
import org.apache.sysds.utils.MemoryEstimates;
@@ -255,9 +255,9 @@ public class MapToInt extends AMapToData {
}
@Override
- public AMapToData appendN(AMapToDataGroup[] d) {
+ public AMapToData appendN(IMapToDataGroup[] d) {
int p = 0; // pointer
- for(AMapToDataGroup gd : d)
+ for(IMapToDataGroup gd : d)
p += gd.getMapToData().size();
final int[] ret = Arrays.copyOf(_data, p);
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToZero.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToZero.java
index 8e2115e5b8..cb53197d38 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToZero.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToZero.java
@@ -25,7 +25,7 @@ import java.io.IOException;
import java.util.BitSet;
import org.apache.commons.lang.NotImplementedException;
-import org.apache.sysds.runtime.compress.colgroup.AMapToDataGroup;
+import org.apache.sysds.runtime.compress.colgroup.IMapToDataGroup;
import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
@@ -161,9 +161,9 @@ public class MapToZero extends AMapToData {
}
@Override
- public AMapToData appendN(AMapToDataGroup[] d) {
+ public AMapToData appendN(IMapToDataGroup[] d) {
int p = 0; // pointer
- for(AMapToDataGroup gd : d)
+ for(IMapToDataGroup gd : d)
p += gd.getMapToData().size();
return new MapToZero(p);
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/ConstEncoding.java b/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/ConstEncoding.java
index 61a67c6b5f..b3839c9e95 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/ConstEncoding.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/ConstEncoding.java
@@ -36,6 +36,11 @@ public class ConstEncoding implements IEncode {
return e;
}
+ @Override
+ public IEncode combineNoResize(IEncode e){
+ return e;
+ }
+
@Override
public int getUnique() {
return 1;
diff --git a/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/DenseEncoding.java b/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/DenseEncoding.java
index 2a14c849ee..5504e1c22d 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/DenseEncoding.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/DenseEncoding.java
@@ -47,18 +47,39 @@ public class DenseEncoding implements IEncode {
return combineDense((DenseEncoding) e);
}
+ @Override
+ public IEncode combineNoResize(IEncode e) {
+ if(e instanceof EmptyEncoding || e instanceof ConstEncoding)
+ return this;
+ else if(e instanceof SparseEncoding)
+ return combineSparseNoResize((SparseEncoding) e);
+ else
+ return combineDenseNoResize((DenseEncoding) e);
+ }
+
protected DenseEncoding combineSparse(SparseEncoding e) {
final int maxUnique = e.getUnique() * getUnique();
final int size = map.size();
final int nVl = getUnique();
+ // temp result
+ final AMapToData ret = assignSparse(e);
+ // Iteration 2 reassign indexes.
+ if(maxUnique + nVl > size)
+ return combineSparseHashMap(ret);
+ else
+ return combineSparseMapToData(ret, maxUnique, nVl);
+ }
+
+ private AMapToData assignSparse(SparseEncoding e) {
+ final int maxUnique = e.getUnique() * getUnique();
+ final int size = map.size();
+ final int nVl = getUnique();
// temp result
final AMapToData ret = MapToFactory.create(size, maxUnique);
// Iteration 1 copy dense data.
ret.copy(map);
-
- // Iterate through indexes that are in the sparse encoding
final AIterator itr = e.off.getIterator();
final int fr = e.off.getOffsetToLast();
@@ -68,12 +89,7 @@ public class DenseEncoding implements IEncode {
ir = itr.next();
}
ret.set(fr, ret.getIndex(fr) + ((e.map.getIndex(itr.getDataIndex()) + 1) * nVl));
-
- // Iteration 2 reassign indexes.
- if(maxUnique + nVl > size)
- return combineSparseHashMap(ret);
- else
- return combineSparseMapToData(ret, maxUnique, nVl);
+ return ret;
}
private final DenseEncoding combineSparseHashMap(final AMapToData ret) {
@@ -126,6 +142,31 @@ public class DenseEncoding implements IEncode {
return combineDenseWithMapToData(lm, rm, size, nVL, ret, maxUnique);
}
+ private DenseEncoding combineDenseNoResize(final DenseEncoding other) {
+ if(map == other.map)
+ return this; // same object
+
+ final AMapToData lm = map;
+ final AMapToData rm = other.map;
+
+ final int nVL = lm.getUnique();
+ final int nVR = rm.getUnique();
+ final int size = map.size();
+ final int maxUnique = nVL * nVR;
+
+ final AMapToData ret = MapToFactory.create(size, maxUnique);
+
+ for(int r = 0; r < size; r++)
+ ret.set(r, lm.getIndex(r) + rm.getIndex(r) * nVL);
+ // there can be less unique.
+
+ return new DenseEncoding(ret);
+ }
+
+ private DenseEncoding combineSparseNoResize(final SparseEncoding other) {
+ return new DenseEncoding(assignSparse(other));
+ }
+
protected final DenseEncoding combineDenseWithHashMap(final AMapToData lm, final AMapToData rm, final int size,
final int nVL, final AMapToData ret) {
final Map<Integer, Integer> m = new HashMap<>(size);
@@ -133,7 +174,6 @@ public class DenseEncoding implements IEncode {
for(int r = 0; r < size; r++)
addValHashMap(lm.getIndex(r) + rm.getIndex(r) * nVL, r, m, ret);
return new DenseEncoding(MapToFactory.resize(ret, m.size()));
-
}
protected final DenseEncoding combineDenseWithMapToData(final AMapToData lm, final AMapToData rm, final int size,
@@ -177,7 +217,7 @@ public class DenseEncoding implements IEncode {
for(int i = 0; i < counts.length; i++)
if(counts[i] > largestOffs)
largestOffs = counts[i];
-
+
if(cs.isRLEAllowed())
return new EstimationFactors(map.getUnique(), nRows, largestOffs, counts, 0, nRows, map.countRuns(), false,
false, matrixSparsity, tupleSparsity);
@@ -187,6 +227,10 @@ public class DenseEncoding implements IEncode {
}
+ public AMapToData getMap() {
+ return map;
+ }
+
@Override
public boolean isDense() {
return true;
diff --git a/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/EmptyEncoding.java b/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/EmptyEncoding.java
index 75eb4b9a09..6aa9f91ca6 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/EmptyEncoding.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/EmptyEncoding.java
@@ -34,6 +34,11 @@ public class EmptyEncoding implements IEncode {
return e;
}
+ @Override
+ public IEncode combineNoResize(IEncode e){
+ return e;
+ }
+
@Override
public int getUnique() {
return 1;
diff --git a/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/EncodingFactory.java b/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/EncodingFactory.java
index 23c5484745..05b21d2c11 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/EncodingFactory.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/EncodingFactory.java
@@ -24,6 +24,8 @@ import java.util.Arrays;
import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.sysds.runtime.compress.colgroup.ColGroupConst;
+import org.apache.sysds.runtime.compress.colgroup.ColGroupEmpty;
import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
import org.apache.sysds.runtime.compress.colgroup.mapping.AMapToData;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory;
@@ -115,6 +117,22 @@ public interface EncodingFactory {
return createFromDense(m, rowCol);
}
+ public static IEncode create(ColGroupConst c){
+ return new ConstEncoding(-1);
+ }
+
+ public static IEncode create(ColGroupEmpty c){
+ return new EmptyEncoding();
+ }
+
+ public static IEncode create(AMapToData d){
+ return new DenseEncoding(d);
+ }
+
+ public static IEncode create(AMapToData d, AOffset i, int nRow){
+ return new SparseEncoding(d, i, nRow);
+ }
+
private static IEncode createFromDenseTransposed(MatrixBlock m, int row) {
final DenseBlock db = m.getDenseBlock();
if(!db.isContiguous())
diff --git a/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/IEncode.java b/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/IEncode.java
index 26b2201428..dcff899217 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/IEncode.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/IEncode.java
@@ -41,6 +41,15 @@ public interface IEncode {
*/
public IEncode combine(IEncode e);
+ /**
+ * Combine two encodings without resizing the output. meaning the mapping of the indexes should be consistant with
+ * left hand side Dictionary indexes and right hand side indexes.
+ *
+ * @param e The other side to combine with
+ * @return The combined encoding
+ */
+ public IEncode combineNoResize(IEncode e);
+
/**
* Get the number of unique values in this encoding
*
diff --git a/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/SparseEncoding.java b/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/SparseEncoding.java
index 2b242e0ffb..ff3b3285af 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/SparseEncoding.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/estim/encoding/SparseEncoding.java
@@ -20,6 +20,7 @@
package org.apache.sysds.runtime.compress.estim.encoding;
import org.apache.sysds.runtime.compress.CompressionSettings;
+import org.apache.sysds.runtime.compress.DMLCompressionException;
import org.apache.sysds.runtime.compress.colgroup.mapping.AMapToData;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory;
import org.apache.sysds.runtime.compress.colgroup.offset.AIterator;
@@ -54,13 +55,27 @@ public class SparseEncoding implements IEncode {
SparseEncoding es = (SparseEncoding) e;
if(es.off == off && es.map == map)
return this;
- return combineSparse((SparseEncoding) e);
+ return combineSparse(es);
}
else
return e.combine(this);
}
+ @Override
+ public IEncode combineNoResize(IEncode e) {
+ if(e instanceof EmptyEncoding || e instanceof ConstEncoding)
+ return this;
+ else if(e instanceof SparseEncoding) {
+ SparseEncoding es = (SparseEncoding) e;
+ if(es.off == off && es.map == map)
+ return this;
+ return combineSparseNoResize(es);
+ }
+ else
+ throw new DMLCompressionException("Not allowed other to be dense");
+ }
+
protected IEncode combineSparse(SparseEncoding e) {
final int maxUnique = e.getUnique() * getUnique();
final int[] d = new int[maxUnique - 1];
@@ -99,6 +114,43 @@ public class SparseEncoding implements IEncode {
}
}
+ private IEncode combineSparseNoResize(SparseEncoding e) {
+ // for now just use the dense... and lets continue.
+ // TODO add sparse combine with sparse output.
+ return combineSparseNoResizeDense(e);
+ }
+
+ private IEncode combineSparseNoResizeDense(SparseEncoding e) {
+
+ final int fl = off.getOffsetToLast();
+ final int fr = e.off.getOffsetToLast();
+ final AIterator itl = off.getIterator();
+ final AIterator itr = e.off.getIterator();
+ final int nVl = getUnique();
+ final int nVr = e.getUnique();
+
+ final AMapToData retMap = MapToFactory.create(nRows, (nVl + 1) * (nVr + 1));
+ int il = itl.value();
+ // parse through one side set all values into the dense.
+ while(il < fl) {
+ retMap.set(il, map.getIndex(itl.getDataIndex()) + 1);
+ il = itl.next();
+ }
+ retMap.set(fl, map.getIndex(itl.getDataIndex()) + 1);
+
+ int ir = itr.value();
+ // parse through other side set all values with offset based on what already is there.
+ while(ir < fr) {
+ final int vl = retMap.getIndex(ir); // probably 0
+ final int vr = e.map.getIndex(itr.getDataIndex()) + 1;
+ retMap.set(ir, vl + vr * nVl);
+ ir = itr.next();
+ }
+ retMap.set(fr, retMap.getIndex(fr) + (e.map.getIndex(itr.getDataIndex()) + 1) * nVl);
+
+ return new DenseEncoding(retMap);
+ }
+
private static int combineSparse(AMapToData lMap, AMapToData rMap, AIterator itl, AIterator itr,
final IntArrayList retOff, final IntArrayList tmpVals, final int fl, final int fr, final int nVl, final int nVr,
final int[] d) {
@@ -327,6 +379,14 @@ public class SparseEncoding implements IEncode {
return off;
}
+ public AMapToData getMap() {
+ return map;
+ }
+
+ public int getNumRows() {
+ return nRows;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombineGroups.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombineGroups.java
index cc9d25c9ce..27ea3f56b9 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombineGroups.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombineGroups.java
@@ -20,9 +20,34 @@
package org.apache.sysds.runtime.compress.lib;
import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
+import org.apache.sysds.runtime.compress.colgroup.AColGroup;
+import org.apache.sysds.runtime.compress.colgroup.AColGroupCompressed;
+import org.apache.sysds.runtime.compress.colgroup.ColGroupConst;
+import org.apache.sysds.runtime.compress.colgroup.ColGroupDDC;
+import org.apache.sysds.runtime.compress.colgroup.ColGroupEmpty;
+import org.apache.sysds.runtime.compress.colgroup.ColGroupSDC;
+import org.apache.sysds.runtime.compress.colgroup.ColGroupUncompressed;
+import org.apache.sysds.runtime.compress.colgroup.IContainDefaultTuple;
+import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
+import org.apache.sysds.runtime.compress.colgroup.dictionary.DictionaryFactory;
+import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory;
+import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
+import org.apache.sysds.runtime.compress.estim.encoding.ConstEncoding;
+import org.apache.sysds.runtime.compress.estim.encoding.DenseEncoding;
+import org.apache.sysds.runtime.compress.estim.encoding.EmptyEncoding;
+import org.apache.sysds.runtime.compress.estim.encoding.IEncode;
+import org.apache.sysds.runtime.compress.estim.encoding.SparseEncoding;
+import org.apache.sysds.runtime.data.DenseBlock;
+import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+/**
+ * Library functions to combine column groups inside a compressed matrix.
+ */
public final class CLALibCombineGroups {
+ protected static final Log LOG = LogFactory.getLog(CLALibCombineGroups.class.getName());
private CLALibCombineGroups() {
// private constructor
@@ -32,4 +57,103 @@ public final class CLALibCombineGroups {
throw new NotImplementedException();
}
+ /**
+ * Combine the column groups A and B together.
+ *
+ * The number of rows should be equal, and it is not verified so there will be unexpected behavior in such cases.
+ *
+ * @param a The first group to combine.
+ * @param b The second group to combine.
+ * @return A new column group containing the two.
+ */
+ public static AColGroup combine(AColGroup a, AColGroup b) {
+ IColIndex combinedColumns = ColIndexFactory.combine(a, b);
+
+ // try to recompress a and b if uncompressed
+ if(a instanceof ColGroupUncompressed)
+ a = a.recompress();
+
+ if(b instanceof ColGroupUncompressed)
+ b = b.recompress();
+
+ if(a instanceof AColGroupCompressed && b instanceof AColGroupCompressed)
+ return combineCompressed(combinedColumns, (AColGroupCompressed) a, (AColGroupCompressed) b);
+ else if(a instanceof ColGroupUncompressed || b instanceof ColGroupUncompressed)
+ // either side is uncompressed
+ return combineUC(combinedColumns, a, b);
+
+ throw new NotImplementedException(
+ "Not implemented combine for " + a.getClass().getSimpleName() + " - " + b.getClass().getSimpleName());
+
+ }
+
+ private static AColGroup combineCompressed(IColIndex combinedColumns, AColGroupCompressed ac,
+ AColGroupCompressed bc) {
+ IEncode ae = ac.getEncoding();
+ IEncode be = bc.getEncoding();
+ if(ae instanceof SparseEncoding && !(be instanceof SparseEncoding)) {
+ // the order must be sparse second unless both sparse.
+ return combineCompressed(combinedColumns, bc, ac);
+ }
+
+ IEncode ce = ae.combineNoResize(be);
+
+ if(ce instanceof DenseEncoding) {
+ DenseEncoding ced = (DenseEncoding) ce;
+ ADictionary cd = DictionaryFactory.combineDictionaries(ac, bc);
+ return ColGroupDDC.create(combinedColumns, cd, ced.getMap(), null);
+ }
+ else if(ce instanceof EmptyEncoding) {
+ return new ColGroupEmpty(combinedColumns);
+ }
+ else if(ce instanceof ConstEncoding) {
+ ADictionary cd = DictionaryFactory.combineDictionaries(ac, bc);
+ return ColGroupConst.create(combinedColumns, cd);
+ }
+ else if(ce instanceof SparseEncoding) {
+ SparseEncoding sed = (SparseEncoding) ce;
+ ADictionary cd = DictionaryFactory.combineDictionariesSparse(ac, bc);
+ double[] defaultTuple = constructDefaultTuple((AColGroupCompressed) ac, (AColGroupCompressed) bc);
+ return ColGroupSDC.create(combinedColumns, sed.getNumRows(), cd, defaultTuple, sed.getOffsets(), sed.getMap(),
+ null);
+ }
+
+ throw new NotImplementedException(
+ "Not implemented combine for " + ac.getClass().getSimpleName() + " - " + bc.getClass().getSimpleName());
+
+ }
+
+ private static AColGroup combineUC(IColIndex combinedColumns, AColGroup a, AColGroup b) {
+ int nRow = a instanceof ColGroupUncompressed ? //
+ ((ColGroupUncompressed) a).getData().getNumRows() : //
+ ((ColGroupUncompressed) b).getData().getNumRows();
+ // step 1 decompress both into target uncompressed MatrixBlock;
+ MatrixBlock target = new MatrixBlock(nRow, combinedColumns.size(), false);
+ target.allocateBlock();
+ DenseBlock db = target.getDenseBlock();
+
+ IColIndex aTempCols = ColIndexFactory.getColumnMapping(combinedColumns, a.getColIndices());
+ a.copyAndSet(aTempCols).decompressToDenseBlock(db, 0, nRow, 0, 0);
+ IColIndex bTempCols = ColIndexFactory.getColumnMapping(combinedColumns, b.getColIndices());
+ b.copyAndSet(bTempCols).decompressToDenseBlock(db, 0, nRow, 0, 0);
+
+ target.recomputeNonZeros();
+
+ return ColGroupUncompressed.create(combinedColumns, target, false);
+
+ }
+
+ public static double[] constructDefaultTuple(AColGroupCompressed ac, AColGroupCompressed bc) {
+ double[] ret = new double[ac.getNumCols() + bc.getNumCols()];
+ if(ac instanceof IContainDefaultTuple ){
+ double[] defa = ((IContainDefaultTuple)ac).getDefaultTuple();
+ System.arraycopy(defa, 0, ret, 0, defa.length);
+ }
+ if(bc instanceof IContainDefaultTuple){
+ double[] defb = ((IContainDefaultTuple)bc).getDefaultTuple();
+ System.arraycopy(defb, 0, ret, ac.getNumCols(), defb.length);
+ }
+ return ret;
+ }
+
}
diff --git a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java
index c6d1a4ddcd..8257351e22 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java
@@ -218,7 +218,7 @@ public class ColGroupNegativeTests {
}
@Override
- protected boolean sameIndexStructure(AColGroupCompressed that) {
+ public boolean sameIndexStructure(AColGroupCompressed that) {
return false;
}
@@ -387,6 +387,12 @@ public class ColGroupNegativeTests {
// TODO Auto-generated method stub
throw new UnsupportedOperationException("Unimplemented method 'getCompressionInfo'");
}
+
+ @Override
+ protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) {
+ // TODO Auto-generated method stub
+ throw new UnsupportedOperationException("Unimplemented method 'fixColIndexes'");
+ }
}
private class FakeDictBasedColGroup extends ADictBasedColGroup {
@@ -625,5 +631,17 @@ public class ColGroupNegativeTests {
// TODO Auto-generated method stub
throw new UnsupportedOperationException("Unimplemented method 'getCompressionInfo'");
}
+
+ @Override
+ public boolean sameIndexStructure(AColGroupCompressed that) {
+ // TODO Auto-generated method stub
+ throw new UnsupportedOperationException("Unimplemented method 'sameIndexStructure'");
+ }
+
+ @Override
+ protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) {
+ // TODO Auto-generated method stub
+ throw new UnsupportedOperationException("Unimplemented method 'fixColIndexes'");
+ }
}
}
diff --git a/src/test/java/org/apache/sysds/test/component/compress/colgroup/NegativeConstTests.java b/src/test/java/org/apache/sysds/test/component/compress/colgroup/NegativeConstTests.java
index d7aba11206..7f60b4341e 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/colgroup/NegativeConstTests.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/colgroup/NegativeConstTests.java
@@ -85,7 +85,7 @@ public class NegativeConstTests {
@Test(expected = NullPointerException.class)
public void testConstConstruction_null_02() {
- ColGroupConst.create(null, null);
+ ColGroupConst.create(null, (double[])null);
}
@Test(expected = NullPointerException.class)
diff --git a/src/test/java/org/apache/sysds/test/component/compress/dictionary/CombineTest.java b/src/test/java/org/apache/sysds/test/component/compress/dictionary/CombineTest.java
new file mode 100644
index 0000000000..20bec2036c
--- /dev/null
+++ b/src/test/java/org/apache/sysds/test/component/compress/dictionary/CombineTest.java
@@ -0,0 +1,505 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.component.compress.dictionary;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.sysds.runtime.compress.colgroup.AColGroup;
+import org.apache.sysds.runtime.compress.colgroup.AColGroup.CompressionType;
+import org.apache.sysds.runtime.compress.colgroup.AColGroupCompressed;
+import org.apache.sysds.runtime.compress.colgroup.ADictBasedColGroup;
+import org.apache.sysds.runtime.compress.colgroup.ColGroupConst;
+import org.apache.sysds.runtime.compress.colgroup.ColGroupDDC;
+import org.apache.sysds.runtime.compress.colgroup.ColGroupEmpty;
+import org.apache.sysds.runtime.compress.colgroup.ColGroupSDC;
+import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
+import org.apache.sysds.runtime.compress.colgroup.dictionary.Dictionary;
+import org.apache.sysds.runtime.compress.colgroup.dictionary.DictionaryFactory;
+import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory;
+import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory;
+import org.apache.sysds.runtime.compress.colgroup.offset.OffsetFactory;
+import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+import org.apache.sysds.test.TestUtils;
+import org.junit.Test;
+
+public class CombineTest {
+
+ protected static final Log LOG = LogFactory.getLog(CombineTest.class.getName());
+
+ @Test
+ public void singleBothSides() {
+ try {
+
+ ADictionary a = Dictionary.create(new double[] {1.2});
+ ADictionary b = Dictionary.create(new double[] {1.4});
+
+ ADictionary c = DictionaryFactory.combineFullDictionaries(a, 1, b, 1);
+
+ assertEquals(c.getValue(0, 0, 2), 1.2, 0.0);
+ assertEquals(c.getValue(0, 1, 2), 1.4, 0.0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void singleOneSideBothSides() {
+ try {
+ ADictionary a = Dictionary.create(new double[] {1.2, 1.3});
+ ADictionary b = Dictionary.create(new double[] {1.4});
+
+ ADictionary c = DictionaryFactory.combineFullDictionaries(a, 1, b, 1);
+
+ assertEquals(c.getValue(0, 0, 2), 1.2, 0.0);
+ assertEquals(c.getValue(0, 1, 2), 1.4, 0.0);
+ assertEquals(c.getValue(1, 0, 2), 1.3, 0.0);
+ assertEquals(c.getValue(1, 1, 2), 1.4, 0.0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void twoBothSides() {
+ try {
+ ADictionary a = Dictionary.create(new double[] {1.2, 1.3});
+ ADictionary b = Dictionary.create(new double[] {1.4, 1.5});
+
+ ADictionary c = DictionaryFactory.combineFullDictionaries(a, 1, b, 1);
+
+ assertEquals(c.getValue(0, 0, 2), 1.2, 0.0);
+ assertEquals(c.getValue(0, 1, 2), 1.4, 0.0);
+ assertEquals(c.getValue(1, 0, 2), 1.3, 0.0);
+ assertEquals(c.getValue(1, 1, 2), 1.4, 0.0);
+ assertEquals(c.getValue(2, 0, 2), 1.2, 0.0);
+ assertEquals(c.getValue(2, 1, 2), 1.5, 0.0);
+ assertEquals(c.getValue(3, 0, 2), 1.3, 0.0);
+ assertEquals(c.getValue(3, 1, 2), 1.5, 0.0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void sparseSparse() {
+ try {
+ ADictionary a = Dictionary.create(new double[] {3});
+ ADictionary b = Dictionary.create(new double[] {4});
+ double[] ad = new double[] {0};
+ double[] bd = new double[] {0};
+
+ ADictionary c = DictionaryFactory.combineSDC(a, ad, b, bd);
+ MatrixBlock ret = c.getMBDict(2).getMatrixBlock();
+
+ MatrixBlock exp = new MatrixBlock(4, 2, new double[] {0, 0, 3, 0, 0, 4, 3, 4});
+ TestUtils.compareMatricesBitAvgDistance(ret, exp, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void sparseSparse2() {
+ try {
+ ADictionary a = Dictionary.create(new double[] {3});
+ ADictionary b = Dictionary.create(new double[] {4, 4});
+ double[] ad = new double[] {0};
+ double[] bd = new double[] {0, 0};
+
+ ADictionary c = DictionaryFactory.combineSDC(a, ad, b, bd);
+ MatrixBlock ret = c.getMBDict(2).getMatrixBlock();
+
+ MatrixBlock exp = new MatrixBlock(4, 3, new double[] {0, 0, 0, 3, 0, 0, 0, 4, 4, 3, 4, 4});
+ TestUtils.compareMatricesBitAvgDistance(ret, exp, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void sparseSparse3() {
+ try {
+ ADictionary a = Dictionary.create(new double[] {3});
+ ADictionary b = Dictionary.create(new double[] {4});
+ double[] ad = new double[] {1};
+ double[] bd = new double[] {2};
+
+ ADictionary c = DictionaryFactory.combineSDC(a, ad, b, bd);
+ MatrixBlock ret = c.getMBDict(2).getMatrixBlock();
+
+ MatrixBlock exp = new MatrixBlock(4, 2, new double[] {//
+ 1, 2, //
+ 3, 2, //
+ 1, 4, //
+ 3, 4});
+ TestUtils.compareMatricesBitAvgDistance(ret, exp, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void sparseSparse4() {
+ try {
+ ADictionary a = Dictionary.create(new double[] {3, 2});
+ ADictionary b = Dictionary.create(new double[] {4, 4});
+ double[] ad = new double[] {0, 1};
+ double[] bd = new double[] {0, 2};
+
+ ADictionary c = DictionaryFactory.combineSDC(a, ad, b, bd);
+ MatrixBlock ret = c.getMBDict(2).getMatrixBlock();
+
+ MatrixBlock exp = new MatrixBlock(4, 4, new double[] {//
+ 0, 1, 0, 2, //
+ 3, 2, 0, 2, //
+ 0, 1, 4, 4, //
+ 3, 2, 4, 4});
+ TestUtils.compareMatricesBitAvgDistance(ret, exp, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void sparseSparse5() {
+ try {
+ ADictionary a = Dictionary.create(new double[] {3, 2, 7, 8});
+ ADictionary b = Dictionary.create(new double[] {4, 4});
+ double[] ad = new double[] {0, 1};
+ double[] bd = new double[] {0, 2};
+
+ ADictionary c = DictionaryFactory.combineSDC(a, ad, b, bd);
+ MatrixBlock ret = c.getMBDict(2).getMatrixBlock();
+
+ MatrixBlock exp = new MatrixBlock(6, 4, new double[] {//
+ 0, 1, 0, 2, //
+ 3, 2, 0, 2, //
+ 7, 8, 0, 2, //
+ 0, 1, 4, 4, //
+ 3, 2, 4, 4, //
+ 7, 8, 4, 4,});
+ TestUtils.compareMatricesBitAvgDistance(ret, exp, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void sparseSparse6() {
+ try {
+ ADictionary a = Dictionary.create(new double[] {3, 2, 7, 8});
+ ADictionary b = Dictionary.create(new double[] {4, 4, 9, 5});
+ double[] ad = new double[] {0, 1};
+ double[] bd = new double[] {0, 2};
+
+ ADictionary c = DictionaryFactory.combineSDC(a, ad, b, bd);
+ MatrixBlock ret = c.getMBDict(2).getMatrixBlock();
+
+ MatrixBlock exp = new MatrixBlock(9, 4, new double[] {//
+ 0, 1, 0, 2, //
+ 3, 2, 0, 2, //
+ 7, 8, 0, 2, //
+ 0, 1, 4, 4, //
+ 3, 2, 4, 4, //
+ 7, 8, 4, 4, //
+ 0, 1, 9, 5, //
+ 3, 2, 9, 5, //
+ 7, 8, 9, 5,});
+ TestUtils.compareMatricesBitAvgDistance(ret, exp, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void combineEmpties() {
+ assertNull(DictionaryFactory.combineDictionaries(ColGroupEmpty.create(120), ColGroupEmpty.create(22)));
+ }
+
+ @Test(expected = NotImplementedException.class)
+ public void combineNotImplemented1() {
+ ADictBasedColGroup m = mock(ADictBasedColGroup.class);
+ when(m.getCompType()).thenReturn(CompressionType.UNCOMPRESSED);
+ DictionaryFactory.combineDictionaries(m, m);
+ }
+
+ @Test(expected = NotImplementedException.class)
+ public void combineNotImplemented2() {
+ ADictBasedColGroup m = mock(ADictBasedColGroup.class);
+ when(m.getCompType()).thenReturn(CompressionType.UNCOMPRESSED);
+ ADictBasedColGroup d = mock(ColGroupDDC.class);
+ when(d.getCompType()).thenReturn(CompressionType.DDC);
+ DictionaryFactory.combineDictionaries(d, m);
+ }
+
+ @Test(expected = NotImplementedException.class)
+ public void combineNotImplemented3() {
+ ADictBasedColGroup m = mock(ADictBasedColGroup.class);
+ when(m.getCompType()).thenReturn(CompressionType.UNCOMPRESSED);
+ ADictBasedColGroup d = mock(ColGroupDDC.class);
+ when(d.getCompType()).thenReturn(CompressionType.DDC);
+ DictionaryFactory.combineDictionaries(m, d);
+ }
+
+ @Test(expected = NotImplementedException.class)
+ public void combineNotImplemented4() {
+ ADictBasedColGroup m = mock(ADictBasedColGroup.class);
+ when(m.getCompType()).thenReturn(CompressionType.UNCOMPRESSED);
+ ADictBasedColGroup s = mock(ColGroupSDC.class);
+ when(s.getCompType()).thenReturn(CompressionType.SDC);
+ DictionaryFactory.combineDictionaries(s, m);
+ }
+
+ @Test(expected = NotImplementedException.class)
+ public void combineNotImplemented5() {
+ ADictBasedColGroup m = mock(ADictBasedColGroup.class);
+ when(m.getCompType()).thenReturn(CompressionType.UNCOMPRESSED);
+ ADictBasedColGroup s = mock(ColGroupSDC.class);
+ when(s.getCompType()).thenReturn(CompressionType.SDC);
+ DictionaryFactory.combineDictionaries(m, s);
+ }
+
+ @Test(expected = NotImplementedException.class)
+ public void combineNotImplemented6() {
+ AColGroupCompressed m = mock(AColGroupCompressed.class);
+ when(m.getCompType()).thenReturn(CompressionType.UNCOMPRESSED);
+ AColGroupCompressed s = mock(ColGroupSDC.class);
+ when(s.getCompType()).thenReturn(CompressionType.SDC);
+ DictionaryFactory.combineDictionaries(s, m);
+ }
+
+ @Test(expected = NotImplementedException.class)
+ public void combineNotImplemented7() {
+ AColGroupCompressed m = mock(AColGroupCompressed.class);
+ when(m.getCompType()).thenReturn(CompressionType.UNCOMPRESSED);
+ AColGroupCompressed s = mock(AColGroupCompressed.class);
+ when(s.getCompType()).thenReturn(CompressionType.UNCOMPRESSED);
+ DictionaryFactory.combineDictionaries(s, m);
+ }
+
+ @Test(expected = NotImplementedException.class)
+ public void combineNotImplemented8() {
+ AColGroupCompressed m = mock(AColGroupCompressed.class);
+ when(m.getCompType()).thenReturn(CompressionType.UNCOMPRESSED);
+ AColGroupCompressed s = mock(ColGroupSDC.class);
+ when(s.getCompType()).thenReturn(CompressionType.SDC);
+ DictionaryFactory.combineDictionaries(m, s);
+ }
+
+ @Test(expected = NotImplementedException.class)
+ public void combineNotImplemented9() {
+ AColGroupCompressed m = mock(ColGroupConst.class);
+ when(m.getCompType()).thenReturn(CompressionType.CONST);
+ AColGroupCompressed s = mock(ColGroupSDC.class);
+ when(s.getCompType()).thenReturn(CompressionType.SDC);
+ DictionaryFactory.combineDictionaries(m, s);
+ }
+
+ @Test(expected = NotImplementedException.class)
+ public void combineNotImplementedSparse1() {
+ ADictBasedColGroup m = mock(ADictBasedColGroup.class);
+ when(m.getCompType()).thenReturn(CompressionType.UNCOMPRESSED);
+ ADictBasedColGroup s = mock(ColGroupSDC.class);
+ when(s.getCompType()).thenReturn(CompressionType.SDC);
+ DictionaryFactory.combineDictionariesSparse(m, s);
+ }
+
+ @Test(expected = NotImplementedException.class)
+ public void combineNotImplementedSparse2() {
+ ADictBasedColGroup m = mock(ADictBasedColGroup.class);
+ when(m.getCompType()).thenReturn(CompressionType.UNCOMPRESSED);
+ ADictBasedColGroup s = mock(ColGroupSDC.class);
+ when(s.getCompType()).thenReturn(CompressionType.SDC);
+ DictionaryFactory.combineDictionariesSparse(s, m);
+ }
+
+ @Test(expected = NotImplementedException.class)
+ public void combineNotImplementedSparse3() {
+ ADictBasedColGroup m = mock(ColGroupSDC.class);
+ when(m.getCompType()).thenReturn(CompressionType.SDC);
+ ADictBasedColGroup s = mock(ColGroupDDC.class);
+ when(s.getCompType()).thenReturn(CompressionType.DDC);
+ DictionaryFactory.combineDictionariesSparse(s, m);
+ }
+
+ @Test(expected = NotImplementedException.class)
+ public void combineNotImplementedSparse4() {
+ ADictBasedColGroup m = mock(ColGroupConst.class);
+ when(m.getCompType()).thenReturn(CompressionType.CONST);
+ ADictBasedColGroup s = mock(ColGroupDDC.class);
+ when(s.getCompType()).thenReturn(CompressionType.DDC);
+ DictionaryFactory.combineDictionariesSparse(s, m);
+ }
+
+ @Test(expected = NotImplementedException.class)
+ public void combineNotImplementedSparse5() {
+ ADictBasedColGroup m = mock(ColGroupSDC.class);
+ when(m.getCompType()).thenReturn(CompressionType.SDC);
+ ADictBasedColGroup s = mock(ColGroupDDC.class);
+ when(s.getCompType()).thenReturn(CompressionType.DDC);
+ DictionaryFactory.combineDictionariesSparse(s, m);
+ }
+
+ @Test(expected = NotImplementedException.class)
+ public void combineNotImplementedSparse6() {
+ ADictBasedColGroup m = mock(ColGroupConst.class);
+ when(m.getCompType()).thenReturn(CompressionType.CONST);
+ ADictBasedColGroup s = mock(ColGroupDDC.class);
+ when(s.getCompType()).thenReturn(CompressionType.DDC);
+ DictionaryFactory.combineDictionariesSparse(m, s);
+ }
+
+ @Test
+ public void sparseSparseConst1() {
+ try {
+ ADictionary a = Dictionary.create(new double[] {3, 2, 7, 8});
+ // ADictionary b = Dictionary.create(new double[] {4, 4, 9, 5});
+
+ double[] bd = new double[] {0, 2};
+
+ ADictionary c = DictionaryFactory.combineSparseConstSparseRet(a, 2, bd);
+ MatrixBlock ret = c.getMBDict(2).getMatrixBlock();
+
+ MatrixBlock exp = new MatrixBlock(2, 4, new double[] {//
+ 3, 2, 0, 2, //
+ 7, 8, 0, 2,});
+ TestUtils.compareMatricesBitAvgDistance(ret, exp, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void sparseSparseConst2() {
+ try {
+ ADictionary a = Dictionary.create(new double[] {3, 2, 7, 8});
+ // ADictionary b = Dictionary.create(new double[] {4, 4, 9, 5});
+
+ double[] bd = new double[] {0, 2};
+
+ ADictionary c = DictionaryFactory.combineSparseConstSparseRet(a, 1, bd);
+ MatrixBlock ret = c.getMBDict(2).getMatrixBlock();
+
+ MatrixBlock exp = new MatrixBlock(2, 3, new double[] {//
+ 3, 0, 2, //
+ 2, 0, 2, //
+ 7, 0, 2, //
+ 8, 0, 2,});
+ TestUtils.compareMatricesBitAvgDistance(ret, exp, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testEmpty() {
+ try {
+ ADictionary d = Dictionary.create(new double[] {3, 2, 7, 8});
+ AColGroup a = ColGroupDDC.create(ColIndexFactory.create(2), d, MapToFactory.create(10, 2), null);
+ ColGroupEmpty b = new ColGroupEmpty(ColIndexFactory.create(4));
+
+ ADictionary c = DictionaryFactory.combineDictionaries((AColGroupCompressed) a, (AColGroupCompressed) b);
+ MatrixBlock ret = c.getMBDict(2).getMatrixBlock();
+
+ MatrixBlock exp = new MatrixBlock(2, 6, new double[] {//
+ 3, 2, 0, 0, 0, 0, //
+ 7, 8, 0, 0, 0, 0,});
+ TestUtils.compareMatricesBitAvgDistance(ret, exp, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void combineDictionariesSparse1() {
+ try {
+ ADictionary d = Dictionary.create(new double[] {3, 2, 7, 8});
+ AColGroup a = ColGroupSDC.create(ColIndexFactory.create(2), 500, d, new double[] {1, 2},
+ OffsetFactory.createOffset(new int[] {3, 4}), MapToFactory.create(10, 2), null);
+ ColGroupEmpty b = new ColGroupEmpty(ColIndexFactory.create(4));
+
+ ADictionary c = DictionaryFactory.combineDictionariesSparse((AColGroupCompressed) a, (AColGroupCompressed) b);
+ MatrixBlock ret = c.getMBDict(2).getMatrixBlock();
+
+ MatrixBlock exp = new MatrixBlock(2, 6, new double[] {//
+ 3, 2, 0, 0, 0, 0, //
+ 7, 8, 0, 0, 0, 0,});
+ TestUtils.compareMatricesBitAvgDistance(ret, exp, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+
+ @Test
+ public void combineDictionariesSparse2() {
+ try {
+ ADictionary d = Dictionary.create(new double[] {3, 2, 7, 8});
+ AColGroup b = ColGroupSDC.create(ColIndexFactory.create(2), 500, d, new double[] {1, 2},
+ OffsetFactory.createOffset(new int[] {3, 4}), MapToFactory.create(10, 2), null);
+ ColGroupEmpty a = new ColGroupEmpty(ColIndexFactory.create(4));
+
+ ADictionary c = DictionaryFactory.combineDictionariesSparse((AColGroupCompressed) a, (AColGroupCompressed) b);
+ MatrixBlock ret = c.getMBDict(2).getMatrixBlock();
+
+ MatrixBlock exp = new MatrixBlock(2, 6, new double[] {//
+ 0, 0, 0, 0, 3, 2, //
+ 0, 0, 0, 0, 7, 8,});
+ TestUtils.compareMatricesBitAvgDistance(ret, exp, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
diff --git a/src/test/java/org/apache/sysds/test/component/compress/dictionary/CustomDictionaryTest.java b/src/test/java/org/apache/sysds/test/component/compress/dictionary/CustomDictionaryTest.java
index 31f301749a..8dd8a1165b 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/dictionary/CustomDictionaryTest.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/dictionary/CustomDictionaryTest.java
@@ -152,4 +152,5 @@ public class CustomDictionaryTest {
public void createZeroRowMatrixBlock() {
MatrixBlockDictionary.create(new MatrixBlock(0, 10, 10.0));
}
+
}
diff --git a/src/test/java/org/apache/sysds/test/component/compress/indexes/CustomIndexTest.java b/src/test/java/org/apache/sysds/test/component/compress/indexes/CustomIndexTest.java
index 1b244fc319..eb2d386a5b 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/indexes/CustomIndexTest.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/indexes/CustomIndexTest.java
@@ -24,8 +24,10 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
+import org.apache.sysds.runtime.compress.DMLCompressionException;
import org.apache.sysds.runtime.compress.colgroup.AColGroup;
import org.apache.sysds.runtime.compress.colgroup.indexes.ArrayIndex;
import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory;
@@ -38,6 +40,8 @@ import org.apache.sysds.runtime.compress.utils.Util;
import org.junit.Test;
import org.mockito.Mockito;
+import scala.util.Random;
+
public class CustomIndexTest {
@Test
public void testSingeSlice1() {
@@ -585,10 +589,219 @@ public class CustomIndexTest {
}
@Test
- public void compareOld2(){
+ public void compareOld2() {
final int domain = 32;
int[] colIndexes = Util.genColsIndices(0, domain);
IColIndex colIndexes2 = ColIndexFactory.create(0, domain);
IndexesTest.compare(colIndexes, colIndexes2);
}
+
+ @Test
+ public void getCombine_1() {
+ IColIndex a = ColIndexFactory.createI(1, 2, 3);
+ IColIndex b = ColIndexFactory.createI(4, 5, 6);
+ IColIndex c = ColIndexFactory.combine(a, b);
+ assertTrue(c.equals(ColIndexFactory.createI(1, 2, 3, 4, 5, 6)));
+ }
+
+ @Test
+ public void getCombine_2() {
+ IColIndex a = ColIndexFactory.createI(1, 3);
+ IColIndex b = ColIndexFactory.createI(4, 5, 6);
+ IColIndex c = ColIndexFactory.combine(a, b);
+ assertTrue(c.equals(ColIndexFactory.createI(1, 3, 4, 5, 6)));
+ }
+
+ @Test
+ public void getCombine_3() {
+ IColIndex a = ColIndexFactory.createI(1, 3);
+ IColIndex b = ColIndexFactory.createI(2, 5, 6);
+ IColIndex c = ColIndexFactory.combine(a, b);
+ assertTrue(c.equals(ColIndexFactory.createI(1, 2, 3, 5, 6)));
+ }
+
+ @Test
+ public void getCombine_4() {
+ IColIndex a = ColIndexFactory.createI(1, 3);
+ IColIndex b = ColIndexFactory.createI(0, 2, 6);
+ IColIndex c = ColIndexFactory.combine(a, b);
+ assertTrue(c.equals(ColIndexFactory.createI(0, 1, 2, 3, 6)));
+ }
+
+ @Test
+ public void getMapping_1() {
+ IColIndex c = ColIndexFactory.createI(1, 2, 3);
+ IColIndex a = ColIndexFactory.createI(1);
+ IColIndex am = ColIndexFactory.getColumnMapping(c, a);
+ assertTrue(am.equals(ColIndexFactory.createI(0)));
+ }
+
+ @Test
+ public void getMapping_2() {
+ IColIndex c = ColIndexFactory.createI(1, 2, 3);
+ IColIndex a = ColIndexFactory.createI(2);
+ IColIndex am = ColIndexFactory.getColumnMapping(c, a);
+ assertTrue(am.equals(ColIndexFactory.createI(1)));
+ }
+
+ @Test
+ public void getMapping_3() {
+ IColIndex c = ColIndexFactory.createI(1, 3);
+ IColIndex a = ColIndexFactory.createI(3);
+ IColIndex am = ColIndexFactory.getColumnMapping(c, a);
+ assertTrue(am.equals(ColIndexFactory.createI(1)));
+ }
+
+ @Test
+ public void getMapping_4() {
+ IColIndex c = ColIndexFactory.createI(1, 2, 3);
+ IColIndex a = ColIndexFactory.createI(3);
+ IColIndex am = ColIndexFactory.getColumnMapping(c, a);
+ assertTrue(am.equals(ColIndexFactory.createI(2)));
+ }
+
+ @Test
+ public void getMapping_5() {
+ IColIndex c = ColIndexFactory.createI(1, 2, 3);
+ IColIndex a = ColIndexFactory.createI(2, 3);
+ IColIndex am = ColIndexFactory.getColumnMapping(c, a);
+ assertTrue(am.equals(ColIndexFactory.createI(1, 2)));
+ }
+
+ @Test
+ public void getMapping_6() {
+ IColIndex c = ColIndexFactory.createI(1, 10, 100, 1000);
+ IColIndex a = ColIndexFactory.createI(10, 1000);
+ IColIndex am = ColIndexFactory.getColumnMapping(c, a);
+ assertTrue(am.equals(ColIndexFactory.createI(1, 3)));
+ }
+
+ @Test
+ public void rangeIndexSameEstimationCost() {
+ assertEquals(ColIndexFactory.estimateMemoryCost(10000, true), ColIndexFactory.estimateMemoryCost(1000000, true));
+ }
+
+ @Test
+ public void rangeIndexSameEstimationCost2() {
+ assertEquals(ColIndexFactory.estimateMemoryCost(134, true), ColIndexFactory.estimateMemoryCost(4215, true));
+ }
+
+ @Test
+ public void rangeIndexSameEstimationCost3() {
+ assertEquals(ColIndexFactory.estimateMemoryCost(1000000, true), ColIndexFactory.estimateMemoryCost(4215, true));
+ }
+
+ @Test
+ public void rangeIndexSameEstimationCost4() {
+ assertTrue(ColIndexFactory.estimateMemoryCost(2, true) <= ColIndexFactory.estimateMemoryCost(4215, true));
+ }
+
+ @Test
+ public void rangeIndexSameEstimationCost5() {
+ assertTrue(ColIndexFactory.estimateMemoryCost(1, true) <= ColIndexFactory.estimateMemoryCost(4215, true));
+ }
+
+ @Test
+ public void rangeIndexSameEstimationCost6() {
+ assertTrue(ColIndexFactory.estimateMemoryCost(1, true) <= ColIndexFactory.estimateMemoryCost(2, true));
+ }
+
+ @Test
+ public void rangeIndexSameEstimationCost7() {
+ assertTrue(ColIndexFactory.estimateMemoryCost(1, false) <= ColIndexFactory.estimateMemoryCost(2, false));
+ }
+
+ @Test
+ public void rangeIndexSameEstimationCost8() {
+ assertTrue(ColIndexFactory.estimateMemoryCost(10000, true) < ColIndexFactory.estimateMemoryCost(20, false));
+ }
+
+ @Test
+ public void estimateVSActual() {
+ Random r = new Random(134);
+ for(int i = 1; i < 50; i++) {
+ IColIndex rl = ColIndexFactory.create(IndexesTest.randomInc(i, r.nextInt()));
+ assertEquals(ColIndexFactory.estimateMemoryCost(i, false), rl.estimateInMemorySize());
+ }
+ }
+
+ @Test(expected = DMLCompressionException.class)
+ public void rangeReordering() {
+ ColIndexFactory.create(0, 10).getReorderingIndex();
+ }
+
+ @Test(expected = DMLCompressionException.class)
+ public void rangeSort() {
+ ColIndexFactory.create(0, 10).sort();
+ }
+
+ @Test
+ public void rangeIsSorted() {
+ assertTrue(ColIndexFactory.create(0, 10).isSorted());
+ }
+
+ @Test(expected = DMLCompressionException.class)
+ public void oneReordering() {
+ ColIndexFactory.createI(0).getReorderingIndex();
+ }
+
+ @Test(expected = DMLCompressionException.class)
+ public void oneSort() {
+ ColIndexFactory.createI(10).sort();
+ }
+
+ @Test
+ public void oneIsSorted() {
+ assertTrue(ColIndexFactory.createI(10).isSorted());
+ }
+
+ @Test
+ public void twoReordering1() {
+ assertTrue(Arrays.equals(new int[] {0, 1}, ColIndexFactory.createI(1, 10).getReorderingIndex()));
+ }
+
+ @Test
+ public void twoReordering2() {
+ assertTrue(Arrays.equals(new int[] {1, 0}, ColIndexFactory.createI(10, 1).getReorderingIndex()));
+ }
+
+ @Test
+ public void twoSort2() {
+ assertTrue(ColIndexFactory.createI(1, 10).equals(ColIndexFactory.createI(10, 1).sort()));
+ }
+
+ @Test
+ public void twoSort1() {
+ assertTrue(ColIndexFactory.createI(1, 10).equals(ColIndexFactory.createI(1, 10).sort()));
+ }
+
+ @Test
+ public void twoSorted1() {
+ assertTrue(ColIndexFactory.createI(0, 10).isSorted());
+ }
+
+ @Test
+ public void twoSorted2() {
+ assertFalse(ColIndexFactory.createI(10, -1).isSorted());
+ }
+
+ @Test
+ public void isSortedArray1() {
+ assertTrue(ColIndexFactory.createI(0, 1, 5, 7, 9).isSorted());
+ }
+
+ @Test
+ public void isSortedArray2() {
+ assertFalse(ColIndexFactory.createI(0, 1, 5, 3, 9).isSorted());
+ }
+
+ @Test
+ public void isSortedArray3() {
+ assertFalse(ColIndexFactory.createI(0, 1, 5, 9, -13).isSorted());
+ }
+
+ @Test
+ public void isSortedArray4() {
+ assertFalse(ColIndexFactory.createI(0, 1, 0, 1, 0).isSorted());
+ }
}
diff --git a/src/test/java/org/apache/sysds/test/component/compress/indexes/IndexesTest.java b/src/test/java/org/apache/sysds/test/component/compress/indexes/IndexesTest.java
index 24ebdf7467..ed5e81c97d 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/indexes/IndexesTest.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/indexes/IndexesTest.java
@@ -97,7 +97,7 @@ public class IndexesTest {
new int[] {4, 5, 6, 7, 8, 9}, //
ColIndexFactory.create(4, 10)});
- tests.add(createWithArray(1, 323));
+ tests.add(createWithArray(1, 323));
tests.add(createWithArray(2, 1414));
tests.add(createWithArray(144, 32));
tests.add(createWithArray(13, 23));
@@ -243,6 +243,12 @@ public class IndexesTest {
assertEquals(actual, ColIndexFactory.create(expected));
}
+ @Test
+ public void isContiguous() {
+ boolean c = expected[expected.length - 1] - expected[0] + 1 == expected.length;
+ assertEquals(c, actual.isContiguous());
+ }
+
@Test
public void combineSingleOneAbove() {
IColIndex b = new SingleIndex(expected[expected.length - 1] + 1);
@@ -327,6 +333,7 @@ public class IndexesTest {
private static void compare(int[] expected, IIterate actual) {
for(int i = 0; i < expected.length; i++) {
assertTrue(actual.hasNext());
+ assertEquals(i, actual.i());
assertEquals(expected[i], actual.next());
}
assertFalse(actual.hasNext());
@@ -343,6 +350,11 @@ public class IndexesTest {
}
private static Object[] createWithArray(int size, int seed) {
+ IntArrayList cols = randomInc(size, seed);
+ return new Object[] {cols.extractValues(true), ColIndexFactory.create(cols)};
+ }
+
+ public static IntArrayList randomInc(int size, int seed) {
IntArrayList cols = new IntArrayList();
Random r = new Random(seed);
cols.appendValue(r.nextInt(1000) + 1);
@@ -350,7 +362,7 @@ public class IndexesTest {
int prev = cols.get(i - 1);
cols.appendValue(r.nextInt(1000) + 1 + prev);
}
- return new Object[] {cols.extractValues(true), ColIndexFactory.create(cols)};
+ return cols;
}
private static Object[] createRangeWithArray(int size, int seed) {
@@ -362,7 +374,7 @@ public class IndexesTest {
cols.appendValue(1 + prev);
}
IColIndex ret = ColIndexFactory.create(cols);
- if(! (ret instanceof ArrayIndex))
+ if(!(ret instanceof ArrayIndex))
return new Object[] {cols.extractValues(true), ret};
else
throw new DMLRuntimeException("Invalid construction of range array");
diff --git a/src/test/java/org/apache/sysds/test/component/compress/lib/CombineGroupsTest.java b/src/test/java/org/apache/sysds/test/component/compress/lib/CombineGroupsTest.java
new file mode 100644
index 0000000000..f599175d3b
--- /dev/null
+++ b/src/test/java/org/apache/sysds/test/component/compress/lib/CombineGroupsTest.java
@@ -0,0 +1,396 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.component.compress.lib;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
+import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory;
+import org.apache.sysds.runtime.compress.colgroup.AColGroup;
+import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory;
+import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
+import org.apache.sysds.runtime.compress.colgroup.indexes.IIterate;
+import org.apache.sysds.runtime.compress.lib.CLALibCombineGroups;
+import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+import org.apache.sysds.test.TestUtils;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(value = Parameterized.class)
+public class CombineGroupsTest {
+ protected static final Log LOG = LogFactory.getLog(CombineGroupsTest.class.getName());
+
+ final MatrixBlock a;
+ final MatrixBlock b;
+ final CompressedMatrixBlock ac;
+ final CompressedMatrixBlock bc;
+
+ public CombineGroupsTest(MatrixBlock a, MatrixBlock b, CompressedMatrixBlock ac, CompressedMatrixBlock bc) {
+ this.a = a;
+ this.b = b;
+ this.ac = ac;
+ this.bc = bc;
+ }
+
+ @Parameters
+ public static Collection<Object[]> data() {
+ List<Object[]> tests = new ArrayList<>();
+
+ try {
+ int[] nCols = new int[] {1, 2, 5};
+ for(int nCol : nCols) {
+
+ MatrixBlock a = TestUtils.generateTestMatrixBlock(100, 1, 100, 103, 0.5, 230);
+ a = TestUtils.ceil(a);
+ a = cbindN(a, nCol);
+ CompressedMatrixBlock ac = com(a);
+ MatrixBlock b = TestUtils.generateTestMatrixBlock(100, 1, 13, 15, 0.5, 132);
+ b = TestUtils.ceil(b);
+ b = cbindN(b, nCol);
+ CompressedMatrixBlock bc = com(b);
+ CompressedMatrixBlock buc = ucom(b); // uncompressed col group
+ MatrixBlock c = new MatrixBlock(100, nCol, 1.34);
+ CompressedMatrixBlock cc = com(c); // const
+ MatrixBlock e = new MatrixBlock(100, nCol, 0);
+ CompressedMatrixBlock ec = com(e); // empty
+
+ MatrixBlock u = TestUtils.generateTestMatrixBlock(100, 1, 0, 1, 1.0, 2315);
+ u = cbindN(u, nCol);
+ CompressedMatrixBlock uuc = ucom(u);
+
+ // Default DDC case
+ tests.add(new Object[] {a, b, ac, bc});
+
+ // Empty and Const cases.
+ tests.add(new Object[] {a, c, ac, cc}); // const ddc
+ tests.add(new Object[] {c, a, cc, ac});
+ tests.add(new Object[] {a, e, ac, ec}); // empty ddc
+ tests.add(new Object[] {e, a, ec, ac});
+ tests.add(new Object[] {c, e, cc, ec}); // empty const
+ tests.add(new Object[] {e, c, ec, cc});
+ tests.add(new Object[] {e, e, ec, ec}); // empty empty
+ tests.add(new Object[] {c, c, cc, cc}); // const const
+
+ // Uncompressed Case
+ tests.add(new Object[] {a, b, ac, buc}); // compressable uncompressed group
+ tests.add(new Object[] {b, a, buc, ac});
+ tests.add(new Object[] {a, u, ac, uuc}); // incompressable input
+ tests.add(new Object[] {u, a, uuc, ac});
+ tests.add(new Object[] {u, u, uuc, uuc}); // both sides incompressable
+
+ MatrixBlock s = TestUtils.generateTestMatrixBlock(100, 1, 1, 3, 0.10, 123);
+ s = TestUtils.ceil(s);
+ s = cbindN(s, nCol);
+ CompressedMatrixBlock sc = com(s); // SDCZeroSingle
+
+ MatrixBlock s2 = TestUtils.generateTestMatrixBlock(100, 1, 0, 3, 0.2, 321);
+ s2 = TestUtils.ceil(s2);
+ s2 = cbindN(s2, nCol);
+ CompressedMatrixBlock s2c = com(s2); // SDCZero
+
+ // SDC cases
+ tests.add(new Object[] {s, a, sc, ac});
+ tests.add(new Object[] {a, s, ac, sc});
+ tests.add(new Object[] {s2, a, s2c, ac});
+ tests.add(new Object[] {a, s2, ac, s2c});
+
+ tests.add(new Object[] {s, s, sc, sc});
+ tests.add(new Object[] {s, s2, sc, s2c});
+
+ // empty and const SDC
+ tests.add(new Object[] {s, e, sc, ec});
+ tests.add(new Object[] {s, c, sc, cc});
+ tests.add(new Object[] {e, s, ec, sc});
+ tests.add(new Object[] {c, s, cc, sc});
+ }
+
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail("failed constructing tests");
+ }
+
+ return tests;
+ }
+
+ private static CompressedMatrixBlock com(MatrixBlock m) {
+ return (CompressedMatrixBlock) CompressedMatrixBlockFactory.compress(m).getLeft();
+ }
+
+ private static CompressedMatrixBlock ucom(MatrixBlock m) {
+ return CompressedMatrixBlockFactory.genUncompressedCompressedMatrixBlock(m);
+ }
+
+ @Test
+ public void combineTest() {
+ try {
+
+ // combined.
+ MatrixBlock c = a.append(b);
+ CompressedMatrixBlock cc = appendNoMerge(ac, bc);
+
+ TestUtils.compareMatricesBitAvgDistance(c, cc, 0, 0, "Not the same verification");
+ CompressedMatrixBlock ccc = (CompressedMatrixBlock) cc;
+ List<AColGroup> groups = ccc.getColGroups();
+ if(groups.size() > 1) {
+
+ AColGroup cg = CLALibCombineGroups.combine(groups.get(0), groups.get(1));
+ ccc.allocateColGroup(cg);
+ TestUtils.compareMatricesBitAvgDistance(c, ccc, 0, 0, "Not the same combined");
+ }
+
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+
+ }
+
+ @Test
+ public void combineWithExtraEmptyColumnsBefore() {
+ try {
+ MatrixBlock e = new MatrixBlock(a.getNumRows(), 2, true);
+ // combined.
+ MatrixBlock c = e.append(a).append(b);
+ CompressedMatrixBlock ec = CompressedMatrixBlockFactory.createConstant(a.getNumRows(), 2, 0.0);
+ CompressedMatrixBlock cc = appendNoMerge(ec, appendNoMerge(ac, bc));
+
+ TestUtils.compareMatricesBitAvgDistance(c, cc, 0, 0, "Not the same verification");
+ CompressedMatrixBlock ccc = (CompressedMatrixBlock) cc;
+ List<AColGroup> groups = ccc.getColGroups();
+ if(groups.size() > 1) {
+
+ AColGroup cg = CLALibCombineGroups.combine(groups.get(1), groups.get(2));
+ ccc.allocateColGroup(cg);
+ TestUtils.compareMatricesBitAvgDistance(c, ccc, 0, 0, "Not the same combined");
+ }
+
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void combineWithExtraConstColumnsBefore() {
+ try {
+ MatrixBlock e = new MatrixBlock(a.getNumRows(), 2, 1.0);
+ // combined.
+ MatrixBlock c = e.append(a).append(b);
+ CompressedMatrixBlock ec = CompressedMatrixBlockFactory.createConstant(a.getNumRows(), 2, 1.0);
+ CompressedMatrixBlock cc = appendNoMerge(ec, appendNoMerge(ac, bc));
+
+ TestUtils.compareMatricesBitAvgDistance(c, cc, 0, 0, "Not the same verification");
+ CompressedMatrixBlock ccc = (CompressedMatrixBlock) cc;
+ List<AColGroup> groups = ccc.getColGroups();
+ if(groups.size() > 1) {
+
+ AColGroup cg = CLALibCombineGroups.combine(groups.get(1), groups.get(2));
+ ccc.allocateColGroup(cg);
+ TestUtils.compareMatricesBitAvgDistance(c, ccc, 0, 0, "Not the same combined");
+ }
+
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void combineWithExtraConstColumnsBetween() {
+ try {
+ MatrixBlock e = new MatrixBlock(a.getNumRows(), 2, 1.0);
+ // combined.
+ MatrixBlock c = a.append(e).append(b);
+ CompressedMatrixBlock ec = CompressedMatrixBlockFactory.createConstant(a.getNumRows(), 2, 1.0);
+ CompressedMatrixBlock cc = appendNoMerge(ac, appendNoMerge(ec, bc));
+
+ TestUtils.compareMatricesBitAvgDistance(c, cc, 0, 0, "Not the same verification");
+ CompressedMatrixBlock ccc = (CompressedMatrixBlock) cc;
+ List<AColGroup> groups = ccc.getColGroups();
+ if(groups.size() > 1) {
+
+ AColGroup cg = CLALibCombineGroups.combine(groups.get(0), groups.get(2));
+ ccc.allocateColGroup(cg);
+ TestUtils.compareMatricesBitAvgDistance(c, ccc, 0, 0, "Not the same combined");
+ }
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void combineWithExtraConstColumnsAfter() {
+ try {
+ MatrixBlock e = new MatrixBlock(a.getNumRows(), 2, 1.0);
+ // combined.
+ MatrixBlock c = a.append(b).append(e);
+ CompressedMatrixBlock ec = CompressedMatrixBlockFactory.createConstant(a.getNumRows(), 2, 1.0);
+ CompressedMatrixBlock cc = appendNoMerge(ac, appendNoMerge(bc, ec));
+
+ TestUtils.compareMatricesBitAvgDistance(c, cc, 0, 0, "Not the same verification");
+ CompressedMatrixBlock ccc = (CompressedMatrixBlock) cc;
+ List<AColGroup> groups = ccc.getColGroups();
+ if(groups.size() > 1) {
+
+ AColGroup cg = CLALibCombineGroups.combine(groups.get(0), groups.get(1));
+ ccc.allocateColGroup(cg);
+ TestUtils.compareMatricesBitAvgDistance(c, ccc, 0, 0, "Not the same combined");
+ }
+
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void combineMixingColumnIndexes() {
+
+ try {
+ if(a.getNumColumns() + b.getNumColumns() == 2)
+ return; // not relevant test
+ // combined.
+ MatrixBlock c = a.append(b);
+ CompressedMatrixBlock cc = appendNoMerge(ac, bc);
+
+ TestUtils.compareMatricesBitAvgDistance(c, cc, 0, 0, "Not the same verification");
+ // mix columns...
+ int[] mix = new int[c.getNumColumns()];
+ inc(mix);
+ shuffle(mix, 13);
+ c = applyShuffle(c, mix);
+ cc = applyCompressedShuffle(cc, mix);
+ TestUtils.compareMatricesBitAvgDistance(c, cc, 0, 0, "Not the same after shuffle verification: ");
+
+ CompressedMatrixBlock ccc = (CompressedMatrixBlock) cc;
+ List<AColGroup> groups = ccc.getColGroups();
+ if(groups.size() > 1) {
+
+ AColGroup cg = CLALibCombineGroups.combine(groups.get(0), groups.get(1));
+ assertTrue(cg.getColIndices().isSorted());
+
+ ccc.allocateColGroup(cg);
+ TestUtils.compareMatricesBitAvgDistance(c, ccc, 0, 0, "Not the same combined " );
+ }
+
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private static CompressedMatrixBlock appendNoMerge(CompressedMatrixBlock a, CompressedMatrixBlock b) {
+ CompressedMatrixBlock cc = new CompressedMatrixBlock(a.getNumRows(), a.getNumColumns() + b.getNumColumns());
+ appendColGroups(cc, a.getColGroups(), b.getColGroups(), a.getNumColumns());
+ cc.setNonZeros(a.getNonZeros() + b.getNonZeros());
+ cc.setOverlapping(a.isOverlapping() || b.isOverlapping());
+ return cc;
+ }
+
+ private static void appendColGroups(CompressedMatrixBlock ret, List<AColGroup> left, List<AColGroup> right,
+ int leftNumCols) {
+
+ // shallow copy of lhs column groups
+ ret.allocateColGroupList(new ArrayList<AColGroup>(left.size() + right.size()));
+
+ for(AColGroup group : left)
+ ret.getColGroups().add(group);
+
+ for(AColGroup group : right)
+ ret.getColGroups().add(group.shiftColIndices(leftNumCols));
+
+ }
+
+ private static MatrixBlock cbindN(MatrixBlock a, int n) {
+ MatrixBlock b = a;
+ for(int i = 1; i < n; i++) {
+ b = b.append(a);
+ }
+ return b;
+ }
+
+ private static void inc(int[] ar) {
+ for(int i = 0; i < ar.length; i++) {
+ ar[i] = i;
+ }
+ }
+
+ private static void shuffle(int[] ar, int seed) {
+ Random r = new Random();
+ for(int i = 0; i < ar.length; i++) {
+ int f = r.nextInt(ar.length);
+ int t = r.nextInt(ar.length);
+ int tmp = ar[t];
+ ar[t] = ar[f];
+ ar[f] = tmp;
+ }
+ }
+
+ private static MatrixBlock applyShuffle(MatrixBlock a, int[] mix) {
+ MatrixBlock ret = new MatrixBlock(a.getNumRows(), a.getNumColumns(), a.getNonZeros());
+ for(int r = 0; r < a.getNumRows(); r++)
+ for(int c = 0; c < a.getNumColumns(); c++)
+ ret.quickSetValue(r, mix[c], a.quickGetValue(r, c));
+ return ret;
+ }
+
+ private static CompressedMatrixBlock applyCompressedShuffle(CompressedMatrixBlock a, int[] mix) {
+ List<AColGroup> in = a.getColGroups();
+ List<AColGroup> out = new ArrayList<>();
+ for(AColGroup g : in)
+ out.add(moveCols(g, mix));
+ a.allocateColGroupList(out);
+ a.clearSoftReferenceToDecompressed();
+ return a;
+ }
+
+ private static AColGroup moveCols(AColGroup g, int[] mix) {
+ IColIndex gi = g.getColIndices();
+ int[] newIndexes = new int[gi.size()];
+
+ IIterate it = gi.iterator();
+ while(it.hasNext()) {
+ newIndexes[it.i()] = mix[it.v()];
+ it.next();
+ }
+
+ g = g.copyAndSet(ColIndexFactory.create(newIndexes));
+ g = g.sortColumnIndexes();
+ return g;
+ }
+}
diff --git a/src/test/java/org/apache/sysds/test/component/compress/mapping/MappingTests.java b/src/test/java/org/apache/sysds/test/component/compress/mapping/MappingTests.java
index c08cbc61f8..284876403e 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/mapping/MappingTests.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/mapping/MappingTests.java
@@ -35,7 +35,7 @@ import java.util.Random;
import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.sysds.runtime.compress.colgroup.AMapToDataGroup;
+import org.apache.sysds.runtime.compress.colgroup.IMapToDataGroup;
import org.apache.sysds.runtime.compress.colgroup.mapping.AMapToData;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToCharPByte;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory;
@@ -302,7 +302,7 @@ public class MappingTests {
try {
- AMapToData m2 = m.appendN(new AMapToDataGroup[] {//
+ AMapToData m2 = m.appendN(new IMapToDataGroup[] {//
new Holder(m), new Holder(m), new Holder(m)});
try {
assertEquals(m.size() * 3, m2.size());
@@ -326,7 +326,7 @@ public class MappingTests {
@Test
public void testAppendVSAppendN() {
final AMapToData m2 = m.append(m).append(m);
- final AMapToData m3 = m.appendN(new AMapToDataGroup[] {//
+ final AMapToData m3 = m.appendN(new IMapToDataGroup[] {//
new Holder(m), new Holder(m), new Holder(m)});
compare(m2, m3);
}
@@ -348,7 +348,7 @@ public class MappingTests {
LOG.error("Did not throw exception with: " + m);
}
- private static class Holder implements AMapToDataGroup {
+ private static class Holder implements IMapToDataGroup {
AMapToData d;
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AMapToDataGroup.java b/src/test/java/org/apache/sysds/test/component/frame/compress/FrameCompressTest.java
similarity index 73%
rename from src/main/java/org/apache/sysds/runtime/compress/colgroup/AMapToDataGroup.java
rename to src/test/java/org/apache/sysds/test/component/frame/compress/FrameCompressTest.java
index 50b65898a1..bdf6038550 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AMapToDataGroup.java
+++ b/src/test/java/org/apache/sysds/test/component/frame/compress/FrameCompressTest.java
@@ -17,10 +17,14 @@
* under the License.
*/
-package org.apache.sysds.runtime.compress.colgroup;
+package org.apache.sysds.test.component.frame.compress;
-import org.apache.sysds.runtime.compress.colgroup.mapping.AMapToData;
+import org.apache.sysds.runtime.frame.data.compress.FrameCompressionStatistics;
+import org.junit.Test;
-public interface AMapToDataGroup {
- public AMapToData getMapToData();
+public class FrameCompressTest {
+ @Test
+ public void testCompressionStatisticsConstruction() {
+ new FrameCompressionStatistics();
+ }
}