You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2023/05/22 09:20:57 UTC
[systemds] branch main updated: [SYSTEMDS-3572] CLA Morphing Interfaces
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 34ca59f9be [SYSTEMDS-3572] CLA Morphing Interfaces
34ca59f9be is described below
commit 34ca59f9be0d4d0a5bd833776b686bec7d91091e
Author: baunsgaard <ba...@tu-berlin.de>
AuthorDate: Sun May 21 18:19:50 2023 +0200
[SYSTEMDS-3572] CLA Morphing Interfaces
This commit adds multiple Interfaces for the different intended behavior
of the Column Groups to enable
- Morphing
- Re-compressing
- getting compression statistics from a column groups
- Combining column groups
Also contained is a change of the CLALib classes to be final and
their constructors private.
Closes #1830
---
.../runtime/compress/CompressedMatrixBlock.java | 84 +++++----------------
.../compress/CompressedMatrixBlockFactory.java | 22 ++++--
.../sysds/runtime/compress/colgroup/AColGroup.java | 31 +++++++-
.../runtime/compress/colgroup/ColGroupConst.java | 11 +++
.../runtime/compress/colgroup/ColGroupDDC.java | 35 +++++----
.../runtime/compress/colgroup/ColGroupDDCFOR.java | 16 +++-
.../runtime/compress/colgroup/ColGroupEmpty.java | 11 +++
.../colgroup/ColGroupLinearFunctional.java | 11 +++
.../runtime/compress/colgroup/ColGroupOLE.java | 14 +++-
.../runtime/compress/colgroup/ColGroupRLE.java | 11 +++
.../runtime/compress/colgroup/ColGroupSDC.java | 12 +++
.../runtime/compress/colgroup/ColGroupSDCFOR.java | 12 +++
.../compress/colgroup/ColGroupSDCSingle.java | 15 +++-
.../compress/colgroup/ColGroupSDCSingleZeros.java | 13 +++-
.../compress/colgroup/ColGroupSDCZeros.java | 11 +++
.../compress/colgroup/ColGroupUncompressed.java | 11 +++
.../runtime/compress/io/ReaderCompressed.java | 4 +-
.../sysds/runtime/compress/lib/CLALibAppend.java | 6 +-
.../runtime/compress/lib/CLALibBinaryCellOp.java | 7 +-
.../sysds/runtime/compress/lib/CLALibCMOps.java | 7 +-
...{CLALibSquash.java => CLALibCombineGroups.java} | 10 ++-
.../sysds/runtime/compress/lib/CLALibCompAgg.java | 6 +-
.../runtime/compress/lib/CLALibDecompress.java | 20 +++--
.../runtime/compress/lib/CLALibLeftMultBy.java | 6 +-
.../sysds/runtime/compress/lib/CLALibMMChain.java | 7 +-
.../runtime/compress/lib/CLALibMatrixMult.java | 6 +-
.../sysds/runtime/compress/lib/CLALibRexpand.java | 9 ++-
.../runtime/compress/lib/CLALibRightMultBy.java | 6 +-
.../sysds/runtime/compress/lib/CLALibScalar.java | 7 +-
.../sysds/runtime/compress/lib/CLALibSlice.java | 7 +-
.../sysds/runtime/compress/lib/CLALibSquash.java | 32 +++++++-
.../lib/{CLALibCombine.java => CLALibStack.java} | 7 +-
.../sysds/runtime/compress/lib/CLALibTSMM.java | 6 +-
.../runtime/compress/lib/CLALibTernaryOp.java | 88 ++++++++++++++++++++++
.../sysds/runtime/compress/lib/CLALibUnary.java | 26 +++++--
.../sysds/runtime/compress/lib/CLALibUtils.java | 4 +
.../sysds/runtime/matrix/data/MatrixBlock.java | 2 +-
.../compress/colgroup/ColGroupNegativeTests.java | 24 ++++++
.../component/compress/combine/CombineTest.java | 6 +-
39 files changed, 481 insertions(+), 142 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
index d1d4af99b0..3c4751e8c1 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
@@ -56,6 +56,7 @@ import org.apache.sysds.runtime.compress.lib.CLALibScalar;
import org.apache.sysds.runtime.compress.lib.CLALibSlice;
import org.apache.sysds.runtime.compress.lib.CLALibSquash;
import org.apache.sysds.runtime.compress.lib.CLALibTSMM;
+import org.apache.sysds.runtime.compress.lib.CLALibTernaryOp;
import org.apache.sysds.runtime.compress.lib.CLALibUnary;
import org.apache.sysds.runtime.compress.lib.CLALibUtils;
import org.apache.sysds.runtime.controlprogram.caching.MatrixObject.UpdateType;
@@ -63,9 +64,6 @@ import org.apache.sysds.runtime.controlprogram.parfor.stat.InfrastructureAnalyze
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.data.SparseRow;
-import org.apache.sysds.runtime.functionobjects.MinusMultiply;
-import org.apache.sysds.runtime.functionobjects.PlusMultiply;
-import org.apache.sysds.runtime.functionobjects.TernaryValueFunction.ValueFunctionWithConstant;
import org.apache.sysds.runtime.instructions.InstructionUtils;
import org.apache.sysds.runtime.instructions.cp.CM_COV_Object;
import org.apache.sysds.runtime.instructions.cp.ScalarObject;
@@ -73,7 +71,6 @@ import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue;
import org.apache.sysds.runtime.matrix.data.CTableMap;
import org.apache.sysds.runtime.matrix.data.IJV;
import org.apache.sysds.runtime.matrix.data.LibMatrixDatagen;
-import org.apache.sysds.runtime.matrix.data.LibMatrixTercell;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.matrix.data.MatrixIndexes;
import org.apache.sysds.runtime.matrix.data.MatrixValue;
@@ -575,21 +572,6 @@ public class CompressedMatrixBlock extends MatrixBlock {
return tmp.reorgOperations(op, ret, startRow, startColumn, length);
}
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("CompressedMatrixBlock:");
- sb.append("\nCols:" + getNumColumns() + " Rows:" + getNumRows() + " Overlapping: " + isOverlapping() + " nnz: "
- + nonZeros);
- if(_colGroups != null)
- for(AColGroup cg : _colGroups) {
- sb.append("\n" + cg);
- }
- else
- sb.append("\nEmptyColGroups");
- return sb.toString();
- }
-
public boolean isOverlapping() {
return _colGroups.size() != 1 && overlappingColGroups;
}
@@ -881,54 +863,7 @@ public class CompressedMatrixBlock extends MatrixBlock {
@Override
public MatrixBlock ternaryOperations(TernaryOperator op, MatrixBlock m2, MatrixBlock m3, MatrixBlock ret) {
-
- // prepare inputs
- final int r1 = getNumRows();
- final int r2 = m2.getNumRows();
- final int r3 = m3.getNumRows();
- final int c1 = getNumColumns();
- final int c2 = m2.getNumColumns();
- final int c3 = m3.getNumColumns();
- final boolean s1 = (r1 == 1 && c1 == 1);
- final boolean s2 = (r2 == 1 && c2 == 1);
- final boolean s3 = (r3 == 1 && c3 == 1);
- final double d1 = s1 ? quickGetValue(0, 0) : Double.NaN;
- final double d2 = s2 ? m2.quickGetValue(0, 0) : Double.NaN;
- final double d3 = s3 ? m3.quickGetValue(0, 0) : Double.NaN;
- final int m = Math.max(Math.max(r1, r2), r3);
- final int n = Math.max(Math.max(c1, c2), c3);
-
- ternaryOperationCheck(s1, s2, s3, m, r1, r2, r3, n, c1, c2, c3);
-
- final boolean PM_Or_MM = (op.fn instanceof PlusMultiply || op.fn instanceof MinusMultiply);
- if(PM_Or_MM && ((s2 && d2 == 0) || (s3 && d3 == 0))) {
- ret = new CompressedMatrixBlock();
- ret.copy(this);
- return ret;
- }
-
- if(m2 instanceof CompressedMatrixBlock)
- m2 = ((CompressedMatrixBlock) m2).getUncompressed("Ternary Operator arg2 " + op.fn.getClass().getSimpleName(),
- op.getNumThreads());
- if(m3 instanceof CompressedMatrixBlock)
- m3 = ((CompressedMatrixBlock) m3).getUncompressed("Ternary Operator arg3 " + op.fn.getClass().getSimpleName(),
- op.getNumThreads());
-
- if(s2 != s3 && (op.fn instanceof PlusMultiply || op.fn instanceof MinusMultiply)) {
- // SPECIAL CASE for sparse-dense combinations of common +* and -*
- BinaryOperator bop = ((ValueFunctionWithConstant) op.fn).setOp2Constant(s2 ? d2 : d3);
- bop.setNumThreads(op.getNumThreads());
- ret = CLALibBinaryCellOp.binaryOperationsRight(bop, this, s2 ? m3 : m2, ret);
- }
- else {
- final boolean sparseOutput = evalSparseFormatInMemory(m, n, (s1 ? m * n * (d1 != 0 ? 1 : 0) : getNonZeros()) +
- Math.min(s2 ? m * n : m2.getNonZeros(), s3 ? m * n : m3.getNonZeros()));
- ret.reset(m, n, sparseOutput);
- final MatrixBlock thisUncompressed = getUncompressed("Ternary Operation not supported");
- LibMatrixTercell.tercellOp(thisUncompressed, m2, m3, ret, op);
- ret.examSparsity();
- }
- return ret;
+ return CLALibTernaryOp.ternaryOperations(this, op, m2, m3, ret);
}
@Override
@@ -1293,4 +1228,19 @@ public class CompressedMatrixBlock extends MatrixBlock {
public void allocateAndResetSparseBlock(boolean clearNNZ, SparseBlock.Type stype) {
throw new DMLCompressionException("Invalid to allocate block on a compressed MatrixBlock");
}
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("CompressedMatrixBlock:");
+ sb.append("\nCols:" + getNumColumns() + " Rows:" + getNumRows() + " Overlapping: " + isOverlapping() + " nnz: "
+ + nonZeros);
+ if(_colGroups != null)
+ for(AColGroup cg : _colGroups) {
+ sb.append("\n" + cg);
+ }
+ else
+ sb.append("\nEmptyColGroups");
+ return sb.toString();
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java
index 63b8320480..423ec0c3f3 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java
@@ -267,7 +267,7 @@ public class CompressedMatrixBlockFactory {
private Pair<MatrixBlock, CompressionStatistics> compressMatrix() {
if(mb instanceof CompressedMatrixBlock) // Redundant compression
- return returnSelf();
+ return recompress((CompressedMatrixBlock) mb);
_stats.denseSize = MatrixBlock.estimateSizeInMemory(mb.getNumRows(), mb.getNumColumns(), 1.0);
_stats.originalSize = mb.getInMemorySize();
@@ -305,8 +305,8 @@ public class CompressedMatrixBlockFactory {
if(LOG.isTraceEnabled()) {
LOG.trace("Logging all individual columns estimated cost:");
for(CompressedSizeInfoColGroup g : compressionGroups.getInfo())
- LOG.trace(String.format("Cost: %8.0f Size: %16d %15s", costEstimator.getCost(g), g.getMinSize(),
- g.getColumns()));
+ LOG.trace(
+ String.format("Cost: %8.0f Size: %16d %15s", costEstimator.getCost(g), g.getMinSize(), g.getColumns()));
}
_stats.estimatedSizeCols = compressionGroups.memoryEstimate();
@@ -452,6 +452,17 @@ public class CompressedMatrixBlockFactory {
return new ImmutablePair<>(mb, _stats);
}
+ private Pair<MatrixBlock, CompressionStatistics> recompress(CompressedMatrixBlock cmb) {
+ LOG.debug("Recompressing an already compressed MatrixBlock");
+ LOG.error("Not Implemented Recompress yet");
+ return new ImmutablePair<>(cmb, null);
+ // _stats.originalSize = cmb.getInMemorySize();
+ // CompressedMatrixBlock combined = CLALibCombineGroups.combine(cmb, k);
+ // CompressedMatrixBlock squashed = CLALibSquash.squash(combined, k);
+ // _stats.compressedSize = squashed.getInMemorySize();
+ // return new ImmutablePair<>(squashed, _stats);
+ }
+
private void logPhase() {
setNextTimePhase(time.stop());
DMLCompressionStatistics.addCompressionTime(getLastTimePhase(), phase);
@@ -559,11 +570,6 @@ public class CompressedMatrixBlockFactory {
return new ImmutablePair<>(res, _stats);
}
- private Pair<MatrixBlock, CompressionStatistics> returnSelf() {
- LOG.info("MatrixBlock already compressed or is Empty");
- return new ImmutablePair<>(mb, null);
- }
-
private static String constructNrColumnString(List<AColGroup> cg) {
StringBuilder sb = new StringBuilder();
sb.append("[");
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java
index e6cd408309..c8e25f9920 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java
@@ -24,12 +24,14 @@ import java.io.IOException;
import java.io.Serializable;
import java.util.Collection;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex.SliceResult;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
+import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.instructions.cp.CM_COV_Object;
@@ -610,12 +612,37 @@ public abstract class AColGroup implements Serializable {
public abstract ICLAScheme getCompressionScheme();
/**
- * Clear variables that can be recomputed from the allocation of this columngroup.
+ * Clear variables that can be recomputed from the allocation of this column group.
*/
- public void clear(){
+ public void clear() {
// do nothing
}
+ /**
+ * Recompress this column group into a new column group.
+ *
+ * @return A new or the same column group depending on optimization goal.
+ */
+ public abstract AColGroup recompress();
+
+ /**
+ * Recompress this column group into a new column group of the given type.
+ *
+ * @param ct The compressionType that the column group should morph into
+ * @return A new column group
+ */
+ public AColGroup morph(CompressionType ct){
+ throw new NotImplementedException();
+ }
+
+ /**
+ * Get the compression info for this column group.
+ *
+ * @param nRow The number of rows in this column group.
+ * @return The compression info for this group.
+ */
+ public abstract CompressedSizeInfoColGroup getCompressionInfo(int nRow);
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java
index 2f94c3186a..c053fa9d6d 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java
@@ -33,6 +33,7 @@ import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
import org.apache.sysds.runtime.compress.colgroup.scheme.ConstScheme;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
+import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
import org.apache.sysds.runtime.compress.lib.CLALibLeftMultBy;
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
@@ -566,6 +567,16 @@ public class ColGroupConst extends ADictBasedColGroup {
return ConstScheme.create(this);
}
+ @Override
+ public AColGroup recompress(){
+ return this;
+ }
+
+ @Override
+ public CompressedSizeInfoColGroup getCompressionInfo(int nRow){
+ return new CompressedSizeInfoColGroup(_colIndexes, 1, nRow, CompressionType.CONST);
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
index 2d03f406db..06b10c1126 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
@@ -37,6 +37,7 @@ import org.apache.sysds.runtime.compress.colgroup.offset.AOffsetIterator;
import org.apache.sysds.runtime.compress.colgroup.scheme.DDCScheme;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
+import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.functionobjects.Builtin;
@@ -55,7 +56,7 @@ public class ColGroupDDC extends APreAgg implements AMapToDataGroup {
protected final AMapToData _data;
- private ColGroupDDC(IColIndex colIndexes, ADictionary dict, AMapToData data, int[] cachedCounts) {
+ private ColGroupDDC(IColIndex colIndexes, ADictionary dict, AMapToData data, int[] cachedCounts) {
super(colIndexes, dict, cachedCounts);
_data = data;
}
@@ -117,8 +118,8 @@ public class ColGroupDDC extends APreAgg implements AMapToDataGroup {
decompressToDenseBlockDenseDictGeneric(db, rl, ru, offR, offC, values);
}
- private final void decompressToDenseBlockDenseDictSingleColContiguous(DenseBlock db, int rl, int ru, int offR, int offC,
- double[] values) {
+ private final void decompressToDenseBlockDenseDictSingleColContiguous(DenseBlock db, int rl, int ru, int offR,
+ int offC, double[] values) {
final double[] c = db.values(0);
final int nCols = db.getDim(1);
final int colOff = _colIndexes.get(0) + offC;
@@ -128,12 +129,12 @@ public class ColGroupDDC extends APreAgg implements AMapToDataGroup {
}
@Override
- public AMapToData getMapToData(){
+ public AMapToData getMapToData() {
return _data;
}
- private final void decompressToDenseBlockDenseDictSingleColOutContiguous(DenseBlock db, int rl, int ru, int offR, int offC,
- double[] values) {
+ private final void decompressToDenseBlockDenseDictSingleColOutContiguous(DenseBlock db, int rl, int ru, int offR,
+ int offC, double[] values) {
final double[] c = db.values(0);
for(int i = rl, offT = rl + offR + _colIndexes.get(0) + offC; i < ru; i++, offT++)
c[offT] += values[_data.getIndex(i)];
@@ -152,7 +153,8 @@ public class ColGroupDDC extends APreAgg implements AMapToDataGroup {
}
}
- private final void decompressToDenseBlockDenseDictNoColOffset(DenseBlock db, int rl, int ru, int offR, double[] values) {
+ private final void decompressToDenseBlockDenseDictNoColOffset(DenseBlock db, int rl, int ru, int offR,
+ double[] values) {
final int nCol = _colIndexes.size();
final int colOut = db.getDim(1);
int off = (rl + offR) * colOut;
@@ -508,8 +510,7 @@ public class ColGroupDDC extends APreAgg implements AMapToDataGroup {
LOG.warn("Not same Dictionaries therefore not appending DDC\n" + _dict + "\n\n" + gDDC._dict);
}
else
- LOG.warn("Not same columns therefore not appending DDC\n" + _colIndexes + "\n\n"
- + g.getColIndices());
+ LOG.warn("Not same columns therefore not appending DDC\n" + _colIndexes + "\n\n" + g.getColIndices());
}
else
LOG.warn("Not DDC but " + g.getClass().getSimpleName() + ", therefore not appending DDC");
@@ -519,9 +520,8 @@ public class ColGroupDDC extends APreAgg implements AMapToDataGroup {
@Override
public AColGroup appendNInternal(AColGroup[] g) {
for(int i = 1; i < g.length; i++) {
- if(!_colIndexes.equals( g[i]._colIndexes)) {
- LOG.warn("Not same columns therefore not appending DDC\n" + _colIndexes + "\n\n"
- + g[i]._colIndexes);
+ if(!_colIndexes.equals(g[i]._colIndexes)) {
+ LOG.warn("Not same columns therefore not appending DDC\n" + _colIndexes + "\n\n" + g[i]._colIndexes);
return null;
}
@@ -540,12 +540,21 @@ public class ColGroupDDC extends APreAgg implements AMapToDataGroup {
return create(_colIndexes, _dict, nd, null);
}
-
@Override
public ICLAScheme getCompressionScheme() {
return DDCScheme.create(this);
}
+ @Override
+ public AColGroup recompress() {
+ return this;
+ }
+
+ @Override
+ public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
+ throw new NotImplementedException();
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java
index b59c72be98..d05b663f85 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java
@@ -35,6 +35,7 @@ import org.apache.sysds.runtime.compress.colgroup.mapping.AMapToData;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
+import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
import org.apache.sysds.runtime.functionobjects.Builtin;
import org.apache.sysds.runtime.functionobjects.Divide;
import org.apache.sysds.runtime.functionobjects.Minus;
@@ -59,7 +60,8 @@ public class ColGroupDDCFOR extends AMorphingMMColGroup {
/** Reference values in this column group */
protected final double[] _reference;
- private ColGroupDDCFOR(IColIndex colIndexes, ADictionary dict, double[] reference, AMapToData data, int[] cachedCounts) {
+ private ColGroupDDCFOR(IColIndex colIndexes, ADictionary dict, double[] reference, AMapToData data,
+ int[] cachedCounts) {
super(colIndexes, dict, cachedCounts);
_data = data;
_reference = reference;
@@ -439,7 +441,7 @@ public class ColGroupDDCFOR extends AMorphingMMColGroup {
public AColGroup append(AColGroup g) {
if(g instanceof ColGroupDDCFOR && g.getColIndices().equals(_colIndexes)) {
ColGroupDDCFOR gDDC = (ColGroupDDCFOR) g;
- if(Arrays.equals(_reference , gDDC._reference) && gDDC._dict.equals(_dict)){
+ if(Arrays.equals(_reference, gDDC._reference) && gDDC._dict.equals(_dict)) {
AMapToData nd = _data.append(gDDC._data);
return create(_colIndexes, _dict, nd, null, _reference);
}
@@ -457,6 +459,16 @@ public class ColGroupDDCFOR extends AMorphingMMColGroup {
return null;
}
+ @Override
+ public AColGroup recompress() {
+ return this;
+ }
+
+ @Override
+ public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
+ throw new NotImplementedException();
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java
index c838850743..cd788a1318 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java
@@ -31,6 +31,7 @@ import org.apache.sysds.runtime.compress.colgroup.indexes.IIterate;
import org.apache.sysds.runtime.compress.colgroup.scheme.EmptyScheme;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
+import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.functionobjects.Builtin;
@@ -338,4 +339,14 @@ public class ColGroupEmpty extends AColGroupCompressed {
public ICLAScheme getCompressionScheme() {
return EmptyScheme.create(this);
}
+
+ @Override
+ public AColGroup recompress() {
+ return this;
+ }
+
+ @Override
+ public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
+ return new CompressedSizeInfoColGroup(_colIndexes, 0, nRow, CompressionType.CONST);
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java
index 8c899ec609..452d5e1f07 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java
@@ -30,6 +30,7 @@ import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory;
import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
+import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
import org.apache.sysds.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
@@ -682,4 +683,14 @@ public class ColGroupLinearFunctional extends AColGroupCompressed {
return null;
}
+ @Override
+ public AColGroup recompress(){
+ return this;
+ }
+
+ @Override
+ public CompressedSizeInfoColGroup getCompressionInfo(int nRow){
+ throw new NotImplementedException("Not Implemented Compressed SizeInfo for Linear col group");
+ }
+
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java
index 9c7ca53ed5..961782522c 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java
@@ -33,6 +33,7 @@ import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory;
import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
+import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.functionobjects.Builtin;
@@ -48,8 +49,8 @@ import org.apache.sysds.runtime.matrix.operators.UnaryOperator;
public class ColGroupOLE extends AColGroupOffset {
private static final long serialVersionUID = 5723227906925121066L;
- private ColGroupOLE(IColIndex colIndices, int numRows, boolean zero, ADictionary dict, char[] bitmaps, int[] bitmapOffs,
- int[] counts) {
+ private ColGroupOLE(IColIndex colIndices, int numRows, boolean zero, ADictionary dict, char[] bitmaps,
+ int[] bitmapOffs, int[] counts) {
super(colIndices, numRows, zero, dict, bitmapOffs, bitmaps, counts);
}
@@ -673,4 +674,13 @@ public class ColGroupOLE extends AColGroupOffset {
return null;
}
+ @Override
+ public AColGroup recompress() {
+ return this;
+ }
+
+ @Override
+ public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
+ throw new NotImplementedException();
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java
index ba8e5f9203..909808ee89 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java
@@ -36,6 +36,7 @@ import org.apache.sysds.runtime.compress.colgroup.offset.AIterator;
import org.apache.sysds.runtime.compress.colgroup.offset.AOffsetIterator;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
+import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.functionobjects.Builtin;
@@ -985,6 +986,16 @@ public class ColGroupRLE extends AColGroupOffset {
return null;
}
+ @Override
+ public AColGroup recompress() {
+ return this;
+ }
+
+ @Override
+ public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
+ throw new NotImplementedException();
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
index 1e45de8924..500986bd23 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
@@ -24,6 +24,7 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.compress.DMLCompressionException;
import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
@@ -40,6 +41,7 @@ import org.apache.sysds.runtime.compress.colgroup.offset.AOffset.OffsetSliceInfo
import org.apache.sysds.runtime.compress.colgroup.offset.OffsetFactory;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
+import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
import org.apache.sysds.runtime.functionobjects.Builtin;
import org.apache.sysds.runtime.instructions.cp.CM_COV_Object;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
@@ -626,6 +628,16 @@ public class ColGroupSDC extends ASDC implements AMapToDataGroup {
return null;
}
+ @Override
+ public AColGroup recompress() {
+ return this;
+ }
+
+ @Override
+ public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
+ throw new NotImplementedException();
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
index 01bf7f4a55..2969d9fc04 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
@@ -24,6 +24,7 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.sysds.runtime.compress.DMLCompressionException;
import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
import org.apache.sysds.runtime.compress.colgroup.dictionary.Dictionary;
@@ -38,6 +39,7 @@ import org.apache.sysds.runtime.compress.colgroup.offset.AOffset.OffsetSliceInfo
import org.apache.sysds.runtime.compress.colgroup.offset.OffsetFactory;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
+import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
import org.apache.sysds.runtime.functionobjects.Builtin;
import org.apache.sysds.runtime.functionobjects.Divide;
import org.apache.sysds.runtime.functionobjects.Minus;
@@ -488,6 +490,16 @@ public class ColGroupSDCFOR extends ASDC implements AMapToDataGroup {
return null;
}
+ @Override
+ public AColGroup recompress() {
+ return this;
+ }
+
+ @Override
+ public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
+ throw new NotImplementedException();
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java
index 94c1f889c0..79e70ad52e 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java
@@ -38,6 +38,7 @@ import org.apache.sysds.runtime.compress.colgroup.offset.AOffsetIterator;
import org.apache.sysds.runtime.compress.colgroup.offset.OffsetFactory;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
+import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
import org.apache.sysds.runtime.functionobjects.Builtin;
import org.apache.sysds.runtime.instructions.cp.CM_COV_Object;
import org.apache.sysds.runtime.matrix.operators.BinaryOperator;
@@ -59,8 +60,8 @@ public class ColGroupSDCSingle extends ASDC {
/** The default value stored in this column group */
protected final double[] _defaultTuple;
- private ColGroupSDCSingle(IColIndex colIndices, int numRows, ADictionary dict, double[] defaultTuple, AOffset offsets,
- int[] cachedCounts) {
+ private ColGroupSDCSingle(IColIndex colIndices, int numRows, ADictionary dict, double[] defaultTuple,
+ AOffset offsets, int[] cachedCounts) {
super(colIndices, numRows, dict == null ? Dictionary.createNoCheck(new double[colIndices.size()]) : dict, offsets,
cachedCounts);
_defaultTuple = defaultTuple;
@@ -589,6 +590,16 @@ public class ColGroupSDCSingle extends ASDC {
return null;
}
+ @Override
+ public AColGroup recompress() {
+ return this;
+ }
+
+ @Override
+ public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
+ throw new NotImplementedException();
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java
index 954122620c..565874d83a 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java
@@ -39,6 +39,7 @@ import org.apache.sysds.runtime.compress.colgroup.offset.AOffsetIterator;
import org.apache.sysds.runtime.compress.colgroup.offset.OffsetFactory;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
+import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.functionobjects.Builtin;
@@ -97,7 +98,7 @@ public class ColGroupSDCSingleZeros extends ASDCZero {
return;
else if(it.value() >= ru)
return;
- // _indexes.cacheIterator(it, ru);
+ // _indexes.cacheIterator(it, ru);
else {
decompressToDenseBlockDenseDictionaryWithProvidedIterator(db, rl, ru, offR, offC, values, it);
// _indexes.cacheIterator(it, ru);
@@ -854,6 +855,16 @@ public class ColGroupSDCSingleZeros extends ASDCZero {
return null;
}
+ @Override
+ public AColGroup recompress() {
+ return this;
+ }
+
+ @Override
+ public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
+ throw new NotImplementedException();
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java
index 2e99179eaf..0931526051 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java
@@ -24,6 +24,7 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.compress.DMLCompressionException;
import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
@@ -40,6 +41,7 @@ import org.apache.sysds.runtime.compress.colgroup.offset.AOffset.OffsetSliceInfo
import org.apache.sysds.runtime.compress.colgroup.offset.OffsetFactory;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
+import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.functionobjects.Builtin;
@@ -793,6 +795,15 @@ public class ColGroupSDCZeros extends ASDCZero implements AMapToDataGroup {
}
@Override
+ public AColGroup recompress() {
+ return this;
+ }
+
+ @Override
+ public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
+ throw new NotImplementedException();
+ }
+
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(super.toString());
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
index 2c4e9e4822..1ce1e1963b 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
@@ -34,6 +34,7 @@ import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory;
import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme;
import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
+import org.apache.sysds.runtime.compress.estim.CompressedSizeInfoColGroup;
import org.apache.sysds.runtime.compress.utils.Util;
import org.apache.sysds.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
import org.apache.sysds.runtime.data.DenseBlock;
@@ -804,6 +805,16 @@ public class ColGroupUncompressed extends AColGroup {
return null;
}
+ @Override
+ public AColGroup recompress() {
+ return this;
+ }
+
+ @Override
+ public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
+ throw new NotImplementedException();
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/io/ReaderCompressed.java b/src/main/java/org/apache/sysds/runtime/compress/io/ReaderCompressed.java
index 22efcbafd1..f1f45abc98 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/io/ReaderCompressed.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/io/ReaderCompressed.java
@@ -35,7 +35,7 @@ import org.apache.sysds.hops.OptimizerUtils;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory;
-import org.apache.sysds.runtime.compress.lib.CLALibCombine;
+import org.apache.sysds.runtime.compress.lib.CLALibStack;
import org.apache.sysds.runtime.io.IOUtilFunctions;
import org.apache.sysds.runtime.io.MatrixReader;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
@@ -82,7 +82,7 @@ public final class ReaderCompressed extends MatrixReader {
if(data.size() == 1)
return data.entrySet().iterator().next().getValue();
else
- return CLALibCombine.combine(data, OptimizerUtils.getParallelTextWriteParallelism());
+ return CLALibStack.combine(data, OptimizerUtils.getParallelTextWriteParallelism());
}
private static void read(Path path, JobConf job, Map<MatrixIndexes, MatrixBlock> data) throws IOException {
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java
index c6b71e3fdc..4d2ce93919 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java
@@ -33,7 +33,11 @@ import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory;
import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
-public class CLALibAppend {
+public final class CLALibAppend {
+
+ private CLALibAppend(){
+ // private constructor.
+ }
private static final Log LOG = LogFactory.getLog(CLALibAppend.class.getName());
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java
index f530247032..54bd51e60f 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java
@@ -59,10 +59,13 @@ import org.apache.sysds.runtime.matrix.operators.RightScalarOperator;
import org.apache.sysds.runtime.matrix.operators.ScalarOperator;
import org.apache.sysds.runtime.util.CommonThreadPool;
-public class CLALibBinaryCellOp {
-
+public final class CLALibBinaryCellOp {
private static final Log LOG = LogFactory.getLog(CLALibBinaryCellOp.class.getName());
+ private CLALibBinaryCellOp() {
+ // empty private constructor.
+ }
+
public static MatrixBlock binaryOperationsRight(BinaryOperator op, CompressedMatrixBlock m1, MatrixBlock that,
MatrixBlock result) {
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCMOps.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCMOps.java
index ec2cfae024..3e77aaefb9 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCMOps.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCMOps.java
@@ -28,7 +28,12 @@ import org.apache.sysds.runtime.matrix.data.LibMatrixAgg;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.matrix.operators.CMOperator;
-public class CLALibCMOps {
+public final class CLALibCMOps {
+
+ private CLALibCMOps() {
+ // private constructor
+ }
+
public static CM_COV_Object centralMoment(CompressedMatrixBlock cmb, CMOperator op) {
MatrixBlock.checkCMOperations(cmb, op);
if(cmb.isEmpty())
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSquash.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombineGroups.java
similarity index 84%
copy from src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSquash.java
copy to src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombineGroups.java
index fec3e800e7..cc9d25c9ce 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSquash.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombineGroups.java
@@ -22,8 +22,14 @@ package org.apache.sysds.runtime.compress.lib;
import org.apache.commons.lang.NotImplementedException;
import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
-public class CLALibSquash {
- public static CompressedMatrixBlock squash(CompressedMatrixBlock m, int k) {
+public final class CLALibCombineGroups {
+
+ private CLALibCombineGroups() {
+ // private constructor
+ }
+
+ public static CompressedMatrixBlock combine(CompressedMatrixBlock cmb, int k) {
throw new NotImplementedException();
}
+
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCompAgg.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCompAgg.java
index 4b90d48ca5..d12abb1167 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCompAgg.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCompAgg.java
@@ -67,10 +67,14 @@ import org.apache.sysds.runtime.matrix.operators.BinaryOperator;
import org.apache.sysds.runtime.util.CommonThreadPool;
import org.apache.sysds.utils.DMLCompressionStatistics;
-public class CLALibCompAgg {
+public final class CLALibCompAgg {
private static final Log LOG = LogFactory.getLog(CLALibCompAgg.class.getName());
private static final long MIN_PAR_AGG_THRESHOLD = 8 * 1024;
+ private CLALibCompAgg(){
+ // private constructor
+ }
+
public static MatrixBlock aggregateUnary(CompressedMatrixBlock inputMatrix, MatrixBlock result,
AggregateUnaryOperator op, int blen, MatrixIndexes indexesIn, boolean inCP) {
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibDecompress.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibDecompress.java
index f92533466b..eee0eced01 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibDecompress.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibDecompress.java
@@ -44,9 +44,13 @@ import org.apache.sysds.utils.DMLCompressionStatistics;
/**
* Library to decompress a list of column groups into a matrix.
*/
-public class CLALibDecompress {
+public final class CLALibDecompress {
private static final Log LOG = LogFactory.getLog(CLALibDecompress.class.getName());
+ private CLALibDecompress() {
+ // private constructor
+ }
+
public static MatrixBlock decompress(CompressedMatrixBlock cmb, int k) {
Timing time = new Timing(true);
MatrixBlock ret = decompressExecute(cmb, k);
@@ -59,7 +63,8 @@ public class CLALibDecompress {
return ret;
}
- public static void decompressTo(CompressedMatrixBlock cmb, MatrixBlock ret, int rowOffset, int colOffset, int k, boolean countNNz) {
+ public static void decompressTo(CompressedMatrixBlock cmb, MatrixBlock ret, int rowOffset, int colOffset, int k,
+ boolean countNNz) {
Timing time = new Timing(true);
if(cmb.getNumColumns() + colOffset > ret.getNumColumns() || cmb.getNumRows() + rowOffset > ret.getNumRows()) {
LOG.warn(
@@ -185,7 +190,7 @@ public class CLALibDecompress {
else if(ret.isInSparseFormat()) {
decompressSparseMultiThread(ret, filteredGroups, nRows, blklen, k);
}
- else{
+ else {
decompressDenseMultiThread(ret, filteredGroups, nRows, blklen, constV, eps, k, overlapping);
}
ret.recomputeNonZeros();
@@ -249,7 +254,8 @@ public class CLALibDecompress {
}
}
- protected static void decompressDenseMultiThread(MatrixBlock ret, List<AColGroup> groups, double[] constV, int k, boolean overlapping) {
+ protected static void decompressDenseMultiThread(MatrixBlock ret, List<AColGroup> groups, double[] constV, int k,
+ boolean overlapping) {
final int nRows = ret.getNumRows();
final double eps = getEps(constV);
final int blklen = Math.max(nRows / k, 512);
@@ -268,11 +274,11 @@ public class CLALibDecompress {
final ExecutorService pool = CommonThreadPool.get(k);
try {
final ArrayList<Callable<Long>> tasks = new ArrayList<>();
- if(overlapping || constV != null){
+ if(overlapping || constV != null) {
for(int i = 0; i < rlen; i += blklen)
tasks.add(new DecompressDenseTask(filteredGroups, ret, eps, i, Math.min(i + blklen, rlen), constV));
}
- else{
+ else {
for(int i = 0; i < rlen; i += blklen)
for(AColGroup g : filteredGroups)
tasks.add(new DecompressDenseSingleColTask(g, ret, eps, i, Math.min(i + blklen, rlen), null));
@@ -402,7 +408,7 @@ public class CLALibDecompress {
for(int b = _rl; b < _ru; b += _blklen) {
final int e = Math.min(b + _blklen, _ru);
// for(AColGroup grp : _colGroups)
- _grp.decompressToDenseBlock(_ret.getDenseBlock(), b, e);
+ _grp.decompressToDenseBlock(_ret.getDenseBlock(), b, e);
if(_constV != null)
addVector(_ret, _constV, _eps, b, e);
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibLeftMultBy.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibLeftMultBy.java
index b50c9b5c0f..731d8f5ecf 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibLeftMultBy.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibLeftMultBy.java
@@ -42,9 +42,13 @@ import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.matrix.operators.BinaryOperator;
import org.apache.sysds.runtime.util.CommonThreadPool;
-public class CLALibLeftMultBy {
+public final class CLALibLeftMultBy {
private static final Log LOG = LogFactory.getLog(CLALibLeftMultBy.class.getName());
+ private CLALibLeftMultBy(){
+ // private constructor
+ }
+
/**
* Left multiplication with a CompressedMatrixBlock on the right following the equation:
*
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java
index dced83556a..bc164a5e91 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMMChain.java
@@ -35,10 +35,13 @@ import org.apache.sysds.runtime.matrix.data.LibMatrixReorg;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.matrix.operators.BinaryOperator;
-public interface CLALibMMChain {
-
+public final class CLALibMMChain {
static final Log LOG = LogFactory.getLog(CLALibMMChain.class.getName());
+ private CLALibMMChain() {
+ // private constructor
+ }
+
public static MatrixBlock mmChain(CompressedMatrixBlock x, MatrixBlock v, MatrixBlock w, MatrixBlock out,
ChainType ctype, int k) {
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMatrixMult.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMatrixMult.java
index 9fee706143..185b684f65 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMatrixMult.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibMatrixMult.java
@@ -28,9 +28,13 @@ import org.apache.sysds.runtime.matrix.data.LibMatrixReorg;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.matrix.operators.ReorgOperator;
-public class CLALibMatrixMult {
+public final class CLALibMatrixMult {
private static final Log LOG = LogFactory.getLog(CLALibMatrixMult.class.getName());
+ private CLALibMatrixMult(){
+ // private constructor
+ }
+
public static MatrixBlock matrixMult(MatrixBlock m1, MatrixBlock m2, MatrixBlock ret, int k) {
return matrixMultiply(m1, m2, ret, k, false, false);
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRexpand.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRexpand.java
index c9f8b496ad..01b0ce14db 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRexpand.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRexpand.java
@@ -19,14 +19,19 @@
package org.apache.sysds.runtime.compress.lib;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
import org.apache.sysds.runtime.matrix.data.LibMatrixReorg;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.util.UtilFunctions;
-public class CLALibRexpand {
+public final class CLALibRexpand {
+ protected static final Log LOG = LogFactory.getLog(CLALibRexpand.class.getName());
- // private static final Log LOG = LogFactory.getLog(CLALibReExpand.class.getName());
+ private CLALibRexpand(){
+ // private constructor
+ }
public static MatrixBlock rexpand(CompressedMatrixBlock in, MatrixBlock ret, double max, boolean rows, boolean cast,
boolean ignore, int k) {
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java
index 79e5d7d725..39468b0cab 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java
@@ -46,9 +46,13 @@ import org.apache.sysds.runtime.matrix.operators.BinaryOperator;
import org.apache.sysds.runtime.util.CommonThreadPool;
import org.apache.sysds.utils.DMLCompressionStatistics;
-public class CLALibRightMultBy {
+public final class CLALibRightMultBy {
private static final Log LOG = LogFactory.getLog(CLALibRightMultBy.class.getName());
+ private CLALibRightMultBy(){
+ // private constructor
+ }
+
public static MatrixBlock rightMultByMatrix(CompressedMatrixBlock m1, MatrixBlock m2, MatrixBlock ret, int k) {
final boolean allowOverlap = ConfigurationManager.getDMLConfig()
.getBooleanValue(DMLConfig.COMPRESSED_OVERLAPPING);
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibScalar.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibScalar.java
index 1540afe059..0860e0f433 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibScalar.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibScalar.java
@@ -48,11 +48,14 @@ import org.apache.sysds.runtime.matrix.operators.RightScalarOperator;
import org.apache.sysds.runtime.matrix.operators.ScalarOperator;
import org.apache.sysds.runtime.util.CommonThreadPool;
-public class CLALibScalar {
-
+public final class CLALibScalar {
private static final Log LOG = LogFactory.getLog(CLALibScalar.class.getName());
private static final int MINIMUM_PARALLEL_SIZE = 8096;
+ private CLALibScalar() {
+ // private constructor
+ }
+
public static MatrixBlock scalarOperations(ScalarOperator sop, CompressedMatrixBlock m1, MatrixValue result) {
if(isInvalidForCompressedOutput(m1, sop)) {
LOG.warn("scalar overlapping not supported for op: " + sop.fn);
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSlice.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSlice.java
index 4b4ab0e429..2fac4eebd2 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSlice.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSlice.java
@@ -35,10 +35,13 @@ import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.util.CommonThreadPool;
-public class CLALibSlice {
-
+public final class CLALibSlice {
protected static final Log LOG = LogFactory.getLog(CLALibSlice.class.getName());
+ private CLALibSlice() {
+ // private constructor
+ }
+
/**
* Slice blocks of compressed matrices from the compressed representation.
*
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSquash.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSquash.java
index fec3e800e7..f861fac454 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSquash.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSquash.java
@@ -19,11 +19,37 @@
package org.apache.sysds.runtime.compress.lib;
-import org.apache.commons.lang.NotImplementedException;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
+import org.apache.sysds.runtime.compress.colgroup.AColGroup;
+
+/**
+ * Squash or recompress is processing each column group and trying to find a better compressed representation for each.
+ */
+public final class CLALibSquash {
-public class CLALibSquash {
+ private CLALibSquash() {
+ // private constructor
+ }
+
+ /**
+ * Squash or recompress is process each column group in the given Compressed Matrix Block and tries to recompress
+ * each column.
+ *
+ * @param m The input compressed matrix
+ * @param k The parallelization degree allowed in this process
+ * @return A replaced Compressed Matrix Block, note the old block is also modified
+ */
public static CompressedMatrixBlock squash(CompressedMatrixBlock m, int k) {
- throw new NotImplementedException();
+ List<AColGroup> before = m.getColGroups();
+ List<AColGroup> groups = new ArrayList<>(before.size());
+
+ for(AColGroup g : before)
+ groups.add(g.recompress());
+
+ m.allocateColGroupList(groups);
+ return m;
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombine.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibStack.java
similarity index 98%
rename from src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombine.java
rename to src/main/java/org/apache/sysds/runtime/compress/lib/CLALibStack.java
index 91238a2927..a1f11573fe 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombine.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibStack.java
@@ -39,9 +39,12 @@ import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.matrix.data.MatrixIndexes;
import org.apache.sysds.runtime.util.CommonThreadPool;
-public class CLALibCombine {
+public final class CLALibStack {
+ protected static final Log LOG = LogFactory.getLog(CLALibStack.class.getName());
- protected static final Log LOG = LogFactory.getLog(CLALibCombine.class.getName());
+ private CLALibStack() {
+ // private constructor
+ }
/**
* Combine the map of index matrix blocks into a single MatrixBlock.
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibTSMM.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibTSMM.java
index ecaaf054fc..bf99adc34b 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibTSMM.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibTSMM.java
@@ -35,9 +35,13 @@ import org.apache.sysds.runtime.matrix.data.LibMatrixMult;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.util.CommonThreadPool;
-public class CLALibTSMM {
+public final class CLALibTSMM {
private static final Log LOG = LogFactory.getLog(CLALibTSMM.class.getName());
+ private CLALibTSMM() {
+ // private constructor
+ }
+
/**
* Self left Matrix multiplication (tsmm)
*
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibTernaryOp.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibTernaryOp.java
new file mode 100644
index 0000000000..ac1ddd814a
--- /dev/null
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibTernaryOp.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.compress.lib;
+
+import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
+import org.apache.sysds.runtime.functionobjects.MinusMultiply;
+import org.apache.sysds.runtime.functionobjects.PlusMultiply;
+import org.apache.sysds.runtime.functionobjects.TernaryValueFunction.ValueFunctionWithConstant;
+import org.apache.sysds.runtime.matrix.data.LibMatrixTercell;
+import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+import org.apache.sysds.runtime.matrix.operators.BinaryOperator;
+import org.apache.sysds.runtime.matrix.operators.TernaryOperator;
+
+public final class CLALibTernaryOp {
+ private CLALibTernaryOp() {
+ // private constructor
+ }
+
+ public static MatrixBlock ternaryOperations(CompressedMatrixBlock m1, TernaryOperator op, MatrixBlock m2,
+ MatrixBlock m3, MatrixBlock ret) {
+
+ // prepare inputs
+ final int r1 = m1.getNumRows();
+ final int r2 = m2.getNumRows();
+ final int r3 = m3.getNumRows();
+ final int c1 = m1.getNumColumns();
+ final int c2 = m2.getNumColumns();
+ final int c3 = m3.getNumColumns();
+ final boolean s1 = (r1 == 1 && c1 == 1);
+ final boolean s2 = (r2 == 1 && c2 == 1);
+ final boolean s3 = (r3 == 1 && c3 == 1);
+ final double d1 = s1 ? m1.quickGetValue(0, 0) : Double.NaN;
+ final double d2 = s2 ? m2.quickGetValue(0, 0) : Double.NaN;
+ final double d3 = s3 ? m3.quickGetValue(0, 0) : Double.NaN;
+ final int m = Math.max(Math.max(r1, r2), r3);
+ final int n = Math.max(Math.max(c1, c2), c3);
+
+ MatrixBlock.ternaryOperationCheck(s1, s2, s3, m, r1, r2, r3, n, c1, c2, c3);
+
+ final boolean PM_Or_MM = (op.fn instanceof PlusMultiply || op.fn instanceof MinusMultiply);
+ if(PM_Or_MM && ((s2 && d2 == 0) || (s3 && d3 == 0))) {
+ ret = new CompressedMatrixBlock();
+ ret.copy(m1);
+ return ret;
+ }
+
+ if(m2 instanceof CompressedMatrixBlock)
+ m2 = ((CompressedMatrixBlock) m2).getUncompressed("Ternary Operator arg2 " + op.fn.getClass().getSimpleName(),
+ op.getNumThreads());
+ if(m3 instanceof CompressedMatrixBlock)
+ m3 = ((CompressedMatrixBlock) m3).getUncompressed("Ternary Operator arg3 " + op.fn.getClass().getSimpleName(),
+ op.getNumThreads());
+
+ if(s2 != s3 && (op.fn instanceof PlusMultiply || op.fn instanceof MinusMultiply)) {
+ // SPECIAL CASE for sparse-dense combinations of common +* and -*
+ BinaryOperator bop = ((ValueFunctionWithConstant) op.fn).setOp2Constant(s2 ? d2 : d3);
+ bop.setNumThreads(op.getNumThreads());
+ ret = CLALibBinaryCellOp.binaryOperationsRight(bop, m1, s2 ? m3 : m2, ret);
+ }
+ else {
+ final boolean sparseOutput = MatrixBlock.evalSparseFormatInMemory(m, n, (s1 ? m * n * (d1 != 0 ? 1 : 0) : m1.getNonZeros()) +
+ Math.min(s2 ? m * n : m2.getNonZeros(), s3 ? m * n : m3.getNonZeros()));
+ ret.reset(m, n, sparseOutput);
+ final MatrixBlock thisUncompressed = m1.getUncompressed("Ternary Operation not supported");
+ LibMatrixTercell.tercellOp(thisUncompressed, m2, m3, ret, op);
+ ret.examSparsity();
+ }
+ return ret;
+ }
+
+}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibUnary.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibUnary.java
index dee1b9022b..f858f15b74 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibUnary.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibUnary.java
@@ -21,6 +21,8 @@ package org.apache.sysds.runtime.compress.lib;
import java.util.ArrayList;
import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
import org.apache.sysds.runtime.compress.colgroup.AColGroup;
import org.apache.sysds.runtime.functionobjects.Builtin;
@@ -30,8 +32,12 @@ import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.matrix.data.MatrixValue;
import org.apache.sysds.runtime.matrix.operators.UnaryOperator;
-public class CLALibUnary {
- // private static final Log LOG = LogFactory.getLog(CLALibUnary.class.getName());
+public final class CLALibUnary {
+ protected static final Log LOG = LogFactory.getLog(CLALibUnary.class.getName());
+
+ private CLALibUnary() {
+ // private constructor
+ }
public static MatrixBlock unaryOperations(CompressedMatrixBlock m, UnaryOperator op, MatrixValue result) {
final boolean overlapping = m.isOverlapping();
@@ -44,17 +50,22 @@ public class CLALibUnary {
// when in overlapping state it is guaranteed that there is no infinites, NA, or NANs.
if(Builtin.isBuiltinCode(op.fn, BuiltinCode.ISINF, BuiltinCode.ISNA, BuiltinCode.ISNAN))
return new MatrixBlock(r, c, 0);
- if(op.fn instanceof Builtin)
- return m.getUncompressed("Unary Op not supported Overlapping builtin: " + ((Builtin)(op.fn)).getBuiltinCode(), op.getNumThreads()).unaryOperations(op, null);
- else
- return m.getUncompressed("Unary Op not supported Overlapping: " + op.fn.getClass().getSimpleName(), op.getNumThreads()).unaryOperations(op, null);
+ if(op.fn instanceof Builtin) {
+ String message = "Unary Op not supported Overlapping builtin: " + ((Builtin) (op.fn)).getBuiltinCode();
+ return m.getUncompressed(message, op.getNumThreads()).unaryOperations(op, null);
+ }
+ else {
+ String message = "Unary Op not supported Overlapping: " + op.fn.getClass().getSimpleName();
+ return m.getUncompressed(message, op.getNumThreads()).unaryOperations(op, null);
+ }
}
else if(Builtin.isBuiltinCode(op.fn, BuiltinCode.ISINF, BuiltinCode.ISNAN, BuiltinCode.ISNA) &&
!m.containsValue(op.getPattern()))
return new MatrixBlock(r, c, 0); // avoid unnecessary allocation
else if(LibMatrixAgg.isSupportedUnaryOperator(op)) {
+ String message = "Unary Op not supported: " + op.fn.getClass().getSimpleName();
// e.g., cumsum/cumprod/cummin/cumax/cumsumprod
- return m.getUncompressed("Unary Op not supported: " + op.fn.getClass().getSimpleName(), op.getNumThreads()).unaryOperations(op, null);
+ return m.getUncompressed(message, op.getNumThreads()).unaryOperations(op, null);
}
else {
@@ -68,6 +79,5 @@ public class CLALibUnary {
ret.recomputeNonZeros();
return ret;
}
-
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibUtils.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibUtils.java
index 1dfe2a0575..06da63d283 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibUtils.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibUtils.java
@@ -41,6 +41,10 @@ import org.apache.sysds.runtime.compress.colgroup.indexes.IIterate;
public final class CLALibUtils {
protected static final Log LOG = LogFactory.getLog(CLALibUtils.class.getName());
+ private CLALibUtils() {
+ // private constructor
+ }
+
/**
* Combine all column groups that are constant types, this include empty and const.
*
diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
index 2ae883ee41..9d79f7f971 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
@@ -2987,7 +2987,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock<MatrixBlock>,
return ret;
}
- protected static void ternaryOperationCheck(boolean s1, boolean s2, boolean s3, int m, int r1, int r2, int r3, int n, int c1, int c2, int c3){
+ public static void ternaryOperationCheck(boolean s1, boolean s2, boolean s3, int m, int r1, int r2, int r3, int n, int c1, int c2, int c3){
//error handling
if( (!s1 && (r1 != m || c1 != n))
|| (!s2 && (r2 != m || c2 != n))
diff --git a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java
index 7ab9f8e019..c6d1a4ddcd 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java
@@ -375,6 +375,18 @@ public class ColGroupNegativeTests {
// TODO Auto-generated method stub
return null;
}
+
+ @Override
+ public AColGroup recompress() {
+ // TODO Auto-generated method stub
+ throw new UnsupportedOperationException("Unimplemented method 'recompress'");
+ }
+
+ @Override
+ public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
+ // TODO Auto-generated method stub
+ throw new UnsupportedOperationException("Unimplemented method 'getCompressionInfo'");
+ }
}
private class FakeDictBasedColGroup extends ADictBasedColGroup {
@@ -601,5 +613,17 @@ public class ColGroupNegativeTests {
// TODO Auto-generated method stub
return null;
}
+
+ @Override
+ public AColGroup recompress() {
+ // TODO Auto-generated method stub
+ throw new UnsupportedOperationException("Unimplemented method 'recompress'");
+ }
+
+ @Override
+ public CompressedSizeInfoColGroup getCompressionInfo(int nRow) {
+ // TODO Auto-generated method stub
+ throw new UnsupportedOperationException("Unimplemented method 'getCompressionInfo'");
+ }
}
}
diff --git a/src/test/java/org/apache/sysds/test/component/compress/combine/CombineTest.java b/src/test/java/org/apache/sysds/test/component/compress/combine/CombineTest.java
index cfb11cf074..d6d814dc10 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/combine/CombineTest.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/combine/CombineTest.java
@@ -33,7 +33,7 @@ import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory;
import org.apache.sysds.runtime.compress.CompressionSettingsBuilder;
import org.apache.sysds.runtime.compress.colgroup.AColGroup;
import org.apache.sysds.runtime.compress.colgroup.AColGroup.CompressionType;
-import org.apache.sysds.runtime.compress.lib.CLALibCombine;
+import org.apache.sysds.runtime.compress.lib.CLALibStack;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.matrix.data.MatrixIndexes;
import org.apache.sysds.test.TestUtils;
@@ -55,7 +55,7 @@ public class CombineTest {
data.put(new MatrixIndexes(2, 1), m1);
try {
- MatrixBlock c = CLALibCombine.combine(data, 100 * 2, 10, 100, k);
+ MatrixBlock c = CLALibStack.combine(data, 100 * 2, 10, 100, k);
assertTrue("The result is not in compressed format", c instanceof CompressedMatrixBlock);
assertEquals(0.0, c.sum(), 0.0);
}
@@ -76,7 +76,7 @@ public class CombineTest {
data.put(new MatrixIndexes(2, 1), m1);
try {
- MatrixBlock c = CLALibCombine.combine(data, 100 * 2, 10, 100, k);
+ MatrixBlock c = CLALibStack.combine(data, 100 * 2, 10, 100, k);
assertTrue("The result is not in compressed format", c instanceof CompressedMatrixBlock);
assertEquals(0.0, c.sum(), 100.0 * 10.0 * 2);
}