You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2021/05/04 16:31:58 UTC
[systemds] branch master updated: [MINOR] CLA add empty check
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new 5af203a [MINOR] CLA add empty check
5af203a is described below
commit 5af203a001229a6f4608040ed7e558f59ffe937c
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Tue May 4 17:58:51 2021 +0200
[MINOR] CLA add empty check
- Add a check for empty colum groups in case of only null columns in
compression factory.
- Add edge case tests for empty and const colGroups
Closes #1253
---
.../compress/colgroup/ColGroupCompressed.java | 4 ++-
.../runtime/compress/colgroup/ColGroupFactory.java | 16 +++++-----
.../compress/colgroup/ColGroupUncompressed.java | 19 ++++++++---
.../component/compress/CompressedTestBase.java | 37 ++++++++++------------
4 files changed, 41 insertions(+), 35 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupCompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupCompressed.java
index 45701ff..a231283 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupCompressed.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupCompressed.java
@@ -90,7 +90,9 @@ public abstract class ColGroupCompressed extends AColGroup {
protected abstract boolean sameIndexStructure(ColGroupCompressed that);
public void leftMultByMatrix(MatrixBlock matrix, double[] result, int numCols, int rl, int ru) {
- if(matrix.isInSparseFormat())
+ if(matrix.isEmpty())
+ return;
+ else if(matrix.isInSparseFormat())
leftMultBySparseMatrix(matrix.getSparseBlock(), result, matrix.getNumRows(), numCols, rl, ru);
else {
leftMultByMatrix(matrix.getDenseBlockValues(), result, matrix.getNumRows(), numCols, rl, ru);
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupFactory.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupFactory.java
index 1b4c51c..49ef877 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupFactory.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupFactory.java
@@ -156,16 +156,11 @@ public class ColGroupFactory {
private static Collection<AColGroup> compressColGroup(MatrixBlock in, int[] colIndexes,
CompressionSettings compSettings) {
if(in.isInSparseFormat() && compSettings.transposed) {
-
+
final SparseBlock sb = in.getSparseBlock();
for(int col : colIndexes)
if(sb.isEmpty(col))
return compressColGroupAndExtractEmptyColumns(in, colIndexes, compSettings);
-
- // return (compRatios == null) ? compressColGroupForced(in,
- // colIndexes,
- // compSettings) : compressColGroupCorrecting(in, compRatios, colIndexes,
- // compSettings);
return Collections.singletonList(compressColGroupForced(in, colIndexes, compSettings));
}
else
@@ -185,8 +180,13 @@ public class ColGroupFactory {
v.appendValue(col);
}
AColGroup empty = compressColGroupForced(in, e.extractValues(true), compSettings);
- AColGroup colGroup = compressColGroupForced(in, v.extractValues(true), compSettings);
- return Arrays.asList(empty, colGroup);
+ if(v.size() > 0) {
+ AColGroup colGroup = compressColGroupForced(in, v.extractValues(true), compSettings);
+ return Arrays.asList(empty, colGroup);
+ }
+ else {
+ return Collections.singletonList(empty);
+ }
}
private static AColGroup compressColGroupForced(MatrixBlock in, int[] colIndexes,
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
index 0a536df..b13078c 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
@@ -163,7 +163,9 @@ public class ColGroupUncompressed extends AColGroup {
final int nCol = _colIndexes.length;
final int tCol = target.getNumColumns();
long nnz = 0;
- if(_data.isInSparseFormat()) {
+ if(_data.isEmpty())
+ return;
+ else if(_data.isInSparseFormat()) {
SparseBlock sb = _data.getSparseBlock();
for(int row = rl; row < ru; row++, offT += tCol) {
if(!sb.isEmpty(row)) {
@@ -539,10 +541,17 @@ public class ColGroupUncompressed extends AColGroup {
public void tsmm(double[] result, int numColumns) {
MatrixBlock tmp = new MatrixBlock(_colIndexes.length, _colIndexes.length, true);
LibMatrixMult.matrixMultTransposeSelf(_data, tmp, true, false);
- double[] tmpV = tmp.getDenseBlockValues();
- for(int i = 0, offD = 0, offT = 0; i < numColumns; i++, offD += numColumns, offT += _colIndexes.length)
- for(int j = i; j < numColumns; j++)
- result[offD + _colIndexes[j]] += tmpV[offT + j];
+ if(tmp.getDenseBlock() == null && tmp.getSparseBlock() == null)
+ return;
+ else if(tmp.isInSparseFormat()) {
+ throw new NotImplementedException("not Implemented sparse output of tsmm in compressed ColGroup.");
+ }
+ else {
+ double[] tmpV = tmp.getDenseBlockValues();
+ for(int i = 0, offD = 0, offT = 0; i < numColumns; i++, offD += numColumns, offT += _colIndexes.length)
+ for(int j = i; j < numColumns; j++)
+ result[offD + _colIndexes[j]] += tmpV[offT + j];
+ }
}
diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java
index 4954390..a93edd2 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java
@@ -75,29 +75,17 @@ import org.junit.runners.Parameterized.Parameters;
public abstract class CompressedTestBase extends TestBase {
protected static final Log LOG = LogFactory.getLog(CompressedTestBase.class.getName());
- protected static SparsityType[] usedSparsityTypes = new SparsityType[] {SparsityType.FULL,
- // SparsityType.DENSE,
- SparsityType.SPARSE,
- // SparsityType.ULTRA_SPARSE,
- // SparsityType.EMPTY
- };
+ protected static SparsityType[] usedSparsityTypes = new SparsityType[] {SparsityType.FULL, SparsityType.SPARSE,};
- protected static ValueType[] usedValueTypes = new ValueType[] {
- // ValueType.RAND,
- // ValueType.CONST,
- ValueType.RAND_ROUND, ValueType.OLE_COMPRESSIBLE, ValueType.RLE_COMPRESSIBLE,};
+ protected static ValueType[] usedValueTypes = new ValueType[] {ValueType.RAND_ROUND, ValueType.OLE_COMPRESSIBLE,
+ ValueType.RLE_COMPRESSIBLE,};
protected static ValueRange[] usedValueRanges = new ValueRange[] {ValueRange.SMALL, ValueRange.NEGATIVE,
- // ValueRange.LARGE,
- ValueRange.BYTE,
- // ValueRange.BOOLEAN,
- };
+ ValueRange.BYTE};
protected static OverLapping[] overLapping = new OverLapping[] {
// OverLapping.COL,
- OverLapping.PLUS,
- OverLapping.MATRIX,
- OverLapping.NONE,
+ OverLapping.PLUS, OverLapping.MATRIX, OverLapping.NONE,
// OverLapping.MATRIX_PLUS,
// OverLapping.SQUASH,
// OverLapping.MATRIX_MULT_NEGATIVE
@@ -129,14 +117,17 @@ public abstract class CompressedTestBase extends TestBase {
.setInvestigateEstimate(true),
new CompressionSettingsBuilder().setSamplingRatio(0.1).setSeed(compressionSeed).setTransposeInput("true")
.setColumnPartitioner(PartitionerType.BIN_PACKING).setInvestigateEstimate(true),
+ new CompressionSettingsBuilder().setSamplingRatio(0.1).setSeed(compressionSeed).setTransposeInput("true")
+ .setColumnPartitioner(PartitionerType.STATIC).setInvestigateEstimate(true),
+ // Forced Uncompressed tests
new CompressionSettingsBuilder().setValidCompressions(EnumSet.of(CompressionType.UNCOMPRESSED)),
// new CompressionSettingsBuilder().setSamplingRatio(0.1).setSeed(compressionSeed).setInvestigateEstimate(true),
// new CompressionSettingsBuilder().setSamplingRatio(1.0).setSeed(compressionSeed).setInvestigateEstimate(true)
// .setAllowSharedDictionary(false).setmaxStaticColGroupCoCode(1),
- // // // // LOSSY TESTS!
+ // LOSSY TESTS!
// new CompressionSettingsBuilder().setSamplingRatio(0.1).setSeed(compressionSeed)
// .setValidCompressions(EnumSet.of(CompressionType.DDC)).setInvestigateEstimate(true).setLossy(true).create(),
@@ -149,7 +140,7 @@ public abstract class CompressedTestBase extends TestBase {
// new CompressionSettingsBuilder().setSamplingRatio(1.0).setSeed(compressionSeed).setInvestigateEstimate(true)
// .setAllowSharedDictionary(false).setmaxStaticColGroupCoCode(1).setLossy(true).create(),
- // COCODING TESTS!!
+ // CO CODING TESTS!!
// new CompressionSettingsBuilder().setSamplingRatio(1.0).setSeed(compressionSeed).setInvestigateEstimate(true)
// .setAllowSharedDDCDictionary(false).setmaxStaticColGroupCoCode(20).create(),
@@ -176,8 +167,6 @@ public abstract class CompressedTestBase extends TestBase {
protected CompressionStatistics cmbStats;
// Decompressed Result
- // protected MatrixBlock cmbDeCompressed;
- // protected double[][] deCompressed;
/** number of threads used for the operation */
protected final int _k;
@@ -296,6 +285,12 @@ public abstract class CompressedTestBase extends TestBase {
for(MatrixTypology mt : usedMatrixTypology)
for(OverLapping ov : overLapping)
tests.add(new Object[] {st, vt, vr, cs, mt, ov});
+ for(CompressionSettingsBuilder cs : usedCompressionSettings)
+ for(MatrixTypology mt : usedMatrixTypology)
+ for(OverLapping ov : overLapping) {
+ tests.add(new Object[] {SparsityType.EMPTY, ValueType.RAND, ValueRange.BOOLEAN, cs, mt, ov});
+ tests.add(new Object[] {SparsityType.FULL, ValueType.CONST, ValueRange.LARGE, cs, mt, ov});
+ }
return tests;
}