You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2021/03/15 17:32:33 UTC
[systemds] branch master updated (e3e57ba -> 96ecdfa)
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git.
from e3e57ba [MINOR] add link to windows dev doc
new 2bc40ed [SYSTEMDS-2897] CLA decompressing write
new 0edc28e [SYSTEMDS-2898] CLA original size bug
new 90bde55 [MINOR] fix various tests
new 96ecdfa [MINOR] Ignore PCA rewrite tests
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hops/rewrite/RewriteCompressedReblock.java | 20 ++++---
.../runtime/compress/CompressedMatrixBlock.java | 69 +++++++++++-----------
.../compress/CompressedMatrixBlockFactory.java | 16 +++--
.../runtime/compress/CompressionStatistics.java | 1 +
.../sysds/runtime/compress/colgroup/AColGroup.java | 2 +
.../runtime/compress/colgroup/ADictionary.java | 2 +
.../runtime/compress/colgroup/ColGroupEmpty.java | 9 +++
.../compress/colgroup/ColGroupUncompressed.java | 5 ++
.../runtime/compress/colgroup/ColGroupValue.java | 6 ++
.../runtime/compress/colgroup/Dictionary.java | 16 +++++
.../runtime/compress/colgroup/QDictionary.java | 16 +++++
.../compress/estim/CompressedSizeInfoColGroup.java | 2 +-
.../sysds/runtime/compress/lib/CLALibAppend.java | 69 ++++++++++++++++------
.../sysds/runtime/compress/lib/CLALibCompAgg.java | 2 +-
.../runtime/compress/lib/CLALibLeftMultBy.java | 50 ++--------------
.../sysds/runtime/compress/lib/CLALibReExpand.java | 2 +-
.../runtime/compress/lib/CLALibRightMultBy.java | 4 +-
.../sysds/runtime/compress/lib/CLALibScalar.java | 6 +-
.../sysds/runtime/compress/lib/CLALibSquash.java | 2 +-
.../controlprogram/caching/CacheableData.java | 2 +-
.../controlprogram/caching/MatrixObject.java | 7 ++-
.../org/apache/sysds/runtime/data/DenseBlock.java | 10 ++++
.../apache/sysds/runtime/data/DenseBlockFP64.java | 7 +++
.../org/apache/sysds/runtime/data/SparseBlock.java | 5 ++
.../apache/sysds/runtime/data/SparseBlockCOO.java | 7 ++-
.../apache/sysds/runtime/data/SparseBlockCSR.java | 9 +--
.../apache/sysds/runtime/data/SparseBlockMCSR.java | 15 +++--
.../runtime/matrix/data/LibMatrixDatagen.java | 17 ++++--
.../sysds/runtime/matrix/data/MatrixBlock.java | 33 +++++------
.../org/apache/sysds/utils/MemoryEstimates.java | 59 +++++++++++++++++-
src/main/python/tests/matrix/test_print.py | 4 +-
src/test/java/org/apache/sysds/test/TestUtils.java | 10 ++--
.../component/compress/CompressedTestBase.java | 14 ++---
.../functions/codegenalg/partone/AlgorithmPCA.java | 19 +++++-
.../compress/compressInstructionRewrite.java | 2 +-
.../compress/configuration/CompressLossy.java | 2 +-
.../compress/configuration/CompressLossyCost.java | 4 +-
.../test/functions/misc/AssertExpressionTest.java | 2 +
.../functions/misc/ConditionalValidateTest.java | 2 +
.../test/functions/misc/DataTypeCastingTest.java | 2 +
.../test/functions/misc/DataTypeChangeTest.java | 2 +
.../test/functions/misc/ExistsVariableTest.java | 4 +-
.../functions/misc/FunctionInExpressionTest.java | 4 +-
.../test/functions/misc/FunctionInliningTest.java | 2 +
.../test/functions/misc/FunctionNamespaceTest.java | 4 ++
.../test/functions/misc/FunctionNotFoundTest.java | 2 +
.../test/functions/misc/FunctionReturnTest.java | 6 +-
...nstantFoldingScalarVariablePropagationTest.java | 2 +
.../functions/misc/IPADeadCodeEliminationTest.java | 2 +
.../functions/misc/IPAFunctionInliningTest.java | 2 +
.../functions/misc/IPALiteralReplacementTest.java | 2 +
.../test/functions/misc/IPANnzPropagationTest.java | 2 +
.../functions/misc/IPAScalarRecursionTest.java | 2 +
.../misc/IPAScalarVariablePropagationTest.java | 4 ++
.../functions/misc/IPAUnknownRecursionTest.java | 2 +
.../test/functions/misc/ListAndStructTest.java | 2 +
.../test/functions/misc/ListAppendRemove.java | 2 +
.../test/functions/misc/LongOverflowTest.java | 2 +
.../functions/misc/NegativeLoopIncrementsTest.java | 4 +-
.../test/functions/misc/NrowNcolStringTest.java | 2 +
.../test/functions/misc/PrintExpressionTest.java | 2 +
.../sysds/test/functions/misc/PrintMatrixTest.java | 2 +
.../test/functions/misc/ReadAfterWriteTest.java | 2 +
.../test/functions/misc/ScalarAssignmentTest.java | 2 +
.../functions/misc/ZeroRowsColsMatrixTest.java | 2 +
src/test/resources/log4j.properties | 4 +-
.../SystemDS-config-compress-cost.xml | 2 +-
.../SystemDS-config-compress-cost-OLE.xml | 2 +-
...xml => SystemDS-config-compress-cost-lossy.xml} | 2 +-
.../cost/SystemDS-config-compress-cost.xml | 2 +-
.../{lossy => }/SystemDS-config-compress-lossy.xml | 2 +-
.../compress/force/SystemDS-config-compress.xml | 2 +-
.../scripts/functions/misc/IPANnzPropagation1.dml | 2 +-
.../scripts/functions/misc/IPANnzPropagation2.dml | 4 +-
74 files changed, 429 insertions(+), 188 deletions(-)
rename src/test/scripts/functions/compress/cost/{lossy/SystemDS-config-compress-lossy-cost.xml => SystemDS-config-compress-cost-lossy.xml} (93%)
rename src/test/scripts/functions/compress/force/{lossy => }/SystemDS-config-compress-lossy.xml (93%)
[systemds] 04/04: [MINOR] Ignore PCA rewrite tests
Posted by ba...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 96ecdfae36056efd976c0870b27e517ea9e05c7b
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Mon Mar 15 17:38:24 2021 +0100
[MINOR] Ignore PCA rewrite tests
Closes #1201
---
.../functions/codegenalg/partone/AlgorithmPCA.java | 19 ++++++++++++++++++-
1 file changed, 18 insertions(+), 1 deletion(-)
diff --git a/src/test/java/org/apache/sysds/test/functions/codegenalg/partone/AlgorithmPCA.java b/src/test/java/org/apache/sysds/test/functions/codegenalg/partone/AlgorithmPCA.java
index 6168265..9a57989 100644
--- a/src/test/java/org/apache/sysds/test/functions/codegenalg/partone/AlgorithmPCA.java
+++ b/src/test/java/org/apache/sysds/test/functions/codegenalg/partone/AlgorithmPCA.java
@@ -22,7 +22,6 @@ package org.apache.sysds.test.functions.codegenalg.partone;
import java.io.File;
import java.util.HashMap;
-import org.junit.Test;
import org.apache.sysds.common.Types.ExecMode;
import org.apache.sysds.hops.OptimizerUtils;
import org.apache.sysds.lops.LopProperties.ExecType;
@@ -31,6 +30,8 @@ import org.apache.sysds.test.AutomatedTestBase;
import org.apache.sysds.test.TestConfiguration;
import org.apache.sysds.test.TestUtils;
import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
public class AlgorithmPCA extends AutomatedTestBase
{
@@ -56,81 +57,97 @@ public class AlgorithmPCA extends AutomatedTestBase
}
@Test
+ @Ignore
public void testPCADenseRewritesCP() {
runPCATest(TEST_NAME1, true, false, ExecType.CP, CodegenTestType.DEFAULT);
}
@Test
+ @Ignore
public void testPCASparseRewritesCP() {
runPCATest(TEST_NAME1, true, true, ExecType.CP, CodegenTestType.DEFAULT);
}
@Test
+ @Ignore
public void testPCADenseCP() {
runPCATest(TEST_NAME1, false, false, ExecType.CP, CodegenTestType.DEFAULT);
}
@Test
+ @Ignore
public void testPCASparseCP() {
runPCATest(TEST_NAME1, false, true, ExecType.CP, CodegenTestType.DEFAULT);
}
@Test
+ @Ignore
public void testPCADenseRewritesSP() {
runPCATest(TEST_NAME1, true, false, ExecType.SPARK, CodegenTestType.DEFAULT);
}
@Test
+ @Ignore
public void testPCASparseRewritesSP() {
runPCATest(TEST_NAME1, true, true, ExecType.SPARK, CodegenTestType.DEFAULT);
}
@Test
+ @Ignore
public void testPCADenseSP() {
runPCATest(TEST_NAME1, false, false, ExecType.SPARK, CodegenTestType.DEFAULT);
}
@Test
+ @Ignore
public void testPCASparseSP() {
runPCATest(TEST_NAME1, false, true, ExecType.SPARK, CodegenTestType.DEFAULT);
}
@Test
+ @Ignore
public void testPCADenseRewritesCPFuseAll() {
runPCATest(TEST_NAME1, true, false, ExecType.CP, CodegenTestType.FUSE_ALL);
}
@Test
+ @Ignore
public void testPCASparseRewritesCPFuseAll() {
runPCATest(TEST_NAME1, true, true, ExecType.CP, CodegenTestType.FUSE_ALL);
}
@Test
+ @Ignore
public void testPCADenseRewritesSPFuseAll() {
runPCATest(TEST_NAME1, true, false, ExecType.SPARK, CodegenTestType.FUSE_ALL);
}
@Test
+ @Ignore
public void testPCASparseRewritesSPFuseAll() {
runPCATest(TEST_NAME1, true, true, ExecType.SPARK, CodegenTestType.FUSE_ALL);
}
@Test
+ @Ignore
public void testPCADenseRewritesCPFuseNoRedundancy() {
runPCATest(TEST_NAME1, true, false, ExecType.CP, CodegenTestType.FUSE_NO_REDUNDANCY);
}
@Test
+ @Ignore
public void testPCASparseRewritesCPFuseNoRedundancy() {
runPCATest(TEST_NAME1, true, true, ExecType.CP, CodegenTestType.FUSE_NO_REDUNDANCY);
}
@Test
+ @Ignore
public void testPCADenseRewritesSPFuseNoRedundancy() {
runPCATest(TEST_NAME1, true, false, ExecType.SPARK, CodegenTestType.FUSE_NO_REDUNDANCY);
}
@Test
+ @Ignore
public void testPCASparseRewritesSPFuseNoRedundancy() {
runPCATest(TEST_NAME1, true, true, ExecType.SPARK, CodegenTestType.FUSE_NO_REDUNDANCY);
}
[systemds] 03/04: [MINOR] fix various tests
Posted by ba...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 90bde55c800f170d142db40dd76e88856c4e70f4
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Sun Mar 14 00:19:35 2021 +0100
[MINOR] fix various tests
- lib matrix DataGen test fix - to create a sparseBlock if null.
- reduce nnzPropagation tests memory usage because sometime we have OOM
fails of this test on GitHub
- misc tests less debug output
- cast to float in python tests
- Compressed append Empty optimization
---
src/main/java/org/apache/sysds/conf/DMLConfig.java | 2 +-
.../hops/rewrite/RewriteCompressedReblock.java | 2 +-
.../runtime/compress/CompressedMatrixBlock.java | 2 +-
.../runtime/compress/colgroup/ColGroupEmpty.java | 9 +++
.../sysds/runtime/compress/lib/CLALibAppend.java | 72 ++++++++++++++++------
.../apache/sysds/runtime/data/SparseBlockCOO.java | 6 +-
.../runtime/matrix/data/LibMatrixDatagen.java | 17 +++--
src/main/python/tests/matrix/test_print.py | 4 +-
src/test/java/org/apache/sysds/test/TestUtils.java | 10 +--
.../test/functions/misc/AssertExpressionTest.java | 2 +
.../functions/misc/ConditionalValidateTest.java | 2 +
.../test/functions/misc/DataTypeCastingTest.java | 2 +
.../test/functions/misc/DataTypeChangeTest.java | 2 +
.../test/functions/misc/ExistsVariableTest.java | 4 +-
.../functions/misc/FunctionInExpressionTest.java | 4 +-
.../test/functions/misc/FunctionInliningTest.java | 2 +
.../test/functions/misc/FunctionNamespaceTest.java | 4 ++
.../test/functions/misc/FunctionNotFoundTest.java | 2 +
.../test/functions/misc/FunctionReturnTest.java | 6 +-
...nstantFoldingScalarVariablePropagationTest.java | 2 +
.../functions/misc/IPADeadCodeEliminationTest.java | 2 +
.../functions/misc/IPAFunctionInliningTest.java | 2 +
.../functions/misc/IPALiteralReplacementTest.java | 2 +
.../test/functions/misc/IPANnzPropagationTest.java | 2 +
.../functions/misc/IPAScalarRecursionTest.java | 2 +
.../misc/IPAScalarVariablePropagationTest.java | 4 ++
.../functions/misc/IPAUnknownRecursionTest.java | 2 +
.../test/functions/misc/ListAndStructTest.java | 2 +
.../test/functions/misc/ListAppendRemove.java | 2 +
.../test/functions/misc/LongOverflowTest.java | 2 +
.../functions/misc/NegativeLoopIncrementsTest.java | 4 +-
.../test/functions/misc/NrowNcolStringTest.java | 2 +
.../test/functions/misc/PrintExpressionTest.java | 2 +
.../sysds/test/functions/misc/PrintMatrixTest.java | 2 +
.../test/functions/misc/ReadAfterWriteTest.java | 2 +
.../test/functions/misc/ScalarAssignmentTest.java | 2 +
.../functions/misc/ZeroRowsColsMatrixTest.java | 2 +
.../scripts/functions/misc/IPANnzPropagation1.dml | 2 +-
.../scripts/functions/misc/IPANnzPropagation2.dml | 4 +-
39 files changed, 158 insertions(+), 42 deletions(-)
diff --git a/src/main/java/org/apache/sysds/conf/DMLConfig.java b/src/main/java/org/apache/sysds/conf/DMLConfig.java
index 74ddf8a..5f90fb4 100644
--- a/src/main/java/org/apache/sysds/conf/DMLConfig.java
+++ b/src/main/java/org/apache/sysds/conf/DMLConfig.java
@@ -126,7 +126,7 @@ public class DMLConfig
_defaultVals.put(CP_PARALLEL_IO, "true" );
_defaultVals.put(COMPRESSED_LINALG, Compression.CompressConfig.FALSE.name() );
_defaultVals.put(COMPRESSED_LOSSY, "false" );
- _defaultVals.put(COMPRESSED_VALID_COMPRESSIONS, "SDC,DDC,RLE,OLE");
+ _defaultVals.put(COMPRESSED_VALID_COMPRESSIONS, "SDC,DDC");
_defaultVals.put(COMPRESSED_OVERLAPPING, "true" );
_defaultVals.put(COMPRESSED_SAMPLING_RATIO, "0.01");
_defaultVals.put(COMPRESSED_COCODE, "COST");
diff --git a/src/main/java/org/apache/sysds/hops/rewrite/RewriteCompressedReblock.java b/src/main/java/org/apache/sysds/hops/rewrite/RewriteCompressedReblock.java
index 194a5d7..0f26fa1 100644
--- a/src/main/java/org/apache/sysds/hops/rewrite/RewriteCompressedReblock.java
+++ b/src/main/java/org/apache/sysds/hops/rewrite/RewriteCompressedReblock.java
@@ -215,7 +215,7 @@ public class RewriteCompressedReblock extends StatementBlockRewriteRule {
private int numberCompressedOpsExecuted = 0;
private int numberDecompressedOpsExecuted = 0;
private int inefficientSupportedOpsExecuted = 0;
- private int superEfficientSuportedOpsExecuted = 0;
+ // private int superEfficientSupportedOpsExecuted = 0;
private boolean foundStart = false;
private boolean usedInLoop = false;
diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
index 828610b..5dc44d1 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
@@ -1235,7 +1235,7 @@ public class CompressedMatrixBlock extends MatrixBlock {
public MatrixBlock randOperationsInPlace(RandomMatrixGenerator rgen, Well1024a bigrand, long bSeed, int k) {
throw new DMLRuntimeException("CompressedMatrixBlock: randOperationsInPlace not supported.");
}
-
+
@Override
public MatrixBlock seqOperationsInPlace(double from, double to, double incr) {
// output should always be uncompressed
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java
index a70d250..7cda071 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java
@@ -38,6 +38,7 @@ public class ColGroupEmpty extends ColGroupValue {
super();
}
+
/**
* Constructs an Constant Colum Group, that contains only one tuple, with the given value.
*
@@ -49,6 +50,14 @@ public class ColGroupEmpty extends ColGroupValue {
_zeros = true;
}
+ public static ColGroupEmpty generate(int nCol, int nRow){
+ int[] cols = new int[nCol];
+ for(int i =0; i < nCol; i++){
+ cols[i] =i;
+ }
+ return new ColGroupEmpty(cols,nRow);
+ }
+
@Override
public int[] getCounts(int[] out) {
// nothing
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java
index 2ea5397..bb130b6 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java
@@ -20,6 +20,7 @@
package org.apache.sysds.runtime.compress.lib;
import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.logging.Log;
@@ -28,6 +29,7 @@ import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory;
import org.apache.sysds.runtime.compress.CompressionStatistics;
import org.apache.sysds.runtime.compress.colgroup.AColGroup;
+import org.apache.sysds.runtime.compress.colgroup.ColGroupEmpty;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
public class CLALibAppend {
@@ -35,25 +37,25 @@ public class CLALibAppend {
private static final Log LOG = LogFactory.getLog(CLALibAppend.class.getName());
public static MatrixBlock append(MatrixBlock left, MatrixBlock right) {
-
- // if(left.isEmpty())
- // return right;
- // else if(right.isEmpty())
- // return left;
+
+ if(left.isEmpty() && right instanceof CompressedMatrixBlock)
+ return appendLeftEmpty(left, (CompressedMatrixBlock) right);
+ else if(right.isEmpty() && left instanceof CompressedMatrixBlock)
+ return appendRightEmpty((CompressedMatrixBlock)left, right);
+
final int m = left.getNumRows();
final int n = left.getNumColumns() + right.getNumColumns();
-
// try to compress both sides (if not already compressed).
- if(!(left instanceof CompressedMatrixBlock) && m > 1000){
- LOG.info("Compressing left for append operation");
+ if(!(left instanceof CompressedMatrixBlock) && m > 1000) {
+ LOG.warn("Compressing left for append operation");
Pair<MatrixBlock, CompressionStatistics> x = CompressedMatrixBlockFactory.compress(left);
if(x.getRight().ratio > 3.0)
left = x.getLeft();
-
+
}
- if(!(right instanceof CompressedMatrixBlock) && m > 1000){
- LOG.info("Compressing right for append operation");
+ if(!(right instanceof CompressedMatrixBlock) && m > 1000) {
+ LOG.warn("Compressing right for append operation");
Pair<MatrixBlock, CompressionStatistics> x = CompressedMatrixBlockFactory.compress(right);
if(x.getRight().ratio > 3.0)
right = x.getLeft();
@@ -69,25 +71,57 @@ public class CLALibAppend {
// init result matrix
CompressedMatrixBlock ret = new CompressedMatrixBlock(m, n);
+ ret = appendColGroups(ret, leftC.getColGroups(), rightC.getColGroups(), leftC.getNumColumns());
+ return ret;
+ }
+
+ private static MatrixBlock appendRightEmpty(CompressedMatrixBlock left, MatrixBlock right) {
+
+ final int m = left.getNumRows();
+ final int n = left.getNumColumns() + right.getNumColumns();
+ CompressedMatrixBlock ret = new CompressedMatrixBlock(m,n);
+
+ List<AColGroup> newGroup = new ArrayList<>(1);
+ newGroup.add(ColGroupEmpty.generate(right.getNumColumns(), right.getNumRows()));
+ ret = appendColGroups(ret, left.getColGroups(), newGroup, left.getNumColumns());
+
+ return ret;
+ }
+
+ private static MatrixBlock appendLeftEmpty(MatrixBlock left, CompressedMatrixBlock right) {
+ final int m = left.getNumRows();
+ final int n = left.getNumColumns() + right.getNumColumns();
+ CompressedMatrixBlock ret = new CompressedMatrixBlock(m,n);
+
+ List<AColGroup> newGroup = new ArrayList<>(1);
+ newGroup.add(ColGroupEmpty.generate(left.getNumColumns(), left.getNumRows()));
+ ret = appendColGroups(ret, newGroup, right.getColGroups(), left.getNumColumns());
+
+ return ret;
+ }
+
+ private static CompressedMatrixBlock appendColGroups(CompressedMatrixBlock ret, List<AColGroup> left,
+ List<AColGroup> right, int leftNumCols) {
+
// shallow copy of lhs column groups
- ret.allocateColGroupList(new ArrayList<AColGroup>(leftC.getColGroups().size() + rightC.getColGroups().size()));
+ ret.allocateColGroupList(new ArrayList<AColGroup>(left.size() + right.size()));
- for(AColGroup group : leftC.getColGroups()){
+ long nnz = 0;
+ for(AColGroup group : left) {
AColGroup tmp = group.copy();
ret.getColGroups().add(tmp);
+ nnz += group.getNumberNonZeros();
}
- for(AColGroup group : rightC.getColGroups()) {
+ for(AColGroup group : right) {
AColGroup tmp = group.copy();
- tmp.shiftColIndices(left.getNumColumns());
+ tmp.shiftColIndices(leftNumCols);
ret.getColGroups().add(tmp);
+ nnz += group.getNumberNonZeros();
}
- long nnzl = (leftC.getNonZeros() <= -1 ) ? leftC.recomputeNonZeros() : leftC.getNonZeros() ;
- long nnzr = (rightC.getNonZeros() <= -1 ) ? rightC.recomputeNonZeros() : rightC.getNonZeros() ;
-
// meta data maintenance
- ret.setNonZeros(nnzl + nnzr);
+ ret.setNonZeros(nnz);
return ret;
}
diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java
index aebf060..ce8e707 100644
--- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java
+++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java
@@ -150,9 +150,9 @@ public class SparseBlockCOO extends SparseBlock
//32B overhead per array, int/int/double arr in nnz
double size = 16 + 8; //object + 2 int fields
- size += MemoryEstimates.intArrayCost((int)lnnz); ; //rindexes array (row indexes)
- size += MemoryEstimates.intArrayCost((int) lnnz); ; //cindexes array (column indexes)
- size += MemoryEstimates.doubleArrayCost((int) lnnz);; //values array (non-zero values)
+ size += MemoryEstimates.intArrayCost((int)lnnz); //rindexes array (row indexes)
+ size += MemoryEstimates.intArrayCost((int) lnnz); //cindexes array (column indexes)
+ size += MemoryEstimates.doubleArrayCost((int) lnnz); //values array (non-zero values)
//robustness for long overflows
return (long) Math.min(size, Long.MAX_VALUE);
diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixDatagen.java b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixDatagen.java
index a8fce54..2f1a51f 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixDatagen.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixDatagen.java
@@ -33,6 +33,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.commons.math3.random.Well1024a;
import org.apache.sysds.hops.DataGenOp;
import org.apache.sysds.runtime.DMLRuntimeException;
+import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
import org.apache.sysds.runtime.controlprogram.parfor.util.IDSequence;
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.SparseBlock;
@@ -186,6 +187,9 @@ public class LibMatrixDatagen
int blen = rgen._blocksize;
double sparsity = rgen._sparsity;
+ if(out instanceof CompressedMatrixBlock)
+ throw new DMLRuntimeException("Invalid to use compressed matrix block as output");
+
// sanity check valid dimensions and sparsity
checkMatrixDimensionsAndSparsity(rows, cols, sparsity);
@@ -258,6 +262,9 @@ public class LibMatrixDatagen
int blen = rgen._blocksize;
double sparsity = rgen._sparsity;
+ if(out instanceof CompressedMatrixBlock)
+ throw new DMLRuntimeException("Invalid to use compressed matrix block as output");
+
//sanity check valid dimensions and sparsity
checkMatrixDimensionsAndSparsity(rows, cols, sparsity);
@@ -496,13 +503,15 @@ public class LibMatrixDatagen
// are always selected uniformly at random.
nnzPRNG.setSeed(seed);
- // block-level sparsity, which may differ from overall sparsity in the matrix.
- // (e.g., border blocks may fall under skinny matrix turn point, in CP this would be
- // irrelevant but we need to ensure consistency with MR)
boolean localSparse = MatrixBlock.evalSparseFormatInMemory(
blockrows, blockcols, (long)(sparsity*blockrows*blockcols));
- if ( localSparse ) {
+ if ( localSparse) {
+
SparseBlock c = out.sparseBlock;
+ if(c == null){
+ out.allocateSparseRowsBlock();
+ c = out.sparseBlock;
+ }
// Prob [k-1 zeros before a nonzero] = Prob [k-1 < log(uniform)/log(1-p) < k] = p*(1-p)^(k-1), where p=sparsity
double log1mp = Math.log(1-sparsity);
int idx = 0; // takes values in range [1, blen*blen] (both ends including)
diff --git a/src/main/python/tests/matrix/test_print.py b/src/main/python/tests/matrix/test_print.py
index 670ce37..4dc5a4b 100644
--- a/src/main/python/tests/matrix/test_print.py
+++ b/src/main/python/tests/matrix/test_print.py
@@ -46,11 +46,11 @@ class TestPrint(unittest.TestCase):
def test_print_01(self):
Matrix(self.sds, np.array([1])).to_string().print().compute()
- self.assertEqual('1.000',self.sds.get_stdout()[0])
+ self.assertEqual(1,float(self.sds.get_stdout()[0]))
def test_print_02(self):
self.sds.scalar(1).print().compute()
- self.assertEqual('1', self.sds.get_stdout()[0])
+ self.assertEqual(1,float(self.sds.get_stdout()[0]))
if __name__ == "__main__":
unittest.main(exit=False)
diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java
index 6244833..48ba4ac 100644
--- a/src/test/java/org/apache/sysds/test/TestUtils.java
+++ b/src/test/java/org/apache/sysds/test/TestUtils.java
@@ -1047,10 +1047,12 @@ public class TestUtils
if (!compareCellValue(v1, v2, 0, ignoreNaN)) {
if (!compareCellValue(v1, v2, tolerance, ignoreNaN)) {
countErrorWithinTolerance++;
- if(!flag)
- System.out.println(e.getKey()+": "+v1+" <--> "+v2);
- else
- System.out.println(e.getKey()+": "+v2+" <--> "+v1);
+ if(LOG.isDebugEnabled()){
+ if(!flag)
+ LOG.debug(e.getKey()+": "+v1+" <--> "+v2);
+ else
+ LOG.debug(e.getKey()+": "+v2+" <--> "+v1);
+ }
}
} else {
countIdentical++;
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/AssertExpressionTest.java b/src/test/java/org/apache/sysds/test/functions/misc/AssertExpressionTest.java
index 7fadbe7..1ee0e4f 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/AssertExpressionTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/AssertExpressionTest.java
@@ -58,6 +58,7 @@ public class AssertExpressionTest extends AutomatedTestBase
*/
private void runPrintExpressionTest( String testname, boolean rewrites )
{
+ setOutputBuffering(true);
String TEST_NAME = testname;
TestConfiguration config = getTestConfiguration(TEST_NAME);
loadTestConfiguration(config);
@@ -82,5 +83,6 @@ public class AssertExpressionTest extends AutomatedTestBase
{
OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldRewriteFlag;
}
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ConditionalValidateTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ConditionalValidateTest.java
index b911a4a..bec66c3 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/ConditionalValidateTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/ConditionalValidateTest.java
@@ -102,6 +102,7 @@ public class ConditionalValidateTest extends AutomatedTestBase
private void runTest( String testName, Class<?> exceptionClass, boolean fileExists )
{
+ setOutputBuffering(true);
String TEST_NAME = testName;
try
@@ -135,5 +136,6 @@ public class ConditionalValidateTest extends AutomatedTestBase
catch(Exception ex) {
throw new RuntimeException(ex);
}
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/DataTypeCastingTest.java b/src/test/java/org/apache/sysds/test/functions/misc/DataTypeCastingTest.java
index 6df7e9c..d524f64 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/DataTypeCastingTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/DataTypeCastingTest.java
@@ -87,6 +87,7 @@ public class DataTypeCastingTest extends AutomatedTestBase
*/
private void runTest( String testName, boolean matrixInput, Class<?> exceptionClass )
{
+ setOutputBuffering(true);
String TEST_NAME = testName;
int numVals = (exceptionClass != null ? 7 : 1);
@@ -134,5 +135,6 @@ public class DataTypeCastingTest extends AutomatedTestBase
{
throw new RuntimeException(ex);
}
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/DataTypeChangeTest.java b/src/test/java/org/apache/sysds/test/functions/misc/DataTypeChangeTest.java
index d50eb23..3b4741a 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/DataTypeChangeTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/DataTypeChangeTest.java
@@ -150,6 +150,7 @@ public class DataTypeChangeTest extends AutomatedTestBase
private void runTest( String testName, Class<?> exceptionExpected )
{
+ setOutputBuffering(true);
String RI_HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = RI_HOME + testName + ".dml";
programArgs = new String[]{};
@@ -159,6 +160,7 @@ public class DataTypeChangeTest extends AutomatedTestBase
//integration test from outside SystemDS
runTest(true, exceptionExpected != null, exceptionExpected, -1);
+ setOutputBuffering(false);
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ExistsVariableTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ExistsVariableTest.java
index dee93cd..9d229fb 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/ExistsVariableTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/ExistsVariableTest.java
@@ -63,12 +63,13 @@ public class ExistsVariableTest extends AutomatedTestBase
}
private void runExistsTest(String testName, boolean pos) {
+ setOutputBuffering(true);
TestConfiguration config = getTestConfiguration(testName);
loadTestConfiguration(config);
String HOME = SCRIPT_DIR + TEST_DIR;
String param = pos ? "1" : "0";
fullDMLScriptName = HOME + testName + ".dml";
- programArgs = new String[]{"-stats", "-args", param, output("R") };
+ programArgs = new String[]{"-args", param, output("R") };
//run script and compare output
runTest(true, false, null, -1);
@@ -78,5 +79,6 @@ public class ExistsVariableTest extends AutomatedTestBase
val = (val!=null) ? val : 0;
Assert.assertTrue("Wrong result: "+param+" vs "+val,
val==Double.parseDouble(param));
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/FunctionInExpressionTest.java b/src/test/java/org/apache/sysds/test/functions/misc/FunctionInExpressionTest.java
index 7c5cd44..6d0b260 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/FunctionInExpressionTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/FunctionInExpressionTest.java
@@ -89,12 +89,13 @@ public class FunctionInExpressionTest extends AutomatedTestBase
private void runFunInExpressionTest( String testName )
{
+ setOutputBuffering(true);
TestConfiguration config = getTestConfiguration(testName);
loadTestConfiguration(config);
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testName + ".dml";
- programArgs = new String[]{"-stats", "-args", output("R") };
+ programArgs = new String[]{"-args", output("R") };
fullRScriptName = HOME + testName + ".R";
rCmd = getRCmd(expectedDir());
@@ -105,5 +106,6 @@ public class FunctionInExpressionTest extends AutomatedTestBase
//compare results
double val = readDMLMatrixFromOutputDir("R").get(new CellIndex(1,1));
Assert.assertTrue("Wrong result: 7 vs "+val, Math.abs(val-7)<Math.pow(10, -13));
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/FunctionInliningTest.java b/src/test/java/org/apache/sysds/test/functions/misc/FunctionInliningTest.java
index 8861d27..b539328 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/FunctionInliningTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/FunctionInliningTest.java
@@ -85,6 +85,7 @@ public class FunctionInliningTest extends AutomatedTestBase
private void runInliningTest( String testname, boolean IPA )
{
+ setOutputBuffering(true);
boolean oldIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
try
@@ -125,6 +126,7 @@ public class FunctionInliningTest extends AutomatedTestBase
{
OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldIPA;
}
+ setOutputBuffering(false);
}
}
\ No newline at end of file
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/FunctionNamespaceTest.java b/src/test/java/org/apache/sysds/test/functions/misc/FunctionNamespaceTest.java
index e8b016b..f3f3fe9 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/FunctionNamespaceTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/FunctionNamespaceTest.java
@@ -161,6 +161,7 @@ public class FunctionNamespaceTest extends AutomatedTestBase
private void runFunctionNamespaceTest(String TEST_NAME)
{
+ setOutputBuffering(true);
getAndLoadTestConfiguration(TEST_NAME);
fullDMLScriptName = SCRIPT_DIR + TEST_DIR + TEST_NAME + ".dml";
@@ -183,10 +184,12 @@ public class FunctionNamespaceTest extends AutomatedTestBase
finally {
System.setErr(origStdErr);
}
+ setOutputBuffering(false);
}
private void runFunctionNoInliningNamespaceTest(String TEST_NAME, boolean IPA)
{
+ setOutputBuffering(true);
boolean origIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
getAndLoadTestConfiguration(TEST_NAME);
@@ -229,5 +232,6 @@ public class FunctionNamespaceTest extends AutomatedTestBase
System.setErr(originalStdErr);
OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = origIPA;
}
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/FunctionNotFoundTest.java b/src/test/java/org/apache/sysds/test/functions/misc/FunctionNotFoundTest.java
index 4c56806..c7ed903 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/FunctionNotFoundTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/FunctionNotFoundTest.java
@@ -50,6 +50,7 @@ public class FunctionNotFoundTest extends AutomatedTestBase {
}
private void runFunctionNotFoundTest(String testName, Class<?> error) {
+ setOutputBuffering(true);
TestConfiguration config = getTestConfiguration(testName);
loadTestConfiguration(config);
@@ -58,5 +59,6 @@ public class FunctionNotFoundTest extends AutomatedTestBase {
programArgs = new String[] {};
runTest(true, true, error, -1);
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/FunctionReturnTest.java b/src/test/java/org/apache/sysds/test/functions/misc/FunctionReturnTest.java
index 60fc339..1dd9d62 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/FunctionReturnTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/FunctionReturnTest.java
@@ -58,6 +58,8 @@ public class FunctionReturnTest extends AutomatedTestBase
}
private void runFunctionReturnTest( String testname, boolean IPA ) {
+
+ setOutputBuffering(true);
boolean oldIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = IPA;
try {
@@ -66,12 +68,14 @@ public class FunctionReturnTest extends AutomatedTestBase
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + testname + ".dml";
- programArgs = new String[]{"-explain"};
+ programArgs = new String[]{};
runTest(true, false, null, -1);
}
finally {
OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldIPA;
}
+
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPAConstantFoldingScalarVariablePropagationTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPAConstantFoldingScalarVariablePropagationTest.java
index 74b60d5..8d2213c 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/IPAConstantFoldingScalarVariablePropagationTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/IPAConstantFoldingScalarVariablePropagationTest.java
@@ -94,6 +94,7 @@ public class IPAConstantFoldingScalarVariablePropagationTest extends AutomatedTe
*/
private void runIPAScalarVariablePropagationTest(String testname, boolean IPA_SECOND_CHANCE)
{
+ setOutputBuffering(true);
// Save old settings
int oldIPANumRep = OptimizerUtils.IPA_NUM_REPETITIONS;
boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
@@ -125,5 +126,6 @@ public class IPAConstantFoldingScalarVariablePropagationTest extends AutomatedTe
DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
rtplatform = platformOld;
}
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPADeadCodeEliminationTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPADeadCodeEliminationTest.java
index daf034a..35a4d34 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/IPADeadCodeEliminationTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/IPADeadCodeEliminationTest.java
@@ -87,6 +87,7 @@ public class IPADeadCodeEliminationTest extends AutomatedTestBase
private void runIPALiteralReplacementTest( String testname, boolean IPA )
{
+ setOutputBuffering(true);
boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
try {
@@ -106,5 +107,6 @@ public class IPADeadCodeEliminationTest extends AutomatedTestBase
finally {
OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA;
}
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPAFunctionInliningTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPAFunctionInliningTest.java
index ca9d156..83c90ca 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/IPAFunctionInliningTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/IPAFunctionInliningTest.java
@@ -122,6 +122,7 @@ public class IPAFunctionInliningTest extends AutomatedTestBase
private void runIPAFunInlineTest( String testName, boolean IPA )
{
+ setOutputBuffering(true);
boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
try
@@ -162,5 +163,6 @@ public class IPAFunctionInliningTest extends AutomatedTestBase
finally {
OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA;
}
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPALiteralReplacementTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPALiteralReplacementTest.java
index dc832ca..1a96f52 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/IPALiteralReplacementTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/IPALiteralReplacementTest.java
@@ -80,6 +80,7 @@ public class IPALiteralReplacementTest extends AutomatedTestBase
*/
private void runIPALiteralReplacementTest( String testname, boolean IPA )
{
+ setOutputBuffering(true);
boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
try
@@ -107,5 +108,6 @@ public class IPALiteralReplacementTest extends AutomatedTestBase
{
OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA;
}
+ setOutputBuffering(false);
}
}
\ No newline at end of file
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPANnzPropagationTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPANnzPropagationTest.java
index a57fee2..ffbaf23 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/IPANnzPropagationTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/IPANnzPropagationTest.java
@@ -54,6 +54,7 @@ public class IPANnzPropagationTest extends AutomatedTestBase
private void runIPANnzPropgationTest(String testname)
{
+ setOutputBuffering(true);
// Save old settings
boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
ExecMode platformOld = rtplatform;
@@ -79,5 +80,6 @@ public class IPANnzPropagationTest extends AutomatedTestBase
DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
rtplatform = platformOld;
}
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarRecursionTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarRecursionTest.java
index c89e7db..a216cb8 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarRecursionTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarRecursionTest.java
@@ -42,6 +42,7 @@ public class IPAScalarRecursionTest extends AutomatedTestBase
@Test
public void testScalarRecursion()
{
+ setOutputBuffering(true);
String TEST_NAME = TEST_NAME1;
try
@@ -59,5 +60,6 @@ public class IPAScalarRecursionTest extends AutomatedTestBase
{
throw new RuntimeException(ex);
}
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarVariablePropagationTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarVariablePropagationTest.java
index 788e1da..533d945 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarVariablePropagationTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarVariablePropagationTest.java
@@ -61,6 +61,8 @@ public class IPAScalarVariablePropagationTest extends AutomatedTestBase
*/
private void runIPAScalarVariablePropagationTest( String testname, boolean IPA )
{
+
+ setOutputBuffering(true);
boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
try
@@ -87,5 +89,7 @@ public class IPAScalarVariablePropagationTest extends AutomatedTestBase
finally {
OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA;
}
+
+ setOutputBuffering(false);
}
}
\ No newline at end of file
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPAUnknownRecursionTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPAUnknownRecursionTest.java
index 24a628d..4d0c180 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/IPAUnknownRecursionTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/IPAUnknownRecursionTest.java
@@ -63,6 +63,7 @@ public class IPAUnknownRecursionTest extends AutomatedTestBase
*/
private void runIPAUnknownRecursionTest( boolean IPA )
{
+ setOutputBuffering(true);
boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
try
@@ -91,5 +92,6 @@ public class IPAUnknownRecursionTest extends AutomatedTestBase
{
OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA;
}
+ setOutputBuffering(false);
}
}
\ No newline at end of file
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ListAndStructTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ListAndStructTest.java
index 4f7c31a..03598f3 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/ListAndStructTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/ListAndStructTest.java
@@ -163,6 +163,7 @@ public class ListAndStructTest extends AutomatedTestBase
private void runListStructTest(String testname, boolean rewrites)
{
+ setOutputBuffering(true);
boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
try {
@@ -193,5 +194,6 @@ public class ListAndStructTest extends AutomatedTestBase
finally {
OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag;
}
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ListAppendRemove.java b/src/test/java/org/apache/sysds/test/functions/misc/ListAppendRemove.java
index 946ef67..aa83b4f 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/ListAppendRemove.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/ListAppendRemove.java
@@ -96,6 +96,7 @@ public class ListAppendRemove extends AutomatedTestBase
private void runListAppendRemove(String testname, ExecType type, boolean rewrites, boolean conditional)
{
+ setOutputBuffering(true);
Types.ExecMode platformOld = setExecMode(type);
boolean rewriteOld = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
@@ -132,5 +133,6 @@ public class ListAppendRemove extends AutomatedTestBase
rtplatform = platformOld;
OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewriteOld;
}
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/LongOverflowTest.java b/src/test/java/org/apache/sysds/test/functions/misc/LongOverflowTest.java
index 5423f98..e16cc3d 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/LongOverflowTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/LongOverflowTest.java
@@ -92,6 +92,7 @@ public class LongOverflowTest extends AutomatedTestBase
*/
private void runOverflowTest( String testscript, boolean error )
{
+ setOutputBuffering(true);
String TEST_NAME = testscript;
try
@@ -113,5 +114,6 @@ public class LongOverflowTest extends AutomatedTestBase
{
throw new RuntimeException(ex);
}
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/NegativeLoopIncrementsTest.java b/src/test/java/org/apache/sysds/test/functions/misc/NegativeLoopIncrementsTest.java
index 7ec6db2..4ab4063 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/NegativeLoopIncrementsTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/NegativeLoopIncrementsTest.java
@@ -21,12 +21,12 @@ package org.apache.sysds.test.functions.misc;
import java.util.HashMap;
-import org.junit.Test;
import org.apache.sysds.runtime.matrix.data.MatrixValue.CellIndex;
import org.apache.sysds.runtime.meta.MatrixCharacteristics;
import org.apache.sysds.test.AutomatedTestBase;
import org.apache.sysds.test.TestConfiguration;
import org.apache.sysds.test.TestUtils;
+import org.junit.Test;
/**
*
@@ -91,6 +91,7 @@ public class NegativeLoopIncrementsTest extends AutomatedTestBase
private void runNegativeLoopIncrementsTest( String testname, boolean vect, boolean multiStep )
{
+ setOutputBuffering(true);
String TEST_NAME = testname;
TestConfiguration config = getTestConfiguration(TEST_NAME);
loadTestConfiguration(config);
@@ -118,5 +119,6 @@ public class NegativeLoopIncrementsTest extends AutomatedTestBase
//check meta data
checkDMLMetaDataFile("R", new MatrixCharacteristics(1,1,1,1));
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/NrowNcolStringTest.java b/src/test/java/org/apache/sysds/test/functions/misc/NrowNcolStringTest.java
index 4a5c503..cd0865e 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/NrowNcolStringTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/NrowNcolStringTest.java
@@ -69,6 +69,7 @@ public class NrowNcolStringTest extends AutomatedTestBase
*/
private void runNxxStringTest( String testName )
{
+ setOutputBuffering(true);
String TEST_NAME = testName;
try
@@ -86,5 +87,6 @@ public class NrowNcolStringTest extends AutomatedTestBase
{
throw new RuntimeException(ex);
}
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/PrintExpressionTest.java b/src/test/java/org/apache/sysds/test/functions/misc/PrintExpressionTest.java
index b05e5dd..a4ab57c 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/PrintExpressionTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/PrintExpressionTest.java
@@ -70,6 +70,7 @@ public class PrintExpressionTest extends AutomatedTestBase
*/
private void runPrintExpressionTest( String testname, boolean rewrites )
{
+ setOutputBuffering(true);
String TEST_NAME = testname;
TestConfiguration config = getTestConfiguration(TEST_NAME);
loadTestConfiguration(config);
@@ -94,5 +95,6 @@ public class PrintExpressionTest extends AutomatedTestBase
{
OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldRewriteFlag;
}
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/PrintMatrixTest.java b/src/test/java/org/apache/sysds/test/functions/misc/PrintMatrixTest.java
index 9b38e37..a9e3038 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/PrintMatrixTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/PrintMatrixTest.java
@@ -45,6 +45,7 @@ public class PrintMatrixTest extends AutomatedTestBase
private void runTest( String testName, boolean exceptionExpected )
{
+ setOutputBuffering(true);
TestConfiguration config = getTestConfiguration(TEST_NAME1);
loadTestConfiguration(config);
@@ -54,5 +55,6 @@ public class PrintMatrixTest extends AutomatedTestBase
//run tests
runTest(true, exceptionExpected, DMLException.class, -1);
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ReadAfterWriteTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ReadAfterWriteTest.java
index fdf2bbc..1b2f609 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/ReadAfterWriteTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/ReadAfterWriteTest.java
@@ -105,6 +105,7 @@ public class ReadAfterWriteTest extends AutomatedTestBase
*/
private void runReadAfterWriteTest( String testName, boolean positive )
{
+ setOutputBuffering(true);
String TEST_NAME = testName;
try
@@ -134,5 +135,6 @@ public class ReadAfterWriteTest extends AutomatedTestBase
//cleanup
TestUtils.clearDirectory(outputDir());
}
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ScalarAssignmentTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ScalarAssignmentTest.java
index 95b93cf..b118bd8 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/ScalarAssignmentTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/ScalarAssignmentTest.java
@@ -156,6 +156,7 @@ public class ScalarAssignmentTest extends AutomatedTestBase
*/
public void runScalarAssignmentTest( ControlFlowConstruct cfc, ValueType vt )
{
+ setOutputBuffering(true);
String TEST_NAME = null;
switch( cfc )
{
@@ -188,5 +189,6 @@ public class ScalarAssignmentTest extends AutomatedTestBase
boolean exceptionExpected = (cfc==ControlFlowConstruct.PARFOR)? true : false; //dependency analysis
int expectedNumberOfJobs = -1;
runTest(true, exceptionExpected, LanguageException.class, expectedNumberOfJobs);
+ setOutputBuffering(false);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ZeroRowsColsMatrixTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ZeroRowsColsMatrixTest.java
index a5bbd98..fb22b62 100644
--- a/src/test/java/org/apache/sysds/test/functions/misc/ZeroRowsColsMatrixTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/misc/ZeroRowsColsMatrixTest.java
@@ -160,6 +160,7 @@ public class ZeroRowsColsMatrixTest extends AutomatedTestBase
private void runEmptyMatrixTest( String testname, boolean rewrites, boolean emptyRet, ExecType et )
{
+ setOutputBuffering(true);
ExecMode platformOld = rtplatform;
switch( et ){
case SPARK: rtplatform = ExecMode.SPARK; break;
@@ -205,5 +206,6 @@ public class ZeroRowsColsMatrixTest extends AutomatedTestBase
rtplatform = platformOld;
DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
}
+ setOutputBuffering(false);
}
}
diff --git a/src/test/scripts/functions/misc/IPANnzPropagation1.dml b/src/test/scripts/functions/misc/IPANnzPropagation1.dml
index 711302a..3884c08 100644
--- a/src/test/scripts/functions/misc/IPANnzPropagation1.dml
+++ b/src/test/scripts/functions/misc/IPANnzPropagation1.dml
@@ -24,7 +24,7 @@ foo = function(matrix[double] X) return (double sum) {
sum = sum(X);
}
-X = rand(rows=1000, cols=1000000000, sparsity=1e-6)
+X = rand(rows=1000, cols=100000000, sparsity=1e-6)
s1 = foo(X);
s2 = foo(X);
print(s1+" "+s2);
diff --git a/src/test/scripts/functions/misc/IPANnzPropagation2.dml b/src/test/scripts/functions/misc/IPANnzPropagation2.dml
index de0a1df..fc4b213 100644
--- a/src/test/scripts/functions/misc/IPANnzPropagation2.dml
+++ b/src/test/scripts/functions/misc/IPANnzPropagation2.dml
@@ -24,8 +24,8 @@ foo = function(matrix[double] X) return (double sum) {
sum = sum(X);
}
-X = rand(rows=1000, cols=1000000000, sparsity=1e-6)
+X = rand(rows=1000, cols=100000000, sparsity=1e-6)
s1 = foo(X);
-X = rand(rows=1000, cols=1000000000, sparsity=1e-7)
+X = rand(rows=1000, cols=100000000, sparsity=1e-7)
s2 = foo(X);
print(s1+" "+s2);
[systemds] 01/04: [SYSTEMDS-2897] CLA decompressing write
Posted by ba...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 2bc40ed35e9be42e8442ca5227c61ec10220b7d4
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Sat Mar 13 11:00:01 2021 +0100
[SYSTEMDS-2897] CLA decompressing write
---
.../runtime/compress/CompressedMatrixBlock.java | 58 ++++++++++------------
.../sysds/runtime/compress/colgroup/AColGroup.java | 2 +
.../runtime/compress/colgroup/ADictionary.java | 2 +
.../compress/colgroup/ColGroupUncompressed.java | 5 ++
.../runtime/compress/colgroup/ColGroupValue.java | 6 +++
.../runtime/compress/colgroup/Dictionary.java | 16 ++++++
.../runtime/compress/colgroup/QDictionary.java | 16 ++++++
.../sysds/runtime/compress/lib/CLALibAppend.java | 9 ++--
.../sysds/runtime/compress/lib/CLALibCompAgg.java | 2 +-
.../runtime/compress/lib/CLALibLeftMultBy.java | 50 +++----------------
.../sysds/runtime/compress/lib/CLALibReExpand.java | 2 +-
.../runtime/compress/lib/CLALibRightMultBy.java | 4 +-
.../sysds/runtime/compress/lib/CLALibScalar.java | 6 ++-
.../sysds/runtime/compress/lib/CLALibSquash.java | 2 +-
.../controlprogram/caching/CacheableData.java | 2 +-
.../controlprogram/caching/MatrixObject.java | 7 ++-
.../apache/sysds/runtime/util/DataConverter.java | 7 ++-
17 files changed, 107 insertions(+), 89 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
index f2958da..58688b1 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
@@ -205,32 +205,16 @@ public class CompressedMatrixBlock extends MatrixBlock {
// preallocation sparse rows to avoid repeated reallocations
MatrixBlock ret = new MatrixBlock(rlen, clen, false, -1);
+ if(nonZeros == -1)
+ ret.setNonZeros(this.recomputeNonZeros());
+ else
+ ret.setNonZeros(nonZeros);
ret.allocateDenseBlock();
- // (nonZeros == -1) ?
- // .allocateBlock() : new MatrixBlock(rlen, clen, sparse,
- // nonZeros).allocateBlock();
-
- // if(ret.isInSparseFormat()) {
- // int[] rnnz = new int[rlen];
- // // for(ColGroup grp : _colGroups)
- // // grp.countNonZerosPerRow(rnnz, 0, rlen);
- // ret.allocateSparseRowsBlock();
- // SparseBlock rows = ret.getSparseBlock();
- // for(int i = 0; i < rlen; i++)
- // rows.allocate(i, rnnz[i]);
- // }
+ // todo Add sparse decompress.
- // core decompression (append if sparse)
for(AColGroup grp : _colGroups)
grp.decompressToBlockUnSafe(ret, 0, rlen, 0, grp.getValues());
- // post-processing (for append in decompress)
- if(ret.getNonZeros() == -1 || nonZeros == -1) {
- ret.recomputeNonZeros();
- }
- else {
- ret.setNonZeros(nonZeros);
- }
if(ret.isInSparseFormat())
ret.sortSparseRows();
@@ -256,8 +240,10 @@ public class CompressedMatrixBlock extends MatrixBlock {
Timing time = new Timing(true);
MatrixBlock ret = new MatrixBlock(rlen, clen, false, -1).allocateBlock();
-
- nonZeros = 0;
+ if(nonZeros == -1)
+ ret.setNonZeros(this.recomputeNonZeros());
+ else
+ ret.setNonZeros(nonZeros);
boolean overlapping = isOverlapping();
try {
ExecutorService pool = CommonThreadPool.get(k);
@@ -272,20 +258,13 @@ public class CompressedMatrixBlock extends MatrixBlock {
List<Future<Long>> rtasks = pool.invokeAll(tasks);
pool.shutdown();
for(Future<Long> rt : rtasks)
- nonZeros += rt.get(); // error handling
+ rt.get(); // error handling
}
catch(InterruptedException | ExecutionException ex) {
LOG.error("Parallel decompression failed defaulting to non parallel implementation " + ex.getMessage());
- nonZeros = -1;
ex.printStackTrace();
return decompress();
}
- if(overlapping) {
- ret.recomputeNonZeros();
- }
- else {
- ret.setNonZeros(nonZeros);
- }
if(DMLScript.STATISTICS || LOG.isDebugEnabled()) {
double t = time.stop();
@@ -299,6 +278,22 @@ public class CompressedMatrixBlock extends MatrixBlock {
return CLALibSquash.squash(this, k);
}
+ @Override
+ public long recomputeNonZeros() {
+ if(overlappingColGroups) {
+ nonZeros = clen * rlen;
+ }
+ else {
+ long nnz = 0;
+ for(AColGroup g : _colGroups) {
+ nnz += g.getNumberNonZeros();
+ }
+ nonZeros = nnz;
+ }
+ return nonZeros;
+
+ }
+
/**
* Obtain an upper bound on the memory used to store the compressed block.
*
@@ -497,6 +492,7 @@ public class CompressedMatrixBlock extends MatrixBlock {
CLALibLeftMultBy.leftMultByMatrixTransposed(this, tmp, out, k);
out = LibMatrixReorg.transposeInPlace(out, k);
+ out.recomputeNonZeros();
return out;
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java
index 808d43b..33e9d11 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java
@@ -646,6 +646,8 @@ public abstract class AColGroup implements Serializable {
public abstract boolean containsValue(double pattern);
+ public abstract long getNumberNonZeros();
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictionary.java
index 8422a3f..10b069e 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictionary.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictionary.java
@@ -250,4 +250,6 @@ public abstract class ADictionary {
public abstract ADictionary reExpandColumns(int max);
public abstract boolean containsValue(double pattern);
+
+ public abstract long getNumberNonZeros(int[] counts, int nCol);
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
index 824ff8c..1722ffd 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
@@ -543,4 +543,9 @@ public class ColGroupUncompressed extends AColGroup {
public boolean containsValue(double pattern){
return _data.containsValue(pattern);
}
+
+ @Override
+ public long getNumberNonZeros(){
+ return _data.getNonZeros();
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupValue.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupValue.java
index d0a6ee9..085dd1d 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupValue.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupValue.java
@@ -1030,4 +1030,10 @@ public abstract class ColGroupValue extends AColGroup implements Cloneable {
public boolean containsValue(double pattern){
return _dict.containsValue(pattern);
}
+
+ @Override
+ public long getNumberNonZeros(){
+ int[] counts = getCounts();
+ return _dict.getNumberNonZeros(counts, _colIndexes.length);
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/Dictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/Dictionary.java
index 658decb..34fea3d 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/Dictionary.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/Dictionary.java
@@ -387,4 +387,20 @@ public class Dictionary extends ADictionary {
return false;
}
+
+ @Override
+ public long getNumberNonZeros(int[] counts, int nCol){
+ long nnz = 0;
+ final int nRow = _values.length / nCol;
+ for(int i = 0; i < nRow; i++){
+ long rowCount = 0;
+ final int off = i * nCol;
+ for(int j = off; j < off + nCol; j++){
+ if(_values[j] != 0)
+ rowCount ++;
+ }
+ nnz += rowCount * counts[i];
+ }
+ return nnz;
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/QDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/QDictionary.java
index 05b1817..8986d71 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/QDictionary.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/QDictionary.java
@@ -452,4 +452,20 @@ public class QDictionary extends ADictionary {
return false;
throw new NotImplementedException("Not contains value on Q Dictionary");
}
+
+ @Override
+ public long getNumberNonZeros(int[] counts, int nCol){
+ long nnz = 0;
+ final int nRow = _values.length / nCol;
+ for(int i = 0; i < nRow; i++){
+ long rowCount = 0;
+ final int off = i * nCol;
+ for(int j = off; j < off + nCol; j++){
+ if(_values[j] != 0)
+ rowCount ++;
+ }
+ nnz += rowCount * counts[i];
+ }
+ return nnz;
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java
index c6bf61d..2ea5397 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java
@@ -42,9 +42,7 @@ public class CLALibAppend {
// return left;
final int m = left.getNumRows();
final int n = left.getNumColumns() + right.getNumColumns();
- long nnz = left.getNonZeros() + right.getNonZeros();
- if(left.getNonZeros() < 0 || right.getNonZeros() < 0)
- nnz = -1;
+
// try to compress both sides (if not already compressed).
if(!(left instanceof CompressedMatrixBlock) && m > 1000){
@@ -85,8 +83,11 @@ public class CLALibAppend {
ret.getColGroups().add(tmp);
}
+ long nnzl = (leftC.getNonZeros() <= -1 ) ? leftC.recomputeNonZeros() : leftC.getNonZeros() ;
+ long nnzr = (rightC.getNonZeros() <= -1 ) ? rightC.recomputeNonZeros() : rightC.getNonZeros() ;
+
// meta data maintenance
- ret.setNonZeros(nnz);
+ ret.setNonZeros(nnzl + nnzr);
return ret;
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCompAgg.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCompAgg.java
index 2eb6888..f7b288d 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCompAgg.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCompAgg.java
@@ -56,7 +56,7 @@ public class CLALibCompAgg {
// private static final Log LOG = LogFactory.getLog(LibCompAgg.class.getName());
// private static final long MIN_PAR_AGG_THRESHOLD = 8 * 1024 * 1024;
- private static final long MIN_PAR_AGG_THRESHOLD = 8;
+ private static final long MIN_PAR_AGG_THRESHOLD = 8 * 1024 ;
private static ThreadLocal<MatrixBlock> memPool = new ThreadLocal<MatrixBlock>() {
@Override
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibLeftMultBy.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibLeftMultBy.java
index f6390c6..3ef03f3 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibLeftMultBy.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibLeftMultBy.java
@@ -59,7 +59,9 @@ public class CLALibLeftMultBy {
public static MatrixBlock leftMultByMatrixTransposed(CompressedMatrixBlock m1, MatrixBlock m2, MatrixBlock ret, int k){
MatrixBlock transposed = new MatrixBlock(m2.getNumColumns(), m2.getNumRows(), false);
LibMatrixReorg.transpose(m2, transposed);
- return leftMultByMatrix(m1, transposed, ret, k );
+ ret = leftMultByMatrix(m1, transposed, ret, k );
+ ret.recomputeNonZeros();
+ return ret;
// return LibMatrixReorg.transpose(ret, new MatrixBlock(ret.getNumColumns(), ret.getNumRows(), false));
}
@@ -75,8 +77,10 @@ public class CLALibLeftMultBy {
public static MatrixBlock leftMultByMatrix(CompressedMatrixBlock m1, MatrixBlock m2, MatrixBlock ret, int k) {
prepareReturnMatrix(m1, m2, ret, false);
- return leftMultByMatrix(m1
+ ret = leftMultByMatrix(m1
.getColGroups(), m2, ret, false, m1.getNumColumns(), m1.isOverlapping(), k, m1.getMaxNumValues());
+ ret.recomputeNonZeros();
+ return ret;
}
private static MatrixBlock leftMultByMatrix(List<AColGroup> groups, MatrixBlock that, MatrixBlock ret,
@@ -172,48 +176,6 @@ public class CLALibLeftMultBy {
}
}
- // public static MatrixBlock leftMultByVectorTranspose(List<AColGroup> colGroups, MatrixBlock vector,
- // MatrixBlock result, boolean doTranspose, int k, Pair<Integer, int[]> v, boolean overlap) {
-
- // // transpose vector if required
- // MatrixBlock rowVector = vector;
- // if(doTranspose) {
- // rowVector = new MatrixBlock(1, vector.getNumRows(), false);
- // LibMatrixReorg.transpose(vector, rowVector);
- // }
-
- // result.reset();
- // result.allocateDenseBlock();
-
- // // multi-threaded execution
- // try {
- // // compute uncompressed column group in parallel
- // // ColGroupUncompressed uc = getUncompressedColGroup();
- // // if(uc != null)
- // // uc.leftMultByRowVector(rowVector, result, k);
-
- // // compute remaining compressed column groups in parallel
- // ExecutorService pool = CommonThreadPool.get(Math.min(colGroups.size(), k));
- // ArrayList<LeftMatrixVectorMultTask> tasks = new ArrayList<>();
-
- // tasks.add(new LeftMatrixVectorMultTask(colGroups, rowVector, result, v));
-
- // List<Future<Object>> ret = pool.invokeAll(tasks);
- // pool.shutdown();
- // for(Future<Object> tmp : ret)
- // tmp.get();
-
- // }
- // catch(InterruptedException | ExecutionException e) {
- // throw new DMLRuntimeException(e);
- // }
-
- // // post-processing
- // result.recomputeNonZeros();
-
- // return result;
- // }
-
private static MatrixBlock leftMultByCompressedTransposedMatrix(List<AColGroup> colGroups,
CompressedMatrixBlock that, MatrixBlock ret, int k, int numColumns, Pair<Integer, int[]> v,
boolean overlapping) {
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibReExpand.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibReExpand.java
index 97cf128..2b4e318 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibReExpand.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibReExpand.java
@@ -68,8 +68,8 @@ public class CLALibReExpand {
ret.allocateColGroupList(newColGroups);
ret.setOverlapping(true);
- ret.setNonZeros(-1);
+ ret.recomputeNonZeros();
return ret;
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java
index 8ce20f2..ec28497 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java
@@ -42,7 +42,9 @@ public class CLALibRightMultBy {
private static final Log LOG = LogFactory.getLog(CLALibRightMultBy.class.getName());
public static MatrixBlock rightMultByMatrix(CompressedMatrixBlock m1, MatrixBlock m2, MatrixBlock ret, int k, boolean allowOverlap){
- return rightMultByMatrix(m1.getColGroups(), m2, ret, k, m1.getMaxNumValues(), allowOverlap);
+ ret = rightMultByMatrix(m1.getColGroups(), m2, ret, k, m1.getMaxNumValues(), allowOverlap);
+ ret.recomputeNonZeros();
+ return ret;
}
private static MatrixBlock rightMultByMatrix(List<AColGroup> colGroups, MatrixBlock that, MatrixBlock ret, int k,
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibScalar.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibScalar.java
index 97a7f67..1c0a980 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibScalar.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibScalar.java
@@ -57,7 +57,9 @@ public class CLALibScalar {
public static MatrixBlock scalarOperations(ScalarOperator sop, CompressedMatrixBlock m1, MatrixValue result) {
// Special case handling of overlapping relational operations
if(CLALibRelationalOp.isValidForRelationalOperation(sop, m1)) {
- return CLALibRelationalOp.overlappingRelativeRelationalOperation(sop, m1);
+ MatrixBlock ret = CLALibRelationalOp.overlappingRelativeRelationalOperation(sop, m1);
+ ret.recomputeNonZeros();
+ return ret;
}
if(isInvalidForCompressedOutput(m1, sop)) {
@@ -96,7 +98,7 @@ public class CLALibScalar {
ret.setOverlapping(m1.isOverlapping());
}
- ret.setNonZeros(-1);
+ ret.recomputeNonZeros();
return ret;
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSquash.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSquash.java
index b4438b2..63177c5 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSquash.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSquash.java
@@ -59,7 +59,7 @@ public class CLALibSquash {
ret.allocateColGroupList(retCg);
ret.setOverlapping(false);
- ret.setNonZeros(-1);
+ ret.recomputeNonZeros();
if(ret.isOverlapping())
throw new DMLCompressionException("Squash should output compressed nonOverlapping matrix");
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java
index 06de8f7..cf6ab3f 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java
@@ -1040,7 +1040,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data
protected void writeMetaData (String filePathAndName, String outputFormat, FileFormatProperties formatProperties)
throws IOException
- {
+ {
MetaDataFormat iimd = (MetaDataFormat) _metaData;
if (iimd == null)
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java
index d0bce6e..e55509b 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java
@@ -35,6 +35,7 @@ import org.apache.sysds.conf.ConfigurationManager;
import org.apache.sysds.hops.OptimizerUtils;
import org.apache.sysds.lops.Lop;
import org.apache.sysds.runtime.DMLRuntimeException;
+import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
import org.apache.sysds.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
import org.apache.sysds.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysds.runtime.controlprogram.federated.FederatedRange;
@@ -583,13 +584,15 @@ public class MatrixObject extends CacheableData<MatrixBlock>
begin = System.currentTimeMillis();
}
- MetaDataFormat iimd = (MetaDataFormat) _metaData;
-
if(this.isFederated() && FileFormat.safeValueOf(ofmt) == FileFormat.FEDERATED){
ReaderWriterFederated.write(fname,this._fedMapping);
}
else if (_data != null)
{
+ if(_data instanceof CompressedMatrixBlock)
+ _data = CompressedMatrixBlock.getUncompressed(_data);
+
+ MetaDataFormat iimd = (MetaDataFormat) _metaData;
// Get the dimension information from the metadata stored within MatrixObject
DataCharacteristics mc = iimd.getDataCharacteristics();
// Write the matrix to HDFS in requested format
diff --git a/src/main/java/org/apache/sysds/runtime/util/DataConverter.java b/src/main/java/org/apache/sysds/runtime/util/DataConverter.java
index c181b5c..51ad590 100644
--- a/src/main/java/org/apache/sysds/runtime/util/DataConverter.java
+++ b/src/main/java/org/apache/sysds/runtime/util/DataConverter.java
@@ -31,6 +31,8 @@ import java.util.Map.Entry;
import java.util.StringTokenizer;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.commons.math3.linear.Array2DRowRealMatrix;
import org.apache.commons.math3.linear.BlockRealMatrix;
import org.apache.commons.math3.linear.RealMatrix;
@@ -79,7 +81,7 @@ import org.apache.sysds.runtime.meta.DataCharacteristics;
*
*/
public class DataConverter {
- // private static final Log LOG = LogFactory.getLog(DataConverter.class.getName());
+ private static final Log LOG = LogFactory.getLog(DataConverter.class.getName());
private static final String DELIM = " ";
//////////////
@@ -100,6 +102,9 @@ public class DataConverter {
public static void writeMatrixToHDFS(MatrixBlock mat, String dir, FileFormat fmt, DataCharacteristics dc, int replication, FileFormatProperties formatProperties, boolean diag)
throws IOException {
MatrixWriter writer = MatrixWriterFactory.createMatrixWriter( fmt, replication, formatProperties );
+ if(mat instanceof CompressedMatrixBlock)
+ mat = CompressedMatrixBlock.getUncompressed(mat);
+ LOG.error(mat.getNonZeros());
writer.writeMatrixToHDFS(mat, dir, dc.getRows(), dc.getCols(), dc.getBlocksize(), dc.getNonZeros(), diag);
}
[systemds] 02/04: [SYSTEMDS-2898] CLA original size bug
Posted by ba...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 0edc28e2a89225e8c6e07cd33ce1e5ba7d652627
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Sat Mar 13 16:42:08 2021 +0100
[SYSTEMDS-2898] CLA original size bug
The compression took the worng size of the original matrix to base the
compression ratio on. This is because it would transpose the input matrix
and then see the size of this transpoed matrix.
A sparse matrix with few rows is smaller in SparseBlockMCSR
---
src/main/java/org/apache/sysds/conf/DMLConfig.java | 2 +-
.../hops/rewrite/RewriteCompressedReblock.java | 18 ++++---
.../runtime/compress/CompressedMatrixBlock.java | 19 ++++---
.../compress/CompressedMatrixBlockFactory.java | 16 ++++--
.../runtime/compress/CompressionStatistics.java | 1 +
.../compress/estim/CompressedSizeInfoColGroup.java | 2 +-
.../org/apache/sysds/runtime/data/DenseBlock.java | 10 ++++
.../apache/sysds/runtime/data/DenseBlockFP64.java | 7 +++
.../org/apache/sysds/runtime/data/SparseBlock.java | 5 ++
.../apache/sysds/runtime/data/SparseBlockCOO.java | 7 +--
.../apache/sysds/runtime/data/SparseBlockCSR.java | 9 ++--
.../apache/sysds/runtime/data/SparseBlockMCSR.java | 15 ++++--
.../sysds/runtime/matrix/data/MatrixBlock.java | 33 +++++-------
.../apache/sysds/runtime/util/DataConverter.java | 7 +--
.../org/apache/sysds/utils/MemoryEstimates.java | 59 ++++++++++++++++++++--
.../component/compress/CompressedTestBase.java | 14 ++---
.../compress/compressInstructionRewrite.java | 2 +-
.../compress/configuration/CompressLossy.java | 2 +-
.../compress/configuration/CompressLossyCost.java | 4 +-
src/test/resources/log4j.properties | 4 +-
.../SystemDS-config-compress-cost.xml | 2 +-
.../SystemDS-config-compress-cost-OLE.xml | 2 +-
...xml => SystemDS-config-compress-cost-lossy.xml} | 2 +-
.../cost/SystemDS-config-compress-cost.xml | 2 +-
.../{lossy => }/SystemDS-config-compress-lossy.xml | 2 +-
.../compress/force/SystemDS-config-compress.xml | 2 +-
26 files changed, 169 insertions(+), 79 deletions(-)
diff --git a/src/main/java/org/apache/sysds/conf/DMLConfig.java b/src/main/java/org/apache/sysds/conf/DMLConfig.java
index 5f90fb4..74ddf8a 100644
--- a/src/main/java/org/apache/sysds/conf/DMLConfig.java
+++ b/src/main/java/org/apache/sysds/conf/DMLConfig.java
@@ -126,7 +126,7 @@ public class DMLConfig
_defaultVals.put(CP_PARALLEL_IO, "true" );
_defaultVals.put(COMPRESSED_LINALG, Compression.CompressConfig.FALSE.name() );
_defaultVals.put(COMPRESSED_LOSSY, "false" );
- _defaultVals.put(COMPRESSED_VALID_COMPRESSIONS, "SDC,DDC");
+ _defaultVals.put(COMPRESSED_VALID_COMPRESSIONS, "SDC,DDC,RLE,OLE");
_defaultVals.put(COMPRESSED_OVERLAPPING, "true" );
_defaultVals.put(COMPRESSED_SAMPLING_RATIO, "0.01");
_defaultVals.put(COMPRESSED_COCODE, "COST");
diff --git a/src/main/java/org/apache/sysds/hops/rewrite/RewriteCompressedReblock.java b/src/main/java/org/apache/sysds/hops/rewrite/RewriteCompressedReblock.java
index 79c70ef..194a5d7 100644
--- a/src/main/java/org/apache/sysds/hops/rewrite/RewriteCompressedReblock.java
+++ b/src/main/java/org/apache/sysds/hops/rewrite/RewriteCompressedReblock.java
@@ -127,8 +127,10 @@ public class RewriteCompressedReblock extends StatementBlockRewriteRule {
}
private static boolean satisfiesSizeConstraintsForCompression(Hop hop) {
- return hop.getDim2() >= 1 &&
- ((hop.getDim1() >= 1000 && hop.getDim2() < 100) || hop.getDim1() / hop.getDim2() >= 1000);
+ if(hop.getDim2() >= 1) {
+ return (hop.getDim1() >= 1000 && hop.getDim2() < 100) || hop.getDim1() / hop.getDim2() >= 75;
+ }
+ return false;
}
private static boolean satisfiesCompressionCondition(Hop hop) {
@@ -191,8 +193,11 @@ public class RewriteCompressedReblock extends StatementBlockRewriteRule {
}
private static boolean satisfiesCostCompressionCondition(Hop hop, DMLProgram prog) {
- return satisfiesAggressiveCompressionCondition(hop) && hop.dimsKnown(false) &&
- analyseProgram(hop, prog).isValidAggressiveCompression();
+ boolean satisfies = true;
+ satisfies &= satisfiesAggressiveCompressionCondition(hop);
+ satisfies &= hop.dimsKnown(false);
+ satisfies &= analyseProgram(hop, prog).isValidAggressiveCompression();
+ return satisfies;
}
@@ -363,7 +368,6 @@ public class RewriteCompressedReblock extends StatementBlockRewriteRule {
private void handleApplicableOps(Hop current) {
// Valid with uncompressed outputs
boolean compUCOut = false;
- LOG.error(current);
// // tsmm
// compUCOut |= (current instanceof AggBinaryOp && current.getDim2() <= current.getBlocksize() &&
// ((AggBinaryOp) current).checkTransposeSelf() == MMTSJType.LEFT);
@@ -403,7 +407,7 @@ public class RewriteCompressedReblock extends StatementBlockRewriteRule {
boolean metaOp = HopRewriteUtils.isUnary(current, OpOp1.NROW, OpOp1.NCOL);
boolean ctableOp = HopRewriteUtils.isTernary(current, OpOp3.CTABLE);
- if(ctableOp){
+ if(ctableOp) {
numberCompressedOpsExecuted += 4;
compCOut = true;
}
@@ -431,7 +435,7 @@ public class RewriteCompressedReblock extends StatementBlockRewriteRule {
if(LOG.isDebugEnabled())
LOG.debug(this.toString());
return (inefficientSupportedOpsExecuted < numberCompressedOpsExecuted) &&
- (usedInLoop || numberCompressedOpsExecuted > 3) && numberDecompressedOpsExecuted < 1;
+ (usedInLoop || numberCompressedOpsExecuted > 3) && numberDecompressedOpsExecuted < 1;
}
@Override
diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
index 58688b1..828610b 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
@@ -205,10 +205,7 @@ public class CompressedMatrixBlock extends MatrixBlock {
// preallocation sparse rows to avoid repeated reallocations
MatrixBlock ret = new MatrixBlock(rlen, clen, false, -1);
- if(nonZeros == -1)
- ret.setNonZeros(this.recomputeNonZeros());
- else
- ret.setNonZeros(nonZeros);
+
ret.allocateDenseBlock();
// todo Add sparse decompress.
@@ -218,6 +215,11 @@ public class CompressedMatrixBlock extends MatrixBlock {
if(ret.isInSparseFormat())
ret.sortSparseRows();
+ if(nonZeros == -1)
+ ret.setNonZeros(this.recomputeNonZeros());
+ else
+ ret.setNonZeros(nonZeros);
+
if(DMLScript.STATISTICS || LOG.isDebugEnabled()) {
double t = time.stop();
LOG.debug("decompressed block w/ k=" + 1 + " in " + t + "ms.");
@@ -240,11 +242,12 @@ public class CompressedMatrixBlock extends MatrixBlock {
Timing time = new Timing(true);
MatrixBlock ret = new MatrixBlock(rlen, clen, false, -1).allocateBlock();
+ ret.allocateDenseBlock();
if(nonZeros == -1)
ret.setNonZeros(this.recomputeNonZeros());
else
ret.setNonZeros(nonZeros);
- boolean overlapping = isOverlapping();
+
try {
ExecutorService pool = CommonThreadPool.get(k);
int rlen = getNumRows();
@@ -254,7 +257,7 @@ public class CompressedMatrixBlock extends MatrixBlock {
ArrayList<DecompressTask> tasks = new ArrayList<>();
for(int i = 0; i < k & i * blklen < getNumRows(); i++)
tasks.add(
- new DecompressTask(_colGroups, ret, i * blklen, Math.min((i + 1) * blklen, rlen), overlapping));
+ new DecompressTask(_colGroups, ret, i * blklen, Math.min((i + 1) * blklen, rlen), overlappingColGroups));
List<Future<Long>> rtasks = pool.invokeAll(tasks);
pool.shutdown();
for(Future<Long> rt : rtasks)
@@ -271,6 +274,7 @@ public class CompressedMatrixBlock extends MatrixBlock {
LOG.debug("decompressed block w/ k=" + k + " in " + time.stop() + "ms.");
DMLCompressionStatistics.addDecompressTime(t, k);
}
+
return ret;
}
@@ -807,7 +811,7 @@ public class CompressedMatrixBlock extends MatrixBlock {
tmp = new MatrixBlock(ru + 1 - rl, getNumColumns(), false).allocateDenseBlock();
for(AColGroup g : getColGroups())
g.decompressToBlock(tmp, rl, ru + 1, 0);
-
+ tmp.recomputeNonZeros();
return tmp;
}
else {
@@ -821,6 +825,7 @@ public class CompressedMatrixBlock extends MatrixBlock {
// this is fine.
tmp = tmp.slice(rl, ru, 0, tmp.getNumColumns() - 1, ret);
}
+ tmp.recomputeNonZeros();
ret = tmp;
return tmp;
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java
index 0fd05e4..0cc75af 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java
@@ -127,6 +127,7 @@ public class CompressedMatrixBlockFactory {
if(res == null)
return abortCompression();
+ res.recomputeNonZeros();
return new ImmutablePair<>(res, _stats);
}
@@ -190,12 +191,15 @@ public class CompressedMatrixBlockFactory {
mb.cleanupBlock(true, true);
_stats.size = res.estimateCompressedSizeInMemory();
- _stats.originalSize = mb.estimateSizeInMemory();
+ _stats.originalSize = original.estimateSizeInMemory();
+ _stats.denseSize = MatrixBlock.estimateSizeInMemory(original.getNumRows(), original.getNumColumns(), 1.0);
_stats.ratio = _stats.originalSize / (double) _stats.size;
if(_stats.ratio < 1) {
- LOG.info("--compressed size: " + _stats.size);
- LOG.info("--compression ratio: " + _stats.ratio);
+ LOG.info("--dense size: " + _stats.denseSize);
+ LOG.info("--original size: " + _stats.originalSize);
+ LOG.info("--compressed size: " + _stats.size);
+ LOG.info("--compression ratio: " + _stats.ratio );
LOG.info("Abort block compression because compression ratio is less than 1.");
res = null;
return;
@@ -239,8 +243,10 @@ public class CompressedMatrixBlockFactory {
LOG.debug("--compression phase " + phase + " Cleanup : " + _stats.getLastTimePhase());
LOG.debug("--col groups types " + _stats.getGroupsTypesString());
LOG.debug("--col groups sizes " + _stats.getGroupsSizesString());
- LOG.debug("--compressed size: " + _stats.size);
- LOG.debug("--compression ratio: " + _stats.ratio);
+ LOG.debug("--dense size: " + _stats.denseSize);
+ LOG.debug("--original size: " + _stats.originalSize);
+ LOG.debug("--compressed size: " + _stats.size);
+ LOG.debug("--compression ratio: " + _stats.ratio );
int[] lengths = new int[res.getColGroups().size()];
int i = 0;
for(AColGroup colGroup : res.getColGroups()) {
diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressionStatistics.java b/src/main/java/org/apache/sysds/runtime/compress/CompressionStatistics.java
index 83fadd2..41ac2a0 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/CompressionStatistics.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/CompressionStatistics.java
@@ -31,6 +31,7 @@ public class CompressionStatistics {
private double lastPhase;
public double ratio;
public long originalSize;
+ public long denseSize;
public long estimatedSizeColGroups;
public long estimatedSizeCols;
public long size;
diff --git a/src/main/java/org/apache/sysds/runtime/compress/estim/CompressedSizeInfoColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/estim/CompressedSizeInfoColGroup.java
index ebe5566..7ae813f 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/estim/CompressedSizeInfoColGroup.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/estim/CompressedSizeInfoColGroup.java
@@ -122,7 +122,7 @@ public class CompressedSizeInfoColGroup {
case UNCOMPRESSED:
return ColGroupSizes.estimateInMemorySizeUncompressed(fact.numRows,
fact.numCols,
- ((double) fact.numVals / (fact.numRows * fact.numCols)));
+ ((double) fact.numOffs / (fact.numRows * fact.numCols)));
case SDC:
if(fact.numOffs == 1)
return ColGroupSizes.estimateInMemorySizeSDCSingle(fact.numCols,
diff --git a/src/main/java/org/apache/sysds/runtime/data/DenseBlock.java b/src/main/java/org/apache/sysds/runtime/data/DenseBlock.java
index b7d09e3..0d414fa 100644
--- a/src/main/java/org/apache/sysds/runtime/data/DenseBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/data/DenseBlock.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.instructions.cp.KahanObject;
import org.apache.sysds.runtime.util.UtilFunctions;
+import org.apache.sysds.utils.MemoryEstimates;
/**
* This DenseBlock is an abstraction for different dense, row-major
@@ -674,4 +675,13 @@ public abstract class DenseBlock implements Serializable
}
return ret;
}
+
+ public static long estimateSizeDenseInMemory(int nRows, int nCols){
+ long size = 16; // object
+ size += 4; // int
+ size += 4; // padding
+ size += MemoryEstimates.intArrayCost(1); // odims typically 1
+ size += 8; // pointer to reuse that is typically null;
+ return size;
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64.java b/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64.java
index 3d38f55..795bee1 100644
--- a/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64.java
+++ b/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64.java
@@ -21,6 +21,7 @@
package org.apache.sysds.runtime.data;
import org.apache.sysds.runtime.util.UtilFunctions;
+import org.apache.sysds.utils.MemoryEstimates;
import java.util.Arrays;
@@ -192,4 +193,10 @@ public class DenseBlockFP64 extends DenseBlockDRB
public long getLong(int[] ix) {
return UtilFunctions.toLong(_data[pos(ix)]);
}
+
+ public static long estimateSizeDenseInMemory(int nRows, int nCols){
+ long size = DenseBlock.estimateSizeDenseInMemory(nRows, nCols);// pointer to reuse that is typically null;
+ size += MemoryEstimates.doubleArrayCost(nRows * nCols);
+ return size;
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlock.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlock.java
index d876946..4375cad 100644
--- a/src/main/java/org/apache/sysds/runtime/data/SparseBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlock.java
@@ -22,6 +22,8 @@ package org.apache.sysds.runtime.data;
import java.io.Serializable;
import java.util.Iterator;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.sysds.runtime.matrix.data.IJV;
/**
@@ -37,6 +39,9 @@ import org.apache.sysds.runtime.matrix.data.IJV;
*/
public abstract class SparseBlock implements Serializable
{
+
+ protected static final Log LOG = LogFactory.getLog(SparseBlock.class.getName());
+
private static final long serialVersionUID = -5008747088111141395L;
//internal configuration parameters for all sparse blocks
diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java
index 37199fa..aebf060 100644
--- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java
+++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java
@@ -25,6 +25,7 @@ import java.util.Iterator;
import org.apache.sysds.runtime.matrix.data.IJV;
import org.apache.sysds.runtime.util.SortUtils;
import org.apache.sysds.runtime.util.UtilFunctions;
+import org.apache.sysds.utils.MemoryEstimates;
/**
* SparseBlock implementation that realizes a traditional 'coordinate matrix'
@@ -149,9 +150,9 @@ public class SparseBlockCOO extends SparseBlock
//32B overhead per array, int/int/double arr in nnz
double size = 16 + 8; //object + 2 int fields
- size += 24 + lnnz * 4d; //rindexes array (row indexes)
- size += 24 + lnnz * 4d; //cindexes array (column indexes)
- size += 24 + lnnz * 8d; //values array (non-zero values)
+ size += MemoryEstimates.intArrayCost((int)lnnz); ; //rindexes array (row indexes)
+ size += MemoryEstimates.intArrayCost((int) lnnz); ; //cindexes array (column indexes)
+ size += MemoryEstimates.doubleArrayCost((int) lnnz);; //values array (non-zero values)
//robustness for long overflows
return (long) Math.min(size, Long.MAX_VALUE);
diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCSR.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCSR.java
index 621a92a..6cf474f 100644
--- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCSR.java
+++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCSR.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
import org.apache.sysds.runtime.util.SortUtils;
import org.apache.sysds.runtime.util.UtilFunctions;
+import org.apache.sysds.utils.MemoryEstimates;
/**
* SparseBlock implementation that realizes a traditional 'compressed sparse row'
@@ -267,10 +268,10 @@ public class SparseBlockCSR extends SparseBlock
double lnnz = Math.max(INIT_CAPACITY, Math.ceil(sparsity*nrows*ncols));
//32B overhead per array, int arr in nrows, int/double arr in nnz
- double size = 16 + 4; //object + int field
- size += 24 + (nrows+1) * 4d; //ptr array (row pointers)
- size += 24 + lnnz * 4d; //indexes array (column indexes)
- size += 24 + lnnz * 8d; //values array (non-zero values)
+ double size = 16 + 4 + 4; //object + int field + padding
+ size += MemoryEstimates.intArrayCost((int)nrows+1); //ptr array (row pointers)
+ size += MemoryEstimates.intArrayCost((int) lnnz); //indexes array (column indexes)
+ size += MemoryEstimates.doubleArrayCost((int) lnnz);//values array (non-zero values)
//robustness for long overflows
return (long) Math.min(size, Long.MAX_VALUE);
diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java
index 707298e..fda83bf 100644
--- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java
+++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java
@@ -19,6 +19,8 @@
package org.apache.sysds.runtime.data;
+import org.apache.sysds.utils.MemoryEstimates;
+
/**
* SparseBlock implementation that realizes a 'modified compressed sparse row'
* representation, where each compressed row is stored as a separate SparseRow
@@ -98,8 +100,8 @@ public class SparseBlockMCSR extends SparseBlock
* @return memory estimate
*/
public static long estimateMemory(long nrows, long ncols, double sparsity) {
- double cnnz = Math.max(SparseRowVector.initialCapacity, Math.ceil(sparsity*ncols));
- double rlen = Math.min(nrows, Math.ceil(sparsity*nrows*ncols));
+ int cnnz = Math.max(SparseRowVector.initialCapacity, (int) Math.ceil(sparsity*ncols));
+ double rlen = Math.min(nrows, Math.ceil(sparsity*nrows*ncols));
//Each sparse row has a fixed overhead of 16B (object) + 12B (3 ints),
//24B (int array), 24B (double array), i.e., in total 76B
@@ -107,8 +109,13 @@ public class SparseBlockMCSR extends SparseBlock
//Overheads for arrays, objects, and references refer to 64bit JVMs
//If nnz < rows we have guaranteed also empty rows.
double size = 16; //object
- size += 24 + nrows * 8d; //references
- size += rlen * (76 + cnnz * 12); //sparse rows
+ size += MemoryEstimates.objectArrayCost((int)rlen); //references
+ long sparseRowSize = 16; // object
+ sparseRowSize += MemoryEstimates.intArrayCost(cnnz);
+ sparseRowSize += MemoryEstimates.doubleArrayCost(cnnz);
+ sparseRowSize += 4*3; // integers.
+ sparseRowSize += 4; // padding to nearest 8 byte.
+ size += rlen * sparseRowSize; //sparse rows
// robustness for long overflows
return (long) Math.min(size, Long.MAX_VALUE);
diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
index 96dd709..ff8cf31 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
@@ -52,6 +52,7 @@ import org.apache.sysds.runtime.controlprogram.caching.CacheBlock;
import org.apache.sysds.runtime.controlprogram.caching.LazyWriteBuffer;
import org.apache.sysds.runtime.controlprogram.caching.MatrixObject.UpdateType;
import org.apache.sysds.runtime.data.DenseBlock;
+import org.apache.sysds.runtime.data.DenseBlockFP64;
import org.apache.sysds.runtime.data.DenseBlockFactory;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.data.SparseBlockCOO;
@@ -2432,24 +2433,24 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
{
//determine sparse/dense representation
boolean sparse = evalSparseFormatInMemory(nrows, ncols, (long)(sparsity*nrows*ncols));
-
+
+ // basic variables and references sizes
+ long size = 16; // header
+ size += 12; // ints
+ size += 1; // boolean
+ size += 3; // padding
+ size += 8 * 2; // object references
+
//estimate memory consumption for sparse/dense
if( sparse )
- return estimateSizeSparseInMemory(nrows, ncols, sparsity);
+ return size + estimateSizeSparseInMemory(nrows, ncols, sparsity);
else
- return estimateSizeDenseInMemory(nrows, ncols);
+ return size + estimateSizeDenseInMemory(nrows, ncols);
}
public static long estimateSizeDenseInMemory(long nrows, long ncols)
{
- // basic variables and references sizes
- double size = 44;
-
- // core dense matrix block (double array)
- size += 8d * nrows * ncols;
-
- // robustness for long overflows
- return (long) Math.min(size, Long.MAX_VALUE);
+ return (long) Math.min(DenseBlockFP64.estimateSizeDenseInMemory((int)nrows, (int)ncols), Long.MAX_VALUE);
}
public static long estimateSizeSparseInMemory(long nrows, long ncols, double sparsity) {
@@ -2458,15 +2459,9 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
public static long estimateSizeSparseInMemory(long nrows, long ncols, double sparsity, SparseBlock.Type stype)
{
- // basic variables and references sizes
- double size = 44;
-
// delegate memory estimate to individual sparse blocks
- size += SparseBlockFactory.estimateSizeSparseInMemory(
- stype, nrows, ncols, sparsity);
-
- // robustness for long overflows
- return (long) Math.min(size, Long.MAX_VALUE);
+ return Math.min(SparseBlockFactory.estimateSizeSparseInMemory(
+ stype, nrows, ncols, sparsity),Long.MAX_VALUE);
}
public long estimateSizeOnDisk()
diff --git a/src/main/java/org/apache/sysds/runtime/util/DataConverter.java b/src/main/java/org/apache/sysds/runtime/util/DataConverter.java
index 51ad590..c181b5c 100644
--- a/src/main/java/org/apache/sysds/runtime/util/DataConverter.java
+++ b/src/main/java/org/apache/sysds/runtime/util/DataConverter.java
@@ -31,8 +31,6 @@ import java.util.Map.Entry;
import java.util.StringTokenizer;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.commons.math3.linear.Array2DRowRealMatrix;
import org.apache.commons.math3.linear.BlockRealMatrix;
import org.apache.commons.math3.linear.RealMatrix;
@@ -81,7 +79,7 @@ import org.apache.sysds.runtime.meta.DataCharacteristics;
*
*/
public class DataConverter {
- private static final Log LOG = LogFactory.getLog(DataConverter.class.getName());
+ // private static final Log LOG = LogFactory.getLog(DataConverter.class.getName());
private static final String DELIM = " ";
//////////////
@@ -102,9 +100,6 @@ public class DataConverter {
public static void writeMatrixToHDFS(MatrixBlock mat, String dir, FileFormat fmt, DataCharacteristics dc, int replication, FileFormatProperties formatProperties, boolean diag)
throws IOException {
MatrixWriter writer = MatrixWriterFactory.createMatrixWriter( fmt, replication, formatProperties );
- if(mat instanceof CompressedMatrixBlock)
- mat = CompressedMatrixBlock.getUncompressed(mat);
- LOG.error(mat.getNonZeros());
writer.writeMatrixToHDFS(mat, dir, dc.getRows(), dc.getCols(), dc.getBlocksize(), dc.getNonZeros(), diag);
}
diff --git a/src/main/java/org/apache/sysds/utils/MemoryEstimates.java b/src/main/java/org/apache/sysds/utils/MemoryEstimates.java
index c22e47e..473332f 100644
--- a/src/main/java/org/apache/sysds/utils/MemoryEstimates.java
+++ b/src/main/java/org/apache/sysds/utils/MemoryEstimates.java
@@ -21,12 +21,20 @@ package org.apache.sysds.utils;
/**
* Memory Estimates is a helper class containing static classes that estimate the memory requirements of different types
- * of objects in java. All estimates are worst case JVM x86-64bit uncompressed object pointers. This in practice means
- * that the objects are most commonly smaller, for instance the object references are often time (at low memory pressure
- * 4 bits)
+ * of objects in java. All estimates are worst case JVM x86-64bit uncompressed object pointers.
+ *
+ * This in practice means that the objects are most commonly smaller, for instance the object references are often time.
+ *
+ * If the memory pressure is low (there is a low number of allocated objects) then object pointers are 4 bits.
*/
public class MemoryEstimates {
+ /**
+ * Get the worst case memory usage of an java.util.BitSet java object.
+ *
+ * @param length The length of the array.
+ * @return The memory estimate in bytes
+ */
public static long bitSetCost(int length) {
long size = 0;
size += 8; // object reference
@@ -37,6 +45,12 @@ public class MemoryEstimates {
return size;
}
+ /**
+ * Get the worst case memory usage of an array of bytes.
+ *
+ * @param length The length of the array.
+ * @return The memory estimate in bytes
+ */
public static long byteArrayCost(int length) {
long size = 0;
size += 8; // Byte array Reference
@@ -54,6 +68,12 @@ public class MemoryEstimates {
return size;
}
+ /**
+ * Get the worst case memory usage of an array of chars.
+ *
+ * @param length The length of the array.
+ * @return The memory estimate in bytes
+ */
public static long charArrayCost(int length) {
long size = 0;
size += 8; // char array Reference
@@ -71,6 +91,12 @@ public class MemoryEstimates {
return size;
}
+ /**
+ * Get the worst case memory usage of an array of integers.
+ *
+ * @param length The length of the array.
+ * @return The memory estimate in bytes
+ */
public static long intArrayCost(int length) {
long size = 0;
size += 8; // _ptr int[] reference
@@ -87,6 +113,12 @@ public class MemoryEstimates {
return size;
}
+ /**
+ * Get the worst case memory usage of an array of doubles.
+ *
+ * @param length The length of the array.
+ * @return The memory estimate in bytes
+ */
public static long doubleArrayCost(int length) {
long size = 0;
size += 8; // _values double array reference
@@ -96,6 +128,27 @@ public class MemoryEstimates {
return size;
}
+ /**
+ * Get the worst case memory usage for an array of objects.
+ *
+ * @param length The length of the array.
+ * @return The memory estimate in bytes
+ */
+ public static long objectArrayCost(int length) {
+ long size = 0;
+ size += 8; // reference to array
+ size += 20; // header
+ size += 4; // padding before first reference
+ size += 8 * length; // references to all objects.
+ return size;
+ }
+
+ /**
+ * Get the worst case memory usage for an array of longs
+ *
+ * @param length The length of the array.
+ * @return The memory estimate in bytes
+ */
public static long longArrayCost(int length) {
return doubleArrayCost(length);
// exactly the same size as a double array
diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java
index c65ef06..ac51d7b 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java
@@ -19,6 +19,7 @@
package org.apache.sysds.test.component.compress;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
@@ -114,7 +115,7 @@ public abstract class CompressedTestBase extends TestBase {
// .setValidCompressions(EnumSet.of(CompressionType.RLE)).setInvestigateEstimate(true),
new CompressionSettingsBuilder().setSamplingRatio(0.1).setSeed(compressionSeed)
- .setValidCompressions(EnumSet.of(CompressionType.SDC)).setInvestigateEstimate(true),
+ .setValidCompressions(EnumSet.of(CompressionType.SDC)).setInvestigateEstimate(true),
// new CompressionSettingsBuilder().setSamplingRatio(0.1).setSeed(compressionSeed)
// .setValidCompressions(EnumSet.of(CompressionType.SDC, CompressionType.DDC)).setInvestigateEstimate(true),
@@ -123,9 +124,9 @@ public abstract class CompressedTestBase extends TestBase {
// .setInvestigateEstimate(true),
// new CompressionSettingsBuilder().setSamplingRatio(0.1).setSeed(compressionSeed).setTransposeInput("false")
- // .setInvestigateEstimate(true),
+ // .setInvestigateEstimate(true),
// new CompressionSettingsBuilder().setSamplingRatio(0.1).setSeed(compressionSeed).setTransposeInput("true")
- // .setInvestigateEstimate(true),
+ // .setInvestigateEstimate(true),
// new CompressionSettingsBuilder().setSamplingRatio(0.1).setSeed(compressionSeed).setInvestigateEstimate(true),
// new CompressionSettingsBuilder().setSamplingRatio(1.0).setSeed(compressionSeed).setInvestigateEstimate(true)
@@ -335,8 +336,8 @@ public abstract class CompressedTestBase extends TestBase {
// LOG.error(ret1);
// LOG.error(ret2);
// compare result with input
- TestUtils.compareMatricesPercentageDistance(DataConverter.convertToDoubleMatrix(
- ret1), DataConverter.convertToDoubleMatrix(ret2), 0.9, 0.9, this.toString());
+ TestUtils.compareMatricesPercentageDistance(DataConverter
+ .convertToDoubleMatrix(ret1), DataConverter.convertToDoubleMatrix(ret2), 0.9, 0.9, this.toString());
}
catch(Exception e) {
@@ -1025,6 +1026,7 @@ public abstract class CompressedTestBase extends TestBase {
return;
MatrixBlock ret2 = cmb.slice(rl, ru, cl, cu);
MatrixBlock ret1 = mb.slice(rl, ru, cl, cu);
+ assertEquals(ret1.getNonZeros(), ret2.getNonZeros());
compareResultMatrices(ret1, ret2, 1);
}
catch(Exception e) {
@@ -1059,6 +1061,4 @@ public abstract class CompressedTestBase extends TestBase {
compareResultMatrices(d1, d2, toleranceMultiplier);
}
-
-
}
diff --git a/src/test/java/org/apache/sysds/test/functions/compress/compressInstructionRewrite.java b/src/test/java/org/apache/sysds/test/functions/compress/compressInstructionRewrite.java
index 940f4f2..5e0df61 100644
--- a/src/test/java/org/apache/sysds/test/functions/compress/compressInstructionRewrite.java
+++ b/src/test/java/org/apache/sysds/test/functions/compress/compressInstructionRewrite.java
@@ -67,7 +67,7 @@ public class compressInstructionRewrite extends AutomatedTestBase {
@Test
public void testCompressInstruction_02_toSmallToCompress() {
- compressTest(1, 999, 0.2, ExecType.CP, 0, 5, 0, 0, "02");
+ compressTest(1, 74, 0.2, ExecType.CP, 0, 5, 0, 0, "02");
}
@Test
diff --git a/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressLossy.java b/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressLossy.java
index 85ac10a..68da2e5 100644
--- a/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressLossy.java
+++ b/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressLossy.java
@@ -24,7 +24,7 @@ import java.io.File;
public class CompressLossy extends CompressForce {
public String TEST_NAME = "compress";
- public String TEST_DIR = "functions/compress/force/lossy/";
+ public String TEST_DIR = "functions/compress/force/";
public String TEST_CLASS_DIR = TEST_DIR + CompressLossy.class.getSimpleName() + "/";
private String TEST_CONF = "SystemDS-config-compress-lossy.xml";
private File TEST_CONF_FILE = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF);
diff --git a/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressLossyCost.java b/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressLossyCost.java
index 8c7f309..e99e791 100644
--- a/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressLossyCost.java
+++ b/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressLossyCost.java
@@ -24,9 +24,9 @@ import java.io.File;
public class CompressLossyCost extends CompressCost {
public String TEST_NAME = "compress";
- public String TEST_DIR = "functions/compress/cost/lossy";
+ public String TEST_DIR = "functions/compress/cost";
public String TEST_CLASS_DIR = TEST_DIR + CompressLossyCost.class.getSimpleName() + "/";
- private String TEST_CONF = "SystemDS-config-compress-lossy-cost.xml";
+ private String TEST_CONF = "SystemDS-config-compress-cost-lossy.xml";
private File TEST_CONF_FILE = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF);
protected String getTestClassDir() {
diff --git a/src/test/resources/log4j.properties b/src/test/resources/log4j.properties
index 8ea2003..e20439e 100644
--- a/src/test/resources/log4j.properties
+++ b/src/test/resources/log4j.properties
@@ -30,8 +30,8 @@ log4j.logger.org.apache.sysds=ERROR
# log4j.logger.org.apache.sysds.runtime.instructions.fed=DEBUG
# log4j.logger.org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory=DEBUG
# log4j.logger.org.apache.sysds.runtime.compress.cocode=DEBUG
-log4j.logger.org.apache.sysds.hops.rewrite.RewriteCompressedReblock=DEBUG
-log4j.logger.org.apache.sysds.test.functions.compress=DEBUG
+# log4j.logger.org.apache.sysds.hops.rewrite.RewriteCompressedReblock=DEBUG
+# log4j.logger.org.apache.sysds.test.functions.compress=DEBUG
log4j.logger.org.apache.sysds.parser.DataExpression=ERROR
log4j.logger.org.apache.spark=OFF
log4j.logger.org.apache.hadoop=OFF
diff --git a/src/test/scripts/functions/compress/compressInstructionRewrite/SystemDS-config-compress-cost.xml b/src/test/scripts/functions/compress/compressInstructionRewrite/SystemDS-config-compress-cost.xml
index 988774b..bec7a57 100644
--- a/src/test/scripts/functions/compress/compressInstructionRewrite/SystemDS-config-compress-cost.xml
+++ b/src/test/scripts/functions/compress/compressInstructionRewrite/SystemDS-config-compress-cost.xml
@@ -20,5 +20,5 @@
<root>
<sysds.compressed.linalg>cost</sysds.compressed.linalg>
<sysds.cp.parallel.ops>true</sysds.cp.parallel.ops>
- <sysds.scratch>cost_scratch_space</sysds.scratch>
+ <sysds.scratch>target/rewrite_cost_scratch_space</sysds.scratch>
</root>
diff --git a/src/test/scripts/functions/compress/compressScale/SystemDS-config-compress-cost-OLE.xml b/src/test/scripts/functions/compress/compressScale/SystemDS-config-compress-cost-OLE.xml
index 3e35db1..85793f8 100644
--- a/src/test/scripts/functions/compress/compressScale/SystemDS-config-compress-cost-OLE.xml
+++ b/src/test/scripts/functions/compress/compressScale/SystemDS-config-compress-cost-OLE.xml
@@ -20,6 +20,6 @@
<root>
<sysds.compressed.linalg>cost</sysds.compressed.linalg>
<sysds.cp.parallel.ops>true</sysds.cp.parallel.ops>
- <sysds.scratch>target/cost_scale_scratch_space)OLE</sysds.scratch>
+ <sysds.scratch>target/cost_scale_scratch_space_OLE</sysds.scratch>
<sysds.compressed.valid.compressions>OLE</sysds.compressed.valid.compressions>
</root>
diff --git a/src/test/scripts/functions/compress/cost/lossy/SystemDS-config-compress-lossy-cost.xml b/src/test/scripts/functions/compress/cost/SystemDS-config-compress-cost-lossy.xml
similarity index 93%
rename from src/test/scripts/functions/compress/cost/lossy/SystemDS-config-compress-lossy-cost.xml
rename to src/test/scripts/functions/compress/cost/SystemDS-config-compress-cost-lossy.xml
index 097d577..1c13c87 100644
--- a/src/test/scripts/functions/compress/cost/lossy/SystemDS-config-compress-lossy-cost.xml
+++ b/src/test/scripts/functions/compress/cost/SystemDS-config-compress-cost-lossy.xml
@@ -21,5 +21,5 @@
<sysds.compressed.linalg>cost</sysds.compressed.linalg>
<sysds.compressed.lossy>true</sysds.compressed.lossy>
<sysds.cp.parallel.ops>true</sysds.cp.parallel.ops>
- <sysds.scratch>cost_lossy_scratch_space</sysds.scratch>
+ <sysds.scratch>target/cost_lossy_scratch_space</sysds.scratch>
</root>
diff --git a/src/test/scripts/functions/compress/cost/SystemDS-config-compress-cost.xml b/src/test/scripts/functions/compress/cost/SystemDS-config-compress-cost.xml
index 988774b..14ff5b8 100644
--- a/src/test/scripts/functions/compress/cost/SystemDS-config-compress-cost.xml
+++ b/src/test/scripts/functions/compress/cost/SystemDS-config-compress-cost.xml
@@ -20,5 +20,5 @@
<root>
<sysds.compressed.linalg>cost</sysds.compressed.linalg>
<sysds.cp.parallel.ops>true</sysds.cp.parallel.ops>
- <sysds.scratch>cost_scratch_space</sysds.scratch>
+ <sysds.scratch>target/cost_scratch_space</sysds.scratch>
</root>
diff --git a/src/test/scripts/functions/compress/force/lossy/SystemDS-config-compress-lossy.xml b/src/test/scripts/functions/compress/force/SystemDS-config-compress-lossy.xml
similarity index 93%
rename from src/test/scripts/functions/compress/force/lossy/SystemDS-config-compress-lossy.xml
rename to src/test/scripts/functions/compress/force/SystemDS-config-compress-lossy.xml
index 452c561..d517dc5 100644
--- a/src/test/scripts/functions/compress/force/lossy/SystemDS-config-compress-lossy.xml
+++ b/src/test/scripts/functions/compress/force/SystemDS-config-compress-lossy.xml
@@ -21,5 +21,5 @@
<sysds.compressed.linalg>true</sysds.compressed.linalg>
<sysds.compressed.lossy>true</sysds.compressed.lossy>
<sysds.cp.parallel.ops>true</sysds.cp.parallel.ops>
- <sysds.scratch>lossy_scratch_space</sysds.scratch>
+ <sysds.scratch>target/force_lossy_scratch_space</sysds.scratch>
</root>
diff --git a/src/test/scripts/functions/compress/force/SystemDS-config-compress.xml b/src/test/scripts/functions/compress/force/SystemDS-config-compress.xml
index b1b6807..34f0351 100644
--- a/src/test/scripts/functions/compress/force/SystemDS-config-compress.xml
+++ b/src/test/scripts/functions/compress/force/SystemDS-config-compress.xml
@@ -20,5 +20,5 @@
<root>
<sysds.compressed.linalg>true</sysds.compressed.linalg>
<sysds.cp.parallel.ops>true</sysds.cp.parallel.ops>
- <sysds.scratch>comp_scratch_space</sysds.scratch>
+ <sysds.scratch>target/force_comp_scratch_space</sysds.scratch>
</root>