You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2021/07/01 13:11:07 UTC
[systemds] branch master updated (bfbc8fd -> 57a45f8)
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git.
from bfbc8fd [MINOR] maven-gpg-plugin update to version 3.0.1 (#1285)
add 33fe1a0 [SYSTEMDS-2994,SYSTEMDS-2991] CLA Workload Analyzer and Workload Representation
add 4626ace [SYSTEMDS-2992,SYSTEMDS-2993] Workload costing and Cost-based Compression Planning
add 80264c1 [SYSTEMDS-2999] Use standard decompress in Binary MM ops
add 252fe4e [SYSTEMDS-3037] CLA Spark Support
add 197bd5a [SYSTEMDS-3015] CLA Workload Aware Instruction Types
add 775d6f6 [SYSTEMDS-3041] CLA Workload Avoid Obviously Bad Compressions
add a403847 [SYSTEMDS-2695,SYSTEMDS-3001] CLA Left Mult Optimization
new 66fcd48 [SYSTEMDS-3048] CLA SoftReference decompreess
new 57a45f8 [SYSTEMDS-3049] CLA SoftReference ColGroup Counts
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/main/java/org/apache/sysds/api/DMLScript.java | 2 +-
src/main/java/org/apache/sysds/conf/DMLConfig.java | 14 +-
src/main/java/org/apache/sysds/hops/Hop.java | 49 ++-
.../java/org/apache/sysds/hops/OptimizerUtils.java | 6 +
.../ipa/IPAPassCompressionWorkloadAnalysis.java | 46 ++-
.../sysds/hops/ipa/InterProceduralAnalysis.java | 1 -
.../apache/sysds/hops/rewrite/ProgramRewriter.java | 3 +-
.../hops/rewrite/RewriteCompressedReblock.java | 8 +-
.../java/org/apache/sysds/lops/Compression.java | 14 +-
.../org/apache/sysds/parser/ForStatementBlock.java | 19 +
.../runtime/compress/CompressedMatrixBlock.java | 96 +++--
.../compress/CompressedMatrixBlockFactory.java | 105 +++--
.../runtime/compress/CompressionSettings.java | 24 +-
.../compress/CompressionSettingsBuilder.java | 37 +-
.../{ => compress}/DMLCompressionException.java | 4 +-
.../runtime/compress/cocode/AColumnCoCoder.java | 18 +-
.../runtime/compress/cocode/CoCodeBinPacking.java | 10 +-
.../compress/cocode/CoCodeCostMatrixMult.java | 138 -------
.../runtime/compress/cocode/CoCodeCostTSMM.java | 176 --------
.../runtime/compress/cocode/CoCodeGreedy.java | 210 ++++++++++
.../{CoCodeCost.java => CoCodePriorityQue.java} | 84 ++--
.../runtime/compress/cocode/CoCodeStatic.java | 6 +-
.../{PlanningCoCoder.java => CoCoderFactory.java} | 54 +--
.../sysds/runtime/compress/colgroup/AColGroup.java | 23 +-
.../compress/colgroup/ColGroupCompressed.java | 1 -
.../runtime/compress/colgroup/ColGroupConst.java | 26 +-
.../runtime/compress/colgroup/ColGroupDDC.java | 84 +++-
.../runtime/compress/colgroup/ColGroupEmpty.java | 9 +-
.../runtime/compress/colgroup/ColGroupFactory.java | 69 ++--
.../runtime/compress/colgroup/ColGroupOLE.java | 12 +-
.../runtime/compress/colgroup/ColGroupOffset.java | 1 -
.../runtime/compress/colgroup/ColGroupRLE.java | 6 +-
.../runtime/compress/colgroup/ColGroupSDC.java | 96 +++--
.../compress/colgroup/ColGroupSDCSingle.java | 131 +++---
.../compress/colgroup/ColGroupSDCSingleZeros.java | 56 ++-
.../compress/colgroup/ColGroupSDCZeros.java | 32 +-
.../runtime/compress/colgroup/ColGroupSizes.java | 2 +-
.../compress/colgroup/ColGroupUncompressed.java | 16 +-
.../runtime/compress/colgroup/ColGroupValue.java | 183 +++++----
.../compress/colgroup/dictionary/ADictionary.java | 8 +-
.../compress/colgroup/dictionary/Dictionary.java | 6 +-
.../colgroup/dictionary/DictionaryFactory.java | 2 +-
.../colgroup/dictionary/MatrixBlockDictionary.java | 18 +-
.../compress/colgroup/dictionary/QDictionary.java | 3 +-
.../colgroup/insertionsort/AInsertionSorter.java | 23 +-
.../insertionsort/InsertionSorterFactory.java | 40 +-
.../colgroup/insertionsort/MaterializeSort.java | 10 +-
.../compress/colgroup/insertionsort/MergeSort.java | 6 +-
.../compress/colgroup/mapping/MapToByte.java | 6 +-
.../compress/colgroup/mapping/MapToChar.java | 6 +-
.../compress/colgroup/mapping/MapToFactory.java | 10 +-
.../compress/colgroup/mapping/MapToInt.java | 4 +
.../compress/colgroup/offset/AIterator.java | 23 +-
.../runtime/compress/colgroup/offset/AOffset.java | 20 +
.../compress/colgroup/offset/OffsetByte.java | 16 +-
.../compress/colgroup/offset/OffsetChar.java | 23 +-
.../compress/colgroup/offset/OffsetFactory.java | 2 +-
.../compress/cost/ComputationCostEstimator.java | 227 +++++++++++
.../compress/cost/CostEstimatorBuilder.java | 145 +++++++
.../compress/cost/CostEstimatorFactory.java | 55 +++
.../compress/cost/DistinctCostEstimator.java | 102 +++++
.../sysds/runtime/compress/cost/ICostEstimate.java | 104 +++++
.../compress/cost/InstructionTypeCounter.java | 86 ++++
.../runtime/compress/cost/MemoryCostEstimator.java | 75 ++++
.../compress/estim/CompressedSizeEstimator.java | 2 +-
.../runtime/compress/estim/EstimationFactors.java | 2 +-
.../estim/sample/SampleEstimatorFactory.java | 2 +-
.../runtime/compress/lib/CLALibBinaryCellOp.java | 147 +------
.../sysds/runtime/compress/lib/CLALibCompAgg.java | 93 ++++-
.../runtime/compress/lib/CLALibLeftMultBy.java | 198 ++++++---
.../sysds/runtime/compress/lib/CLALibSquash.java | 2 +-
.../compress/readers/ReaderColumnSelection.java | 2 +-
.../ReaderColumnSelectionDenseSingleBlock.java | 2 +-
...rColumnSelectionDenseSingleBlockTransposed.java | 2 +-
.../sysds/runtime/compress/utils/ABitmap.java | 2 +-
.../runtime/compress/workload/AWTreeNode.java | 109 +++++
.../workload/Op.java} | 35 +-
.../workload/OpDecompressing.java} | 29 +-
.../workload/OpMetadata.java} | 29 +-
.../workload/OpNormal.java} | 32 +-
.../workload/OpOverlappingDecompress.java} | 28 +-
.../sysds/runtime/compress/workload/OpSided.java | 91 +++++
.../sysds/runtime/compress/workload/WTreeNode.java | 108 +----
.../CoCodeStatic.java => workload/WTreeRoot.java} | 36 +-
.../compress/workload/WorkloadAnalyzer.java | 453 ++++++++++++++-------
.../controlprogram/SingletonLookupHashMap.java | 68 ++++
.../context/SparkExecutionContext.java | 5 +-
.../instructions/cp/CompressionCPInstruction.java | 25 +-
.../instructions/cp/VariableCPInstruction.java | 2 +-
.../spark/CompressionSPInstruction.java | 53 ++-
.../instructions/spark/MapmmSPInstruction.java | 16 +-
.../functions/MatrixMatrixBinaryOpFunction.java | 16 +-
.../sysds/runtime/matrix/data/LibMatrixReorg.java | 21 +-
.../sysds/runtime/matrix/data/MatrixBlock.java | 16 +-
.../matrix/data/OperationsOnMatrixValues.java | 10 +-
src/test/java/org/apache/sysds/test/TestUtils.java | 1 +
.../compress/AbstractCompressedUnaryTests.java | 196 ++++++---
.../component/compress/CompressedMatrixTest.java | 4 +-
.../component/compress/CompressedTestBase.java | 37 +-
.../component/compress/CompressedVectorTest.java | 1 -
.../compress/ParCompressedMatrixTest.java | 4 +-
.../compress/colgroup/SDCConstructionTest.java | 165 ++++++++
.../insertionsort/TestInsertionSorters.java | 30 +-
.../compress/offset/OffsetNegativeTests.java | 4 +-
.../component/compress/offset/OffsetTests.java | 3 +-
.../component/compress/workload/WorkloadTest.java | 224 ++++++++++
...ssInstruction.java => CompressInstruction.java} | 2 +-
...ewrite.java => CompressInstructionRewrite.java} | 4 +-
.../{compressScale.java => CompressScale.java} | 4 +-
.../compress/configuration/CompressBase.java | 47 +--
.../compress/configuration/CompressCost.java | 7 +-
.../compress/configuration/CompressForce.java | 123 +++++-
.../WorkloadAlgorithmTest.java} | 68 ++--
.../{ => workload}/WorkloadAnalysisTest.java | 67 ++-
src/test/resources/component/compress/1-1.csv | 1 +
src/test/resources/component/compress/1-1.csv.mtd | 8 +
src/test/resources/component/compress/README.md | 23 ++
src/test/resources/log4j.properties | 4 +-
.../compress/workload/TLeftMult.dml} | 7 +-
.../compress/workload/TRightMult.dml} | 7 +-
.../compress/workload/functions/l2svm.dml} | 8 +-
.../compress/workload/functions/lmCG.dml} | 8 +-
.../compress/workload/functions/lmDS.dml} | 8 +-
.../compress/workload/functions/pca.dml} | 6 +-
.../compress/workload/functions/scale.dml} | 6 +-
.../workload/functions/scale_continued.dml} | 7 +-
.../workload/functions/scale_onlySide.dml} | 7 +-
.../compress/workload/leftMult.dml} | 7 +-
.../compress/workload/loop/leftMult.dml} | 11 +-
.../compress/workload/loop/leftMultStaticLoop.dml} | 11 +-
.../compress/workload/loop/leftMultWhile.dml} | 14 +-
.../compress/workload/mean.dml} | 6 +-
.../compress/workload/mmrbem+.dml} | 10 +-
.../compress/workload/plus.dml} | 6 +-
.../compress/workload/rightMult.dml} | 7 +-
.../compress/workload/sliceCols.dml} | 6 +-
.../compress/workload/sliceIndex.dml} | 6 +-
.../compress/workload/sum.dml} | 6 +-
...compress_row_min.dml => compress_ewbm_left.dml} | 7 +-
...ompress_row_min.dml => compress_ewbm_right.dml} | 7 +-
.../{compress_row_min.dml => compress_ewbp.dml} | 7 +-
.../{compress_row_min.dml => compress_mml_sum.dml} | 9 +-
.../{compress_row_min.dml => compress_mmr_sum.dml} | 8 +-
.../{compress_row_min.dml => compress_plus.dml} | 8 +-
.../{compress_row_min.dml => compress_plus_mm.dml} | 7 +-
...s_row_min.dml => compress_plus_mm_ewbm_sum.dml} | 8 +-
.../functions/compress/compress_row_min.dml | 2 +-
.../SystemDS-config-compress-workload.xml} | 26 +-
.../WorkloadAnalysisLeftMultLoop.dml} | 10 +-
.../compress/{ => workload}/WorkloadAnalysisLm.dml | 2 +-
.../{ => workload}/WorkloadAnalysisMlogreg.dml | 2 +-
.../WorkloadAnalysisPCA.dml} | 12 +-
.../WorkloadAnalysisRightMultLoop.dml} | 10 +-
153 files changed, 4096 insertions(+), 1737 deletions(-)
copy src/main/java/org/apache/sysds/runtime/{ => compress}/DMLCompressionException.java (92%)
delete mode 100644 src/main/java/org/apache/sysds/runtime/compress/cocode/CoCodeCostMatrixMult.java
delete mode 100644 src/main/java/org/apache/sysds/runtime/compress/cocode/CoCodeCostTSMM.java
create mode 100644 src/main/java/org/apache/sysds/runtime/compress/cocode/CoCodeGreedy.java
rename src/main/java/org/apache/sysds/runtime/compress/cocode/{CoCodeCost.java => CoCodePriorityQue.java} (61%)
rename src/main/java/org/apache/sysds/runtime/compress/cocode/{PlanningCoCoder.java => CoCoderFactory.java} (59%)
create mode 100644 src/main/java/org/apache/sysds/runtime/compress/cost/ComputationCostEstimator.java
create mode 100644 src/main/java/org/apache/sysds/runtime/compress/cost/CostEstimatorBuilder.java
create mode 100644 src/main/java/org/apache/sysds/runtime/compress/cost/CostEstimatorFactory.java
create mode 100644 src/main/java/org/apache/sysds/runtime/compress/cost/DistinctCostEstimator.java
create mode 100644 src/main/java/org/apache/sysds/runtime/compress/cost/ICostEstimate.java
create mode 100644 src/main/java/org/apache/sysds/runtime/compress/cost/InstructionTypeCounter.java
create mode 100644 src/main/java/org/apache/sysds/runtime/compress/cost/MemoryCostEstimator.java
create mode 100644 src/main/java/org/apache/sysds/runtime/compress/workload/AWTreeNode.java
copy src/main/java/org/apache/sysds/runtime/{DMLCompressionException.java => compress/workload/Op.java} (68%)
copy src/main/java/org/apache/sysds/runtime/{DMLCompressionException.java => compress/workload/OpDecompressing.java} (68%)
copy src/main/java/org/apache/sysds/runtime/{DMLCompressionException.java => compress/workload/OpMetadata.java} (68%)
copy src/main/java/org/apache/sysds/runtime/{DMLCompressionException.java => compress/workload/OpNormal.java} (68%)
copy src/main/java/org/apache/sysds/runtime/{DMLCompressionException.java => compress/workload/OpOverlappingDecompress.java} (68%)
create mode 100644 src/main/java/org/apache/sysds/runtime/compress/workload/OpSided.java
copy src/main/java/org/apache/sysds/runtime/compress/{cocode/CoCodeStatic.java => workload/WTreeRoot.java} (52%)
create mode 100644 src/main/java/org/apache/sysds/runtime/controlprogram/SingletonLookupHashMap.java
create mode 100644 src/test/java/org/apache/sysds/test/component/compress/colgroup/SDCConstructionTest.java
create mode 100644 src/test/java/org/apache/sysds/test/component/compress/workload/WorkloadTest.java
rename src/test/java/org/apache/sysds/test/functions/compress/{compressInstruction.java => CompressInstruction.java} (98%)
rename src/test/java/org/apache/sysds/test/functions/compress/{compressInstructionRewrite.java => CompressInstructionRewrite.java} (97%)
rename src/test/java/org/apache/sysds/test/functions/compress/{compressScale.java => CompressScale.java} (97%)
copy src/test/java/org/apache/sysds/test/functions/compress/{WorkloadAnalysisTest.java => workload/WorkloadAlgorithmTest.java} (58%)
rename src/test/java/org/apache/sysds/test/functions/compress/{ => workload}/WorkloadAnalysisTest.java (56%)
create mode 100644 src/test/resources/component/compress/1-1.csv
create mode 100644 src/test/resources/component/compress/1-1.csv.mtd
create mode 100644 src/test/resources/component/compress/README.md
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/TLeftMult.dml} (90%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/TRightMult.dml} (90%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/functions/l2svm.dml} (88%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/functions/lmCG.dml} (87%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/functions/lmDS.dml} (87%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/functions/pca.dml} (93%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/functions/scale.dml} (93%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/functions/scale_continued.dml} (93%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/functions/scale_onlySide.dml} (93%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/leftMult.dml} (90%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/loop/leftMult.dml} (87%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/loop/leftMultStaticLoop.dml} (86%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/loop/leftMultWhile.dml} (84%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/mean.dml} (93%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/mmrbem+.dml} (81%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/plus.dml} (93%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/rightMult.dml} (90%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/sliceCols.dml} (93%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/sliceIndex.dml} (93%)
copy src/test/scripts/{functions/compress/compress_row_min.dml => component/compress/workload/sum.dml} (93%)
copy src/test/scripts/functions/compress/{compress_row_min.dml => compress_ewbm_left.dml} (89%)
copy src/test/scripts/functions/compress/{compress_row_min.dml => compress_ewbm_right.dml} (89%)
copy src/test/scripts/functions/compress/{compress_row_min.dml => compress_ewbp.dml} (90%)
copy src/test/scripts/functions/compress/{compress_row_min.dml => compress_mml_sum.dml} (88%)
copy src/test/scripts/functions/compress/{compress_row_min.dml => compress_mmr_sum.dml} (89%)
copy src/test/scripts/functions/compress/{compress_row_min.dml => compress_plus.dml} (85%)
copy src/test/scripts/functions/compress/{compress_row_min.dml => compress_plus_mm.dml} (89%)
copy src/test/scripts/functions/compress/{compress_row_min.dml => compress_plus_mm_ewbm_sum.dml} (83%)
rename src/{main/java/org/apache/sysds/runtime/DMLCompressionException.java => test/scripts/functions/compress/workload/SystemDS-config-compress-workload.xml} (67%)
copy src/test/scripts/functions/compress/{WorkloadAnalysisMlogreg.dml => workload/WorkloadAnalysisLeftMultLoop.dml} (89%)
rename src/test/scripts/functions/compress/{ => workload}/WorkloadAnalysisLm.dml (98%)
copy src/test/scripts/functions/compress/{ => workload}/WorkloadAnalysisMlogreg.dml (98%)
copy src/test/scripts/functions/compress/{compress_row_min.dml => workload/WorkloadAnalysisPCA.dml} (92%)
rename src/test/scripts/functions/compress/{WorkloadAnalysisMlogreg.dml => workload/WorkloadAnalysisRightMultLoop.dml} (89%)
[systemds] 02/02: [SYSTEMDS-3049] CLA SoftReference ColGroup Counts
Posted by ba...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 57a45f84e6540a3dfcab8a93f14106fae1b9fae8
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Thu Jul 1 15:06:59 2021 +0200
[SYSTEMDS-3049] CLA SoftReference ColGroup Counts
---
.../runtime/compress/colgroup/ColGroupValue.java | 36 ++++++++++++++--------
1 file changed, 23 insertions(+), 13 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupValue.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupValue.java
index 2667527..024e8a9 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupValue.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupValue.java
@@ -22,6 +22,7 @@ package org.apache.sysds.runtime.compress.colgroup;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.lang.ref.SoftReference;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
@@ -78,7 +79,7 @@ public abstract class ColGroupValue extends ColGroupCompressed implements Clonea
protected ADictionary _dict;
/** The count of each distinct value contained in the dictionary */
- private int[] counts;
+ private SoftReference<int[]> counts;
protected ColGroupValue(int numRows) {
super(numRows);
@@ -92,7 +93,7 @@ public abstract class ColGroupValue extends ColGroupCompressed implements Clonea
protected ColGroupValue(int[] colIndices, int numRows, ADictionary dict, int[] cachedCounts) {
super(colIndices, numRows);
_dict = dict;
- counts = cachedCounts;
+ counts = new SoftReference<>(cachedCounts);
}
@Override
@@ -183,18 +184,29 @@ public abstract class ColGroupValue extends ColGroupCompressed implements Clonea
* @return the count of each value in the MatrixBlock.
*/
public final int[] getCounts() {
+ int[] countsActual = null;
+ if(_dict != null) {
+ if(counts == null || counts.get() == null) {
+ countsActual = getCounts(new int[getNumValues() + (_zeros ? 1 : 0)]);
+ counts = new SoftReference<>(countsActual);
+ }
+ else
+ countsActual = counts.get();
- if(counts == null && _dict != null) {
- counts = getCounts(new int[getNumValues() + (_zeros ? 1 : 0)]);
- return counts;
}
- else
- return counts;
+
+ return countsActual;
}
+ /**
+ * Get the cached counts. If they are not materialized or the garbage collector have removed them, then null is
+ * returned
+ *
+ * @return the counts or null.
+ */
public final int[] getCachedCounts() {
- return counts;
+ return counts != null ? counts.get() : null;
}
/**
@@ -1046,7 +1058,7 @@ public abstract class ColGroupValue extends ColGroupCompressed implements Clonea
try {
final int numVals = getNumValues();
// Pre aggregate the matrix into same size as dictionary
- MatrixBlock preAgg = allocatePreAggregate(matrix,numVals, rl, ru);
+ MatrixBlock preAgg = allocatePreAggregate(matrix, numVals, rl, ru);
preAggregate(matrix, preAgg, rl, ru);
preAgg.recomputeNonZeros();
MatrixBlock tmpRes = leftMultByPreAggregateMatrix(preAgg);
@@ -1059,7 +1071,6 @@ public abstract class ColGroupValue extends ColGroupCompressed implements Clonea
public final MatrixBlock leftMultByPreAggregateMatrix(MatrixBlock preAgg) {
-
// Allocate temporary matrix to multiply into.
final int tmpCol = _colIndexes.length;
final int tmpRow = preAgg.getNumRows();
@@ -1077,8 +1088,7 @@ public abstract class ColGroupValue extends ColGroupCompressed implements Clonea
return leftMultByPreAggregateMatrix(preAgg, tmpRes);
}
-
- public final MatrixBlock leftMultByPreAggregateMatrix(MatrixBlock preAgg, MatrixBlock tmpRes){
+ public final MatrixBlock leftMultByPreAggregateMatrix(MatrixBlock preAgg, MatrixBlock tmpRes) {
// Get dictionary.
MatrixBlock dictM = forceMatrixBlockDictionary().getMatrixBlock();
LibMatrixMult.matrixMult(preAgg, dictM, tmpRes);
@@ -1088,7 +1098,7 @@ public abstract class ColGroupValue extends ColGroupCompressed implements Clonea
private void leftMultByMatrix(MatrixBlock matrix, MatrixBlock result, int[] outputRows) {
try {
final int numVals = getNumValues();
- MatrixBlock preAgg = allocatePreAggregate(matrix, numVals, 0, matrix.getNumRows());
+ MatrixBlock preAgg = allocatePreAggregate(matrix, numVals, 0, matrix.getNumRows());
preAggregate(matrix, preAgg, 0, matrix.getNumRows());
preAgg.recomputeNonZeros();
MatrixBlock tmpRes = leftMultByPreAggregateMatrix(preAgg);
[systemds] 01/02: [SYSTEMDS-3048] CLA SoftReference decompreess
Posted by ba...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 66fcd4800576858d4cff98c37ae4d754d0885d68
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Thu Jul 1 14:57:04 2021 +0200
[SYSTEMDS-3048] CLA SoftReference decompreess
---
.../runtime/compress/CompressedMatrixBlock.java | 29 ++++++++++++++++++++++
.../runtime/compress/lib/CLALibBinaryCellOp.java | 10 --------
2 files changed, 29 insertions(+), 10 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
index ff61f9c..3ddc302 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java
@@ -24,6 +24,7 @@ import java.io.DataOutput;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
+import java.lang.ref.SoftReference;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -120,6 +121,11 @@ public class CompressedMatrixBlock extends MatrixBlock {
protected boolean overlappingColGroups = false;
/**
+ * Soft reference to a decompressed version of this matrix block.
+ */
+ protected SoftReference<MatrixBlock> decompressedVersion;
+
+ /**
* Constructor for building an empty Compressed Matrix block object.
*
* OBS! Only to be used for serialization.
@@ -195,6 +201,16 @@ public class CompressedMatrixBlock extends MatrixBlock {
Timing time = new Timing(true);
+ if(decompressedVersion != null && decompressedVersion.get() != null){
+ if(DMLScript.STATISTICS || LOG.isDebugEnabled()) {
+ double t = time.stop();
+ LOG.debug("decompressed block was in soft reference.");
+ DMLCompressionStatistics.addDecompressTime(t, 1);
+ }
+ return decompressedVersion.get();
+ }
+
+
long nnz = getNonZeros() == -1 ? recomputeNonZeros() : nonZeros;
if(isEmpty())
return new MatrixBlock(rlen, clen, true, 0);
@@ -215,6 +231,8 @@ public class CompressedMatrixBlock extends MatrixBlock {
LOG.debug("decompressed block w/ k=" + 1 + " in " + t + "ms.");
DMLCompressionStatistics.addDecompressTime(t, 1);
}
+
+ decompressedVersion = new SoftReference<>(ret);
return ret;
}
@@ -245,6 +263,16 @@ public class CompressedMatrixBlock extends MatrixBlock {
return decompress();
Timing time = new Timing(true);
+
+ if(decompressedVersion != null && decompressedVersion.get() != null){
+ if(DMLScript.STATISTICS || LOG.isDebugEnabled()) {
+ double t = time.stop();
+ LOG.debug("decompressed block was in soft reference.");
+ DMLCompressionStatistics.addDecompressTime(t, k);
+ }
+ return decompressedVersion.get();
+ }
+
MatrixBlock ret = getUncompressedColGroupAndRemoveFromListOfColGroups();
if(ret != null && getColGroups().size() == 0)
return ret;
@@ -260,6 +288,7 @@ public class CompressedMatrixBlock extends MatrixBlock {
DMLCompressionStatistics.addDecompressTime(t, k);
}
+ decompressedVersion = new SoftReference<>(ret);
return ret;
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java
index 2777e72..e39932c 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibBinaryCellOp.java
@@ -92,17 +92,7 @@ public class CLALibBinaryCellOp {
MatrixBlock d_compressed = m1.decompress(op.getNumThreads());
LibMatrixBincell.bincellOpInPlace(d_compressed, that, op);
return d_compressed;
- // if(left) {
- // return that.binaryOperations(op, d_compressed, result);
- // }
- // else {
- // return d_compressed.binaryOperations(op, that, result);
- // }
}
- // else if(that.isInSparseFormat())
- // return binaryMMSparse(m1, that, op, left);
- // else
- // return binaryMMDense(m1, that, op, left);
}
else if(isSupportedBinaryCellOp(op.fn))
return bincellOp(m1, that, setupCompressedReturnMatrixBlock(m1, result), op, left);