You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2022/10/21 07:45:07 UTC
[systemds] branch main updated (1e3dff7600 -> 451e84c538)
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
from 1e3dff7600 [SYSTEMDS-3452] Fix warnings scalar reads, cleanup csv default handling
new 4cebe8ed34 [SYSTEMDS-3444] Spark Write compressed format
new 07dd88aba3 [SYSTEMDS-3446] DDC Append
new 02a6a6c798 [SYSTEMDS-3450] DDCFOR Append
new 8e4f06a89a [SYSTEMDS-3448] Uncompressed Append
new 678b28a221 [MINOR] Disable Hadoop Write compression
new 657a1bacca [SYSTEMDS-3445] N Way combining Interface
new ead3c5e9fb [SYSTEMDS-3445] N Way combining Mapping
new 451e84c538 [MINOR] Fix rand instruction to return empty matrix if sparsity == 0.0
The 8 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/sysds/parser/DMLTranslator.java | 6 +-
.../org/apache/sysds/parser/DataExpression.java | 2 +-
.../sysds/runtime/compress/colgroup/AColGroup.java | 34 ++-
.../compress/colgroup/AColGroupCompressed.java | 7 +
.../runtime/compress/colgroup/AMapToDataGroup.java | 13 +-
.../runtime/compress/colgroup/ColGroupConst.java | 9 +
.../runtime/compress/colgroup/ColGroupDDC.java | 51 ++++-
.../runtime/compress/colgroup/ColGroupDDCFOR.java | 14 +-
.../runtime/compress/colgroup/ColGroupEmpty.java | 8 +
.../colgroup/ColGroupLinearFunctional.java | 5 +
.../runtime/compress/colgroup/ColGroupOLE.java | 7 +-
.../runtime/compress/colgroup/ColGroupRLE.java | 7 +-
.../runtime/compress/colgroup/ColGroupSDC.java | 15 +-
.../runtime/compress/colgroup/ColGroupSDCFOR.java | 7 +-
.../compress/colgroup/ColGroupSDCSingle.java | 7 +-
.../compress/colgroup/ColGroupSDCSingleZeros.java | 7 +-
.../compress/colgroup/ColGroupSDCZeros.java | 7 +-
.../compress/colgroup/ColGroupUncompressed.java | 15 ++
.../compress/colgroup/dictionary/Dictionary.java | 1 -
.../compress/colgroup/mapping/AMapToData.java | 6 +
.../compress/colgroup/mapping/MapToBit.java | 63 +++++-
.../compress/colgroup/mapping/MapToByte.java | 48 ++++-
.../compress/colgroup/mapping/MapToChar.java | 54 ++++-
.../compress/colgroup/mapping/MapToCharPByte.java | 30 +++
.../compress/colgroup/mapping/MapToInt.java | 39 ++++
.../compress/colgroup/mapping/MapToZero.java | 18 ++
.../runtime/compress/colgroup/offset/AOffset.java | 32 +--
.../io/CompressWrap.java} | 18 +-
.../runtime/compress/io/CompressedWriteBlock.java | 26 ++-
.../runtime/compress/io/ReaderCompressed.java | 20 +-
.../runtime/compress/io/WriterCompressed.java | 75 +++++--
.../sysds/runtime/compress/lib/CLALibCombine.java | 85 ++++----
.../runtime/compress/lib/CLALibDecompress.java | 20 +-
.../sysds/runtime/compress/lib/CLALibSlice.java | 19 +-
.../runtime/compress/utils/DoubleCountHashMap.java | 32 +--
.../instructions/cp/DataGenCPInstruction.java | 7 +-
.../instructions/spark/WriteSPInstruction.java | 24 ++-
.../functions/ExtractBlockForBinaryReblock.java | 19 +-
.../spark/utils/RDDAggregateUtils.java | 23 +-
.../sysds/runtime/matrix/data/MatrixBlock.java | 46 ++++
.../compress/colgroup/ColGroupNegativeTests.java | 12 ++
.../component/compress/combine/CombineTest.java | 30 ++-
...OTestUtils.java => IOCompressionTestUtils.java} | 6 +-
.../sysds/test/component/compress/io/IOEmpty.java | 14 +-
.../sysds/test/component/compress/io/IOSpark.java | 240 ++++++++++++++++++---
.../sysds/test/component/compress/io/IOTest.java | 33 +--
.../component/compress/mapping/MappingTests.java | 96 ++++++++-
.../io/compressed/WriteCompressedTest.java | 95 ++++++++
.../compressed/WriteCompressedTest.dml} | 13 +-
49 files changed, 1205 insertions(+), 260 deletions(-)
copy scripts/monitoring/src/app/modules/dashboard/dialog-dashboard/dialog-dashboard.component.scss => src/main/java/org/apache/sysds/runtime/compress/colgroup/AMapToDataGroup.java (80%)
copy src/main/java/org/apache/sysds/runtime/{instructions/spark/functions/FilterNonEmptyBlocksFunction2.java => compress/io/CompressWrap.java} (73%)
rename src/test/java/org/apache/sysds/test/component/compress/io/{IOTestUtils.java => IOCompressionTestUtils.java} (94%)
create mode 100644 src/test/java/org/apache/sysds/test/functions/io/compressed/WriteCompressedTest.java
copy src/test/scripts/functions/{builtin/lmDS.dml => io/compressed/WriteCompressedTest.dml} (90%)
[systemds] 08/08: [MINOR] Fix rand instruction to return empty matrix if sparsity == 0.0
Posted by ba...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 451e84c538abf1f4bc274cdb2ee3388ea49915bf
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Thu Oct 20 17:54:02 2022 +0200
[MINOR] Fix rand instruction to return empty matrix if sparsity == 0.0
Closes #1706
---
.../compress/colgroup/mapping/MapToBit.java | 12 --------
.../runtime/compress/utils/DoubleCountHashMap.java | 32 +---------------------
.../instructions/cp/DataGenCPInstruction.java | 7 +++--
3 files changed, 6 insertions(+), 45 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToBit.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToBit.java
index 4d06e1efd6..061b69f41e 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToBit.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToBit.java
@@ -385,16 +385,4 @@ public class MapToBit extends AMapToData {
return new MapToBit(getUnique(), retBS, p);
}
-
- private static String bl(long l) {
- int lead = Long.numberOfLeadingZeros(l);
- if(lead == 64)
- return "0000000000000000000000000000000000000000000000000000000000000000";
- StringBuilder sb = new StringBuilder(64);
- for(int i = 0; i < lead; i++) {
- sb.append('0');
- }
- sb.append(Long.toBinaryString(l));
- return sb.toString();
- }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/utils/DoubleCountHashMap.java b/src/main/java/org/apache/sysds/runtime/compress/utils/DoubleCountHashMap.java
index 4785c65cbf..ddf344c71d 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/utils/DoubleCountHashMap.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/utils/DoubleCountHashMap.java
@@ -231,39 +231,9 @@ public class DoubleCountHashMap {
}
private final int hashIndex(final double key) {
-
- // previous require pow2 size.:
- // long bits = Double.doubleToRawLongBits(key);
- // int h =(int)( bits ^ (bits >>> 32));
- // h = h ^ (h >>> 20) ^ (h >>> 12);
- // h = h ^ (h >>> 7) ^ (h >>> 4);
- // return h & (_data.length - 1);
- // 100.809.414.955 instructions
-
// Option 1 ... conflict on 1 vs -1
- long bits = Double.doubleToLongBits(key);
+ final long bits = Double.doubleToLongBits(key);
return Math.abs((int)(bits ^ (bits >>> 32)) % _data.length);
- // 102.356.926.448 instructions
-
- // Option 2
- // long bits = Double.doubleToRawLongBits(key);
- // return (int) ((bits ^ (bits >> 32) % _data.length));
-
-
- // basic double hash code (w/o object creation)
- // return Double.hashCode(key) % _data.length;
- // return (int) ((bits ^ (bits >>> 32)) % _data.length);
- // long bits = Double.doubleToLongBits(key);
- // return (int) Long.remainderUnsigned(bits, (long) _data.length);
- // long bits = Double.doubleToLongBits(key);
- // long bits = Double.doubleToRawLongBits(key);
- // return (int) (bits % (long) _data.length);
-
- // return h;
-
- // This function ensures that hashCodes that differ only by
- // constant multiples at each bit position have a bounded
- // number of collisions (approximately 8 at default load factor).
}
// private static int indexFor(int h, int length) {
diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/DataGenCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/DataGenCPInstruction.java
index 8d2e83ff19..3d7867b218 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/cp/DataGenCPInstruction.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/DataGenCPInstruction.java
@@ -411,13 +411,16 @@ public class DataGenCPInstruction extends UnaryCPInstruction {
long lcols = ec.getScalarInput(cols).getLongValue();
checkValidDimensions(lrows, lcols);
+ if(sparsity == 0.0 && lrows < Integer.MAX_VALUE && lcols < Integer.MAX_VALUE)
+ return new MatrixBlock((int)lrows,(int)lcols, 0.0);
+
if(ConfigurationManager.isCompressionEnabled() && minValue == maxValue && sparsity == 1.0) {
// contains constant
if(lrows > 1000 && lcols > 0 && lrows / lcols > 1)
return CompressedMatrixBlockFactory.createConstant((int)lrows, (int)lcols, minValue);
- else
- return MatrixBlock.randOperations(getGenerator(lrows, lcols), lSeed, numThreads);
+
+ return MatrixBlock.randOperations(getGenerator(lrows, lcols), lSeed, numThreads);
}
else
return MatrixBlock.randOperations(getGenerator(lrows, lcols), lSeed, numThreads);
[systemds] 04/08: [SYSTEMDS-3448] Uncompressed Append
Posted by ba...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 8e4f06a89a2c82fc2b32bb11106849fe2cd83522
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Sat Oct 15 16:26:08 2022 +0200
[SYSTEMDS-3448] Uncompressed Append
This commit adds support for uncompressed matrices to append to themselves.
---
.../runtime/compress/colgroup/ColGroupDDC.java | 20 ++++++--
.../runtime/compress/colgroup/ColGroupSDC.java | 8 ++++
.../compress/colgroup/ColGroupUncompressed.java | 5 ++
.../runtime/compress/colgroup/offset/AOffset.java | 32 ++++++++-----
.../runtime/compress/io/CompressedWriteBlock.java | 26 ++++++++---
.../runtime/compress/io/WriterCompressed.java | 54 +++++++++++++++-------
.../sysds/runtime/compress/lib/CLALibCombine.java | 17 ++++---
.../sysds/runtime/compress/lib/CLALibSlice.java | 19 +++-----
.../sysds/runtime/matrix/data/MatrixBlock.java | 44 ++++++++++++++++++
.../sysds/test/component/compress/io/IOTest.java | 23 +++++----
10 files changed, 181 insertions(+), 67 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
index 008d66ce1b..ce2c691cca 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
@@ -486,13 +486,23 @@ public class ColGroupDDC extends APreAgg {
@Override
public AColGroup append(AColGroup g) {
- if(g instanceof ColGroupDDC && Arrays.equals(g.getColIndices(), _colIndexes)) {
- ColGroupDDC gDDC = (ColGroupDDC) g;
- if(gDDC._dict.eq(_dict)){
- AMapToData nd = _data.append(gDDC._data);
- return create(_colIndexes, _dict, nd, null);
+ if(g instanceof ColGroupDDC) {
+ if(Arrays.equals(g.getColIndices(), _colIndexes)) {
+
+ ColGroupDDC gDDC = (ColGroupDDC) g;
+ if(gDDC._dict.eq(_dict)) {
+ AMapToData nd = _data.append(gDDC._data);
+ return create(_colIndexes, _dict, nd, null);
+ }
+ else
+ LOG.warn("Not same Dictionaries therefore not appending DDC\n" + _dict + "\n\n" + gDDC._dict);
}
+ else
+ LOG.warn("Not same columns therefore not appending DDC\n" + Arrays.toString(_colIndexes) + "\n\n"
+ + Arrays.toString(g.getColIndices()));
}
+ else
+ LOG.warn("Not DDC but " + g.getClass().getSimpleName() + ", therefore not appending DDC");
return null;
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
index a40ad71644..f266f9a8fb 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
@@ -571,6 +571,14 @@ public class ColGroupSDC extends ASDC {
@Override
public AColGroup append(AColGroup g) {
+ if(g instanceof ColGroupSDC && Arrays.equals(g.getColIndices(), _colIndexes)) {
+ final ColGroupSDC gSDC = (ColGroupSDC) g;
+ if(Arrays.equals(_defaultTuple, gSDC._defaultTuple) && gSDC._dict.eq(_dict)) {
+ final AMapToData nd = _data.append(gSDC._data);
+ final AOffset ofd = _indexes.append(gSDC._indexes);
+ return create(_colIndexes, _numRows + gSDC._numRows, _dict, _defaultTuple, ofd, nd, null);
+ }
+ }
return null;
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
index fa888261f1..7c173cb6d1 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
@@ -767,6 +767,11 @@ public class ColGroupUncompressed extends AColGroup {
@Override
public AColGroup append(AColGroup g) {
+ if(g instanceof ColGroupUncompressed && Arrays.equals(g.getColIndices(), _colIndexes)) {
+ final ColGroupUncompressed gDDC = (ColGroupUncompressed) g;
+ final MatrixBlock nd = _data.append(gDDC._data, false);
+ return create(nd, _colIndexes);
+ }
return null;
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java
index 43657c15cd..2298e0ec80 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/offset/AOffset.java
@@ -439,18 +439,14 @@ public abstract class AOffset implements Serializable {
return ((OffsetChar) this).slice(lowOff, highOff, lowValue, highValue, low, high);
}
- // protected abstract OffsetSliceInfo slice(int lowOff, int highOff, int lowValue, int highValue, int low, int high);
-
- public static final class OffsetSliceInfo {
- public final int lIndex;
- public final int uIndex;
- public final AOffset offsetSlice;
-
- protected OffsetSliceInfo(int l, int u, AOffset off) {
- this.lIndex = l;
- this.uIndex = u;
- this.offsetSlice = off;
- }
+ /**
+ * Append the offsets from that other offset to the offsets in this.
+ *
+ * @param t that offsets
+ * @return this offsets followed by thats offsets.
+ */
+ public AOffset append(AOffset t){
+ throw new NotImplementedException();
}
@Override
@@ -477,6 +473,18 @@ public abstract class AOffset implements Serializable {
return sb.toString();
}
+ public static final class OffsetSliceInfo {
+ public final int lIndex;
+ public final int uIndex;
+ public final AOffset offsetSlice;
+
+ protected OffsetSliceInfo(int l, int u, AOffset off) {
+ this.lIndex = l;
+ this.uIndex = u;
+ this.offsetSlice = off;
+ }
+ }
+
protected static class OffsetCache {
protected final AIterator it;
protected final int row;
diff --git a/src/main/java/org/apache/sysds/runtime/compress/io/CompressedWriteBlock.java b/src/main/java/org/apache/sysds/runtime/compress/io/CompressedWriteBlock.java
index 8c53b3ed59..1ccbec7dcd 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/io/CompressedWriteBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/io/CompressedWriteBlock.java
@@ -31,10 +31,14 @@ import org.apache.sysds.runtime.matrix.data.MatrixBlock;
* Write block for serializing either a instance of MatrixBlock or CompressedMatrixBlock, To allow spark to read in
* either or.
*/
-public class CompressedWriteBlock implements WritableComparable<CompressedWriteBlock> {
+public class CompressedWriteBlock implements WritableComparable<CompressedWriteBlock> {
public MatrixBlock mb;
+ private enum CONTENT {
+ Comp, MB;
+ }
+
/**
* Write block used to point to a underlying instance of CompressedMatrixBlock or MatrixBlock, Unfortunately spark
* require a specific object type to serialize therefore we use this class.
@@ -49,17 +53,25 @@ public class CompressedWriteBlock implements WritableComparable<CompressedWrite
@Override
public void write(DataOutput out) throws IOException {
- out.writeBoolean(mb instanceof CompressedMatrixBlock);
+
+ if(mb instanceof CompressedMatrixBlock)
+ out.writeByte(CONTENT.Comp.ordinal());
+ else
+ out.writeByte(CONTENT.MB.ordinal());
mb.write(out);
+
}
@Override
public void readFields(DataInput in) throws IOException {
- if(in.readBoolean())
- mb = CompressedMatrixBlock.read(in);
- else {
- mb = new MatrixBlock();
- mb.readFields(in);
+ switch(CONTENT.values()[in.readByte()]) {
+ case Comp:
+ mb = CompressedMatrixBlock.read(in);
+ break;
+ case MB:
+ mb = new MatrixBlock();
+ mb.readFields(in);
+ break;
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/io/WriterCompressed.java b/src/main/java/org/apache/sysds/runtime/compress/io/WriterCompressed.java
index 207c6961fa..d2194b98f2 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/io/WriterCompressed.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/io/WriterCompressed.java
@@ -20,6 +20,7 @@
package org.apache.sysds.runtime.compress.io;
import java.io.IOException;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -35,6 +36,7 @@ import org.apache.sysds.hops.OptimizerUtils;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory;
+import org.apache.sysds.runtime.compress.lib.CLALibSlice;
import org.apache.sysds.runtime.instructions.spark.CompressionSPInstruction.CompressionFunction;
import org.apache.sysds.runtime.instructions.spark.utils.RDDConverterUtils;
import org.apache.sysds.runtime.io.FileFormatProperties;
@@ -112,7 +114,7 @@ public final class WriterCompressed extends MatrixWriter {
write(m, fname, blen);
}
- private void write(MatrixBlock src, String fname, int blen) throws IOException {
+ private void write(MatrixBlock src, final String fname, final int blen) throws IOException {
final int k = OptimizerUtils.getParallelTextWriteParallelism();
final Path path = new Path(fname);
final JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
@@ -120,13 +122,17 @@ public final class WriterCompressed extends MatrixWriter {
HDFSTool.deleteFileIfExistOnHDFS(path, job);
// Make Writer (New interface)
- Writer w = SequenceFile.createWriter(job, Writer.file(path), Writer.bufferSize(4096), Writer.blockSize(4096),
- Writer.keyClass(MatrixIndexes.class), Writer.valueClass(CompressedWriteBlock.class),
+ final Writer w = SequenceFile.createWriter(job, Writer.file(path), Writer.bufferSize(4096),
+ Writer.blockSize(4096), Writer.keyClass(MatrixIndexes.class), Writer.valueClass(CompressedWriteBlock.class),
Writer.compression(SequenceFile.CompressionType.RECORD), Writer.replication((short) 1));
final int rlen = src.getNumRows();
final int clen = src.getNumColumns();
+ // Try to compress!
+ if(!(src instanceof CompressedMatrixBlock))
+ src = CompressedMatrixBlockFactory.compress(src, k).getLeft();
+
if(rlen <= blen && clen <= blen)
writeSingleBlock(w, src, k);
else
@@ -145,20 +151,36 @@ public final class WriterCompressed extends MatrixWriter {
w.append(idx, new CompressedWriteBlock(mc));
}
- private void writeMultiBlock(Writer w, MatrixBlock b, int rlen, int clen, int blen, int k) throws IOException {
+ private void writeMultiBlock(Writer w, MatrixBlock b, final int rlen, final int clen, final int blen, int k)
+ throws IOException {
final MatrixIndexes indexes = new MatrixIndexes();
- for(int br = 0; br * blen < rlen; br++) {
- for(int bc = 0; bc * blen < clen; bc++) {
- // Max Row and col in block
- int sR = br * blen;
- int sC = bc * blen;
- int mR = Math.min(br * blen + blen, rlen) - 1;
- int mC = Math.min(bc * blen + blen, clen) - 1;
-
- MatrixBlock mb = b.slice(sR, mR, sC, mC);
- MatrixBlock mc = CompressedMatrixBlockFactory.compress(mb, k).getLeft();
- indexes.setIndexes(br + 1, bc + 1);
- w.append(indexes, new CompressedWriteBlock(mc));
+ if(!(b instanceof CompressedMatrixBlock))
+ LOG.warn("Writing compressed format with non identical compression scheme");
+
+ for(int bc = 0; bc * blen < clen; bc++) {
+ final int sC = bc * blen;
+ final int mC = Math.min(sC + blen, clen) - 1;
+ if(b instanceof CompressedMatrixBlock) {
+ final CompressedMatrixBlock mc = //mC == clen - 1 ? (CompressedMatrixBlock) b :
+ CLALibSlice
+ .sliceColumns((CompressedMatrixBlock) b, sC, mC); // slice columns!
+
+ final List<MatrixBlock> blocks = CLALibSlice.sliceBlocks(mc, blen); // Slice compressed blocks
+ for(int br = 0; br * blen < rlen; br++) {
+ indexes.setIndexes(br + 1, bc + 1);
+ w.append(indexes, new CompressedWriteBlock(blocks.get(br)));
+ }
+ }
+ else {
+ for(int br = 0; br * blen < rlen; br++) {
+ // Max Row and col in block
+ final int sR = br * blen;
+ final int mR = Math.min(sR + blen, rlen) - 1;
+ MatrixBlock mb = b.slice(sR, mR, sC, mC);
+ MatrixBlock mc = CompressedMatrixBlockFactory.compress(mb, k).getLeft();
+ indexes.setIndexes(br + 1, bc + 1);
+ w.append(indexes, new CompressedWriteBlock(mc));
+ }
}
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombine.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombine.java
index c63aaef193..7600167250 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombine.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombine.java
@@ -151,8 +151,11 @@ public class CLALibCombine {
final List<AColGroup> gs = cmb.getColGroups();
for(AColGroup g : gs) {
- final int[] cols = g.getColIndices();
- finalCols[cols[0] + bc * blen] = g; // only assign first column of each group.
+ AColGroup gc = g;
+ if(bc > 0)
+ gc = g.shiftColIndices(bc * blen);
+ final int[] cols = gc.getColIndices();
+ finalCols[cols[0]] = gc; // only assign first column of each group.
}
}
@@ -169,12 +172,14 @@ public class CLALibCombine {
if(bc > 0)
gc = g.shiftColIndices(bc * blen);
final int[] cols = gc.getColIndices();
-
- finalCols[cols[0]] = finalCols[cols[0]].append(gc);
- if(finalCols[cols[0]] == null) {
- LOG.warn("Combining of columns was non trivial, therefore falling back to decompression");
+ AColGroup prev = finalCols[cols[0]];
+ AColGroup comb = prev.append(gc);
+ if(comb == null) {
+ LOG.warn("Combining of columns from group: " + prev.getClass().getSimpleName() + " and "
+ + gc.getClass().getSimpleName() + " was non trivial, therefore falling back to decompression");
return combineViaDecompression(m, rlen, clen, blen);
}
+ finalCols[cols[0]] = comb;
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSlice.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSlice.java
index 52276962b3..32fa6c1b59 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSlice.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSlice.java
@@ -39,18 +39,12 @@ public class CLALibSlice {
*
* @param cmb The input block to slice.
* @param blen The length of the blocks.
- * @return A list containing CompressedMatrixBlocks where there is values, and null if there is no values in the sub
- * block.
+ * @return A list containing CompressedMatrixBlocks or MatrixBlocks
*/
- public static List<CompressedMatrixBlock> sliceBlocks(CompressedMatrixBlock cmb, int blen) {
- List<CompressedMatrixBlock> mbs = new ArrayList<>();
- for(int b = 0; b < cmb.getNumRows(); b += blen) {
- MatrixBlock mb = sliceRowsCompressed(cmb, b, Math.min(b + blen, cmb.getNumRows()));
- if(mb instanceof CompressedMatrixBlock)
- mbs.add((CompressedMatrixBlock) mb);
- else
- mbs.add(null);
- }
+ public static List<MatrixBlock> sliceBlocks(CompressedMatrixBlock cmb, int blen) {
+ final List<MatrixBlock> mbs = new ArrayList<>();
+ for(int b = 0; b < cmb.getNumRows(); b += blen)
+ mbs.add(sliceRowsCompressed(cmb, b, Math.min(b + blen, cmb.getNumRows()) - 1));
return mbs;
}
@@ -79,7 +73,6 @@ public class CLALibSlice {
return sliceRowsDecompress(cmb, rl, ru);
else
return sliceRowsCompressed(cmb, rl, ru);
-
}
private static boolean shouldDecompressSliceRows(CompressedMatrixBlock cmb, int rl, int ru) {
@@ -139,7 +132,7 @@ public class CLALibSlice {
return tmp;
}
- private static CompressedMatrixBlock sliceColumns(CompressedMatrixBlock cmb, int cl, int cu) {
+ public static CompressedMatrixBlock sliceColumns(CompressedMatrixBlock cmb, int cl, int cu) {
final int cue = cu + 1;
final CompressedMatrixBlock ret = new CompressedMatrixBlock(cmb.getNumRows(), cue - cl);
diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
index b69fb99873..ebd0ff88b0 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
@@ -3659,22 +3659,66 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
return result;
}
+ /**
+ * Append that matrix to this matrix, while allocating a new matrix.
+ * Default is cbind making the matrix "wider"
+ *
+ * @param that the other matrix to append
+ * @return A new MatrixBlock object with the appended result
+ */
public final MatrixBlock append(MatrixBlock that) {
return append(that, null, true); // default cbind
}
+ /**
+ * Append that matrix to this matrix, while allocating a new matrix.
+ * cbind true makes the matrix "wider" while cbind false make it "taller"
+ *
+ * @param that the other matrix to append
+ * @param cbind if binding on columns or rows
+ * @return a new MatrixBlock object with the appended result
+ */
public final MatrixBlock append(MatrixBlock that, boolean cbind) {
return append(that, null, cbind);
}
+ /**
+ * Append that matrix to this matrix.
+ *
+ * Default is cbind making the matrix "wider"
+ *
+ * @param that the other matrix to append
+ * @param ret the output matrix to modify, (is also returned)
+ * @return the ret MatrixBlock object with the appended result
+ */
public final MatrixBlock append( MatrixBlock that, MatrixBlock ret ) {
return append(that, ret, true); //default cbind
}
+ /**
+ * Append that matrix to this matrix.
+ *
+ * cbind true makes the matrix "wider" while cbind false make it "taller"
+ *
+ * @param that the other matrix to append
+ * @param ret the output matrix to modify, (is also returned)
+ * @param cbind if binding on columns or rows
+ * @return the ret MatrixBlock object with the appended result
+ */
public final MatrixBlock append( MatrixBlock that, MatrixBlock ret, boolean cbind ) {
return append(new MatrixBlock[]{that}, ret, cbind);
}
+ /**
+ * Append that list of matrixes to this matrix.
+ *
+ * cbind true makes the matrix "wider" while cbind false make it "taller"
+ *
+ * @param that a list of matrices to append in order
+ * @param result the output matrix to modify, (is also returned)
+ * @param cbind if binding on columns or rows
+ * @return the ret MatrixBlock object with the appended result
+ */
public MatrixBlock append( MatrixBlock[] that, MatrixBlock result, boolean cbind) {
checkDimensionsForAppend(that, cbind);
diff --git a/src/test/java/org/apache/sysds/test/component/compress/io/IOTest.java b/src/test/java/org/apache/sysds/test/component/compress/io/IOTest.java
index 11f28b6fd5..003ef85d08 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/io/IOTest.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/io/IOTest.java
@@ -117,12 +117,12 @@ public class IOTest {
@Test
public void testWriteAndReadSmallBlen() throws Exception {
- writeAndRead(TestUtils.ceil(TestUtils.generateTestMatrixBlock(1000, 3, 1, 3, 1.0, 2514)), 100);
+ writeAndRead(TestUtils.ceil(TestUtils.generateTestMatrixBlock(200, 3, 1, 3, 1.0, 2514)), 100);
}
@Test
public void testWriteAndReadSmallBlenBiggerClen() throws Exception {
- writeAndRead(TestUtils.ceil(TestUtils.generateTestMatrixBlock(1000, 51, 1, 3, 1.0, 2514)), 50);
+ writeAndRead(TestUtils.ceil(TestUtils.generateTestMatrixBlock(200, 51, 1, 3, 1.0, 2514)), 50);
}
@Test
@@ -141,11 +141,18 @@ public class IOTest {
}
protected static void writeAndRead(MatrixBlock mb, int blen) throws Exception {
- String filename = getName();
- WriterCompressed.writeCompressedMatrixToHDFS(mb, filename, blen);
- File f = new File(filename);
- assertTrue(f.isFile() || f.isDirectory());
- MatrixBlock mbr = IOCompressionTestUtils.read(filename);
- IOCompressionTestUtils.verifyEquivalence(mb, mbr);
+ try{
+
+ String filename = getName();
+ WriterCompressed.writeCompressedMatrixToHDFS(mb, filename, blen);
+ File f = new File(filename);
+ assertTrue(f.isFile() || f.isDirectory());
+ MatrixBlock mbr = IOCompressionTestUtils.read(filename);
+ IOCompressionTestUtils.verifyEquivalence(mb, mbr);
+ }
+ catch(Exception e){
+ e.printStackTrace();
+ throw e;
+ }
}
}
[systemds] 03/08: [SYSTEMDS-3450] DDCFOR Append
Posted by ba...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 02a6a6c798e5f0aa4228ce13f11096c930eadd10
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Sat Oct 15 16:12:16 2022 +0200
[SYSTEMDS-3450] DDCFOR Append
---
.../org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java
index 35bb7ab65a..b2f3ff1dd5 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java
@@ -435,6 +435,13 @@ public class ColGroupDDCFOR extends AMorphingMMColGroup {
@Override
public AColGroup append(AColGroup g) {
+ if(g instanceof ColGroupDDCFOR && Arrays.equals(g.getColIndices(), _colIndexes)) {
+ ColGroupDDCFOR gDDC = (ColGroupDDCFOR) g;
+ if(Arrays.equals(_reference , gDDC._reference) && gDDC._dict.eq(_dict)){
+ AMapToData nd = _data.append(gDDC._data);
+ return create(_colIndexes, _dict, nd, null, _reference);
+ }
+ }
return null;
}
[systemds] 01/08: [SYSTEMDS-3444] Spark Write compressed format
Posted by ba...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 4cebe8ed343801e565c769fc8208001251242970
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Wed Oct 12 23:51:33 2022 +0200
[SYSTEMDS-3444] Spark Write compressed format
This commit fixes/adds support for spark writing,
the previous versions did not properly support the edge cases,
while this commit fixes inconsistencies, and allow local reading
of folders containing sub blocks.
---
.../org/apache/sysds/parser/DMLTranslator.java | 6 +-
.../org/apache/sysds/parser/DataExpression.java | 2 +-
.../sysds/runtime/compress/io/CompressWrap.java | 36 ++++
.../runtime/compress/io/ReaderCompressed.java | 20 +-
.../runtime/compress/io/WriterCompressed.java | 18 ++
.../runtime/compress/lib/CLALibDecompress.java | 20 +-
.../instructions/spark/WriteSPInstruction.java | 24 ++-
.../functions/ExtractBlockForBinaryReblock.java | 19 +-
.../spark/utils/RDDAggregateUtils.java | 23 +-
.../sysds/runtime/matrix/data/MatrixBlock.java | 2 +
...OTestUtils.java => IOCompressionTestUtils.java} | 6 +-
.../sysds/test/component/compress/io/IOEmpty.java | 14 +-
.../sysds/test/component/compress/io/IOSpark.java | 240 ++++++++++++++++++---
.../sysds/test/component/compress/io/IOTest.java | 14 +-
.../io/compressed/WriteCompressedTest.java | 95 ++++++++
.../io/compressed/WriteCompressedTest.dml | 26 +++
16 files changed, 485 insertions(+), 80 deletions(-)
diff --git a/src/main/java/org/apache/sysds/parser/DMLTranslator.java b/src/main/java/org/apache/sysds/parser/DMLTranslator.java
index c196f780c3..315f54ff72 100644
--- a/src/main/java/org/apache/sysds/parser/DMLTranslator.java
+++ b/src/main/java/org/apache/sysds/parser/DMLTranslator.java
@@ -1053,20 +1053,18 @@ public class DMLTranslator
case MM:
case CSV:
case LIBSVM:
+ case HDF5:
// write output in textcell format
ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), -1);
break;
case BINARY:
+ case COMPRESSED:
// write output in binary block format
ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), ae.getBlocksize());
break;
case FEDERATED:
ae.setOutputParams(ae.getDim1(), ae.getDim2(), -1, ae.getUpdateType(), -1);
break;
- case HDF5:
- // write output in HDF5 format
- ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), -1);
- break;
default:
throw new LanguageException("Unrecognized file format: " + ae.getFileFormat());
}
diff --git a/src/main/java/org/apache/sysds/parser/DataExpression.java b/src/main/java/org/apache/sysds/parser/DataExpression.java
index c3d6edd5e3..a76aa3dd4c 100644
--- a/src/main/java/org/apache/sysds/parser/DataExpression.java
+++ b/src/main/java/org/apache/sysds/parser/DataExpression.java
@@ -1327,7 +1327,7 @@ public class DataExpression extends DataIdentifier
//validate read filename
if (getVarParam(FORMAT_TYPE) == null || FileFormat.isTextFormat(getVarParam(FORMAT_TYPE).toString()))
getOutput().setBlocksize(-1);
- else if (getVarParam(FORMAT_TYPE).toString().equalsIgnoreCase(FileFormat.BINARY.toString())) {
+ else if (getVarParam(FORMAT_TYPE).toString().equalsIgnoreCase(FileFormat.BINARY.toString()) || getVarParam(FORMAT_TYPE).toString().equalsIgnoreCase(FileFormat.COMPRESSED.toString())) {
if( getVarParam(ROWBLOCKCOUNTPARAM)!=null )
getOutput().setBlocksize(Integer.parseInt(getVarParam(ROWBLOCKCOUNTPARAM).toString()));
else
diff --git a/src/main/java/org/apache/sysds/runtime/compress/io/CompressWrap.java b/src/main/java/org/apache/sysds/runtime/compress/io/CompressWrap.java
new file mode 100644
index 0000000000..e3e370b694
--- /dev/null
+++ b/src/main/java/org/apache/sysds/runtime/compress/io/CompressWrap.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.compress.io;
+
+import org.apache.spark.api.java.function.Function;
+import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
+import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+
+public class CompressWrap implements Function<MatrixBlock, CompressedWriteBlock> {
+ private static final long serialVersionUID = 966405324406154236L;
+
+ public CompressWrap() {
+ }
+
+ @Override
+ public CompressedWriteBlock call(MatrixBlock arg0) throws Exception {
+ return new CompressedWriteBlock(arg0);
+ }
+}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/io/ReaderCompressed.java b/src/main/java/org/apache/sysds/runtime/compress/io/ReaderCompressed.java
index 0b57a8ec02..eb5fa08d60 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/io/ReaderCompressed.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/io/ReaderCompressed.java
@@ -69,11 +69,23 @@ public final class ReaderCompressed extends MatrixReader {
private static MatrixBlock readCompressedMatrix(Path path, JobConf job, FileSystem fs, int rlen, int clen, int blen)
throws IOException {
- final Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path));
+ final Map<MatrixIndexes, MatrixBlock> data = new HashMap<>();
+
+ for(Path subPath : IOUtilFunctions.getSequenceFilePaths(fs, path))
+ read(subPath, job, data);
+
+ if(data.size() == 1)
+ return data.entrySet().iterator().next().getValue();
+ else
+ return CLALibCombine.combine(data);
+ }
+
+ private static void read(Path path, JobConf job, Map<MatrixIndexes, MatrixBlock> data) throws IOException {
+
+ final Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(path));
try {
// Materialize all sub blocks.
- Map<MatrixIndexes, MatrixBlock> data = new HashMap<>();
// Use write and read interface to read and write this object.
MatrixIndexes key = new MatrixIndexes();
@@ -84,10 +96,6 @@ public final class ReaderCompressed extends MatrixReader {
key = new MatrixIndexes();
value = new CompressedWriteBlock();
}
- if(data.size() == 1)
- return data.entrySet().iterator().next().getValue();
- else
- return CLALibCombine.combine(data);
}
finally {
IOUtilFunctions.closeSilently(reader);
diff --git a/src/main/java/org/apache/sysds/runtime/compress/io/WriterCompressed.java b/src/main/java/org/apache/sysds/runtime/compress/io/WriterCompressed.java
index 7a3e07cf3c..207c6961fa 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/io/WriterCompressed.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/io/WriterCompressed.java
@@ -28,16 +28,22 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.Writer;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.SequenceFileOutputFormat;
+import org.apache.spark.api.java.JavaPairRDD;
import org.apache.sysds.conf.ConfigurationManager;
import org.apache.sysds.hops.OptimizerUtils;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory;
+import org.apache.sysds.runtime.instructions.spark.CompressionSPInstruction.CompressionFunction;
+import org.apache.sysds.runtime.instructions.spark.utils.RDDConverterUtils;
import org.apache.sysds.runtime.io.FileFormatProperties;
import org.apache.sysds.runtime.io.IOUtilFunctions;
import org.apache.sysds.runtime.io.MatrixWriter;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysds.runtime.meta.DataCharacteristics;
+import org.apache.sysds.runtime.meta.MatrixCharacteristics;
import org.apache.sysds.runtime.util.HDFSTool;
public final class WriterCompressed extends MatrixWriter {
@@ -62,6 +68,18 @@ public final class WriterCompressed extends MatrixWriter {
create(null).writeMatrixToHDFS(src, fname, rlen, clen, blen, nnz, diag);
}
+ public static void writeRDDToHDFS(JavaPairRDD<MatrixIndexes, MatrixBlock> src, String path, int blen,
+ DataCharacteristics mc) {
+ final DataCharacteristics outC = new MatrixCharacteristics(mc).setBlocksize(blen);
+ writeRDDToHDFS(RDDConverterUtils.binaryBlockToBinaryBlock(src, mc, outC), path);
+ }
+
+ public static void writeRDDToHDFS(JavaPairRDD<MatrixIndexes, MatrixBlock> src, String path) {
+ src.mapValues(new CompressionFunction()) // Try to compress each block.
+ .mapValues(new CompressWrap()) // Wrap in writable
+ .saveAsHadoopFile(path, MatrixIndexes.class, CompressedWriteBlock.class, SequenceFileOutputFormat.class);
+ }
+
@Override
public void writeMatrixToHDFS(MatrixBlock src, String fname, long rlen, long clen, int blen, long nnz, boolean diag)
throws IOException {
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibDecompress.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibDecompress.java
index 30ea5de820..b4a38b1ce7 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibDecompress.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibDecompress.java
@@ -61,6 +61,21 @@ public class CLALibDecompress {
public static void decompressTo(CompressedMatrixBlock cmb, MatrixBlock ret, int rowOffset, int colOffset, int k) {
Timing time = new Timing(true);
+ if(cmb.getNumColumns() + colOffset > ret.getNumColumns() || cmb.getNumRows() + rowOffset > ret.getNumRows()) {
+ LOG.warn(
+ "Slow slicing off excess parts for decompressTo because decompression into is implemented for fitting blocks");
+ MatrixBlock mbSliced = cmb.slice( //
+ Math.min(Math.abs(rowOffset), 0), Math.min(cmb.getNumRows(), ret.getNumRows() - rowOffset) - 1, // Rows
+ Math.min(Math.abs(colOffset), 0), Math.min(cmb.getNumColumns(), ret.getNumColumns() - colOffset) - 1); // Cols
+ if(mbSliced instanceof MatrixBlock) {
+ mbSliced.putInto(ret, rowOffset, colOffset, false);
+ return;
+ }
+
+ cmb = (CompressedMatrixBlock) mbSliced;
+ decompress(cmb, 1);
+ }
+
final boolean outSparse = ret.isInSparseFormat();
if(!cmb.isEmpty()) {
if(outSparse && cmb.isOverlapping())
@@ -78,8 +93,7 @@ public class CLALibDecompress {
LOG.trace("decompressed block w/ k=" + k + " in " + t + "ms.");
}
- if(ret.getNonZeros() <= 0)
- ret.setNonZeros(cmb.getNonZeros());
+ ret.recomputeNonZeros();
}
private static void decompressToSparseBlock(CompressedMatrixBlock cmb, MatrixBlock ret, int rowOffset,
@@ -96,6 +110,8 @@ public class CLALibDecompress {
else
for(AColGroup g : groups)
g.decompressToSparseBlock(sb, 0, nRows, rowOffset, colOffset);
+ sb.sort();
+ ret.checkSparseRows();
}
private static void decompressToDenseBlock(CompressedMatrixBlock cmb, DenseBlock ret, int rowOffset, int colOffset) {
diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/WriteSPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/WriteSPInstruction.java
index bad0d2fde5..97371a477f 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/spark/WriteSPInstruction.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/WriteSPInstruction.java
@@ -19,6 +19,10 @@
package org.apache.sysds.runtime.instructions.spark;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Random;
+
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.io.LongWritable;
@@ -31,6 +35,7 @@ import org.apache.sysds.common.Types.FileFormat;
import org.apache.sysds.common.Types.ValueType;
import org.apache.sysds.conf.ConfigurationManager;
import org.apache.sysds.runtime.DMLRuntimeException;
+import org.apache.sysds.runtime.compress.io.WriterCompressed;
import org.apache.sysds.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysds.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysds.runtime.instructions.InstructionUtils;
@@ -52,10 +57,6 @@ import org.apache.sysds.runtime.meta.DataCharacteristics;
import org.apache.sysds.runtime.meta.MatrixCharacteristics;
import org.apache.sysds.runtime.util.HDFSTool;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Random;
-
public class WriteSPInstruction extends SPInstruction implements LineageTraceable {
public CPOperand input1 = null;
private CPOperand input2 = null;
@@ -252,6 +253,21 @@ public class WriteSPInstruction extends SPInstruction implements LineageTraceabl
if( nonDefaultBlen )
mcOut = new MatrixCharacteristics(mc).setBlocksize(blen);
}
+ else if(fmt == FileFormat.COMPRESSED) {
+ // reblock output if needed
+ final int blen = Integer.parseInt(input4.getName());
+ final boolean nonDefaultBlen = ConfigurationManager.getBlocksize() != blen;
+ mc.setNonZeros(-1); // default to unknown non zeros for compressed matrix block
+
+ if(nonDefaultBlen)
+ WriterCompressed.writeRDDToHDFS(in1, fname, blen, mc);
+ else
+ WriterCompressed.writeRDDToHDFS(in1, fname);
+
+ if(nonDefaultBlen)
+ mcOut = new MatrixCharacteristics(mc).setBlocksize(blen);
+
+ }
else if(fmt == FileFormat.LIBSVM) {
if(mc.getRows() == 0 || mc.getCols() == 0) {
throw new IOException(
diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java
index f9b735e39c..4f20216760 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java
@@ -23,7 +23,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
-import org.apache.commons.lang.NotImplementedException;
import org.apache.spark.api.java.function.PairFlatMapFunction;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
@@ -92,15 +91,19 @@ public class ExtractBlockForBinaryReblock implements PairFlatMapFunction<Tuple2<
final int cixi = UtilFunctions.computeCellInBlock(rowLower, out_blen);
final int cixj = UtilFunctions.computeCellInBlock(colLower, out_blen);
- if(in instanceof CompressedMatrixBlock){
- blk.allocateSparseRowsBlock(false);
- CLALibDecompress.decompressTo((CompressedMatrixBlock) in, blk, cixi, cixj, 1);
- }
- else if( aligned ) {
- blk.appendToSparse(in, cixi, cixj);
- blk.setNonZeros(in.getNonZeros());
+ if( aligned ) {
+ if(in instanceof CompressedMatrixBlock){
+ blk.allocateSparseRowsBlock(false);
+ CLALibDecompress.decompressTo((CompressedMatrixBlock) in, blk, cixi- aixi, cixj-aixj, 1);
+ }else{
+ blk.appendToSparse(in, cixi, cixj);
+ blk.setNonZeros(in.getNonZeros());
+ }
}
else { //general case
+ if(in instanceof CompressedMatrixBlock){
+ in = CompressedMatrixBlock.getUncompressed(in);
+ }
for(int i2 = 0; i2 <= (int)(rowUpper-rowLower); i2++)
for(int j2 = 0; j2 <= (int)(colUpper-colLower); j2++)
blk.appendValue(cixi+i2, cixj+j2, in.quickGetValue(aixi+i2, aixj+j2));
diff --git a/src/main/java/org/apache/sysds/runtime/instructions/spark/utils/RDDAggregateUtils.java b/src/main/java/org/apache/sysds/runtime/instructions/spark/utils/RDDAggregateUtils.java
index 3a3e9602cb..22b313f8e6 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/spark/utils/RDDAggregateUtils.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/spark/utils/RDDAggregateUtils.java
@@ -704,32 +704,27 @@ public class RDDAggregateUtils
}
@Override
- public MatrixBlock call(MatrixBlock b1, MatrixBlock b2)
- throws Exception
- {
+ public MatrixBlock call(MatrixBlock b1, MatrixBlock b2) throws Exception {
long b1nnz = b1.getNonZeros();
long b2nnz = b2.getNonZeros();
-
+
// sanity check input dimensions
- if (b1.getNumRows() != b2.getNumRows() || b1.getNumColumns() != b2.getNumColumns()) {
- throw new DMLRuntimeException("Mismatched block sizes for: "
- + b1.getNumRows() + " " + b1.getNumColumns() + " "
- + b2.getNumRows() + " " + b2.getNumColumns());
+ if(b1.getNumRows() != b2.getNumRows() || b1.getNumColumns() != b2.getNumColumns()) {
+ throw new DMLRuntimeException("Mismatched block sizes for: " + b1.getNumRows() + " " + b1.getNumColumns()
+ + " " + b2.getNumRows() + " " + b2.getNumColumns());
}
// execute merge (never pass by reference)
MatrixBlock ret = _deep ? new MatrixBlock(b1) : b1;
ret.merge(b2, false, false, _deep);
ret.examSparsity();
-
+
// sanity check output number of non-zeros
- if (ret.getNonZeros() != b1nnz + b2nnz) {
- throw new DMLRuntimeException("Number of non-zeros does not match: "
- + ret.getNonZeros() + " != " + b1nnz + " + " + b2nnz);
+ if(ret.getNonZeros() != b1nnz + b2nnz) {
+ throw new DMLRuntimeException(
+ "Number of non-zeros does not match: " + ret.getNonZeros() + " != " + b1nnz + " + " + b2nnz);
}
-
return ret;
}
-
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
index ad0ca1c776..b69fb99873 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
@@ -1435,6 +1435,8 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab
for(int k = apos; k < apos + alen; k++)
if(avals[k] == 0)
throw new RuntimeException("Wrong sparse row: zero at " + k);
+ if(aix[apos + alen-1] > clen)
+ throw new RuntimeException("Invalid offset outside of matrix");
}
}
diff --git a/src/test/java/org/apache/sysds/test/component/compress/io/IOTestUtils.java b/src/test/java/org/apache/sysds/test/component/compress/io/IOCompressionTestUtils.java
similarity index 94%
rename from src/test/java/org/apache/sysds/test/component/compress/io/IOTestUtils.java
rename to src/test/java/org/apache/sysds/test/component/compress/io/IOCompressionTestUtils.java
index fe5605ec31..785fc4444d 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/io/IOTestUtils.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/io/IOCompressionTestUtils.java
@@ -28,14 +28,14 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.sysds.runtime.compress.io.ReaderCompressed;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
-public class IOTestUtils {
+public class IOCompressionTestUtils {
final static Object lock = new Object();
static final AtomicInteger id = new AtomicInteger(0);
protected static void deleteDirectory(File file) {
- synchronized(IOTestUtils.lock) {
+ synchronized(IOCompressionTestUtils.lock) {
for(File subfile : file.listFiles()) {
if(subfile.isDirectory())
deleteDirectory(subfile);
@@ -59,7 +59,7 @@ public class IOTestUtils {
// assertTrue("Disk size is not equivalent", a.getExactSizeOnDisk() > b.getExactSizeOnDisk());
}
- protected static MatrixBlock read(String path) {
+ public static MatrixBlock read(String path) {
try {
return ReaderCompressed.readCompressedMatrixFromHDFS(path);
}
diff --git a/src/test/java/org/apache/sysds/test/component/compress/io/IOEmpty.java b/src/test/java/org/apache/sysds/test/component/compress/io/IOEmpty.java
index 5f913c70d3..41e4f2e904 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/io/IOEmpty.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/io/IOEmpty.java
@@ -39,18 +39,18 @@ public class IOEmpty {
+ IOEmpty.class.getSimpleName() + "/";
public IOEmpty() {
- synchronized(IOTestUtils.lock) {
+ synchronized(IOCompressionTestUtils.lock) {
new File(nameBeginning).mkdirs();
}
}
@AfterClass
public static void cleanup() {
- IOTestUtils.deleteDirectory(new File(nameBeginning));
+ IOCompressionTestUtils.deleteDirectory(new File(nameBeginning));
}
public static String getName() {
- return IOTestUtils.getName(nameBeginning);
+ return IOCompressionTestUtils.getName(nameBeginning);
}
@Test
@@ -65,8 +65,8 @@ public class IOEmpty {
public void writeEmptyAndRead() {
String n = getName();
write(n, 10, 10, 1000);
- MatrixBlock mb = IOTestUtils.read(n);
- IOTestUtils.verifyEquivalence(mb, new MatrixBlock(10, 10, 0.0));
+ MatrixBlock mb = IOCompressionTestUtils.read(n);
+ IOCompressionTestUtils.verifyEquivalence(mb, new MatrixBlock(10, 10, 0.0));
}
@Test
@@ -83,8 +83,8 @@ public class IOEmpty {
write(n, 1000, 10, 100);
File f = new File(n);
assertTrue(f.isDirectory() || f.isFile());
- MatrixBlock mb = IOTestUtils.read(n);
- IOTestUtils.verifyEquivalence(mb, new MatrixBlock(1000, 10, 0.0));
+ MatrixBlock mb = IOCompressionTestUtils.read(n);
+ IOCompressionTestUtils.verifyEquivalence(mb, new MatrixBlock(1000, 10, 0.0));
}
protected static void write(String path, int nRows, int nCols, int blen) {
diff --git a/src/test/java/org/apache/sysds/test/component/compress/io/IOSpark.java b/src/test/java/org/apache/sysds/test/component/compress/io/IOSpark.java
index 61b699c254..847938bff3 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/io/IOSpark.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/io/IOSpark.java
@@ -20,6 +20,7 @@
package org.apache.sysds.test.component.compress.io;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.File;
@@ -31,14 +32,18 @@ import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.sysds.common.Types.FileFormat;
import org.apache.sysds.common.Types.ValueType;
+import org.apache.sysds.runtime.compress.io.CompressUnwrap;
import org.apache.sysds.runtime.compress.io.CompressedWriteBlock;
import org.apache.sysds.runtime.compress.io.WriterCompressed;
import org.apache.sysds.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysds.runtime.controlprogram.context.ExecutionContextFactory;
import org.apache.sysds.runtime.controlprogram.context.SparkExecutionContext;
+import org.apache.sysds.runtime.instructions.spark.utils.RDDConverterUtils;
import org.apache.sysds.runtime.io.InputOutputInfo;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysds.runtime.meta.DataCharacteristics;
+import org.apache.sysds.runtime.meta.MatrixCharacteristics;
import org.apache.sysds.test.TestUtils;
import org.junit.AfterClass;
import org.junit.Test;
@@ -54,11 +59,11 @@ public class IOSpark {
@AfterClass
public static void cleanup() {
- IOTestUtils.deleteDirectory(new File(nameBeginning));
+ IOCompressionTestUtils.deleteDirectory(new File(nameBeginning));
}
private static String getName() {
- return IOTestUtils.getName(nameBeginning);
+ return IOCompressionTestUtils.getName(nameBeginning);
}
@Test
@@ -109,38 +114,202 @@ public class IOSpark {
readWrite(mb);
}
- private void readWrite(MatrixBlock mb) {
- double sum = mb.sum();
- String n = getName();
- try {
- WriterCompressed.writeCompressedMatrixToHDFS(mb, n, 50);
- }
- catch(Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
+ @Test
+ public void writeSparkReadCPMultiColBlock() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(50, 124, 1, 3, 1.0, 2514));
+ testWriteSparkReadCP(mb, 100, 100);
+ }
+
+ @Test
+ public void writeSparkReadCPMultiRowBlock() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(1322, 33, 1, 3, 1.0, 2514));
+ testWriteSparkReadCP(mb, 100, 100);
+ }
+
+ @Test
+ public void writeSparkReadCPSingleBlock() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(50, 99, 1, 3, 1.0, 33));
+ testWriteSparkReadCP(mb, 100, 100);
+ }
+
+ @Test
+ public void writeSparkReadCPMultiBlock() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(580, 244, 1, 3, 1.0, 33));
+ testWriteSparkReadCP(mb, 100, 100);
+ }
+
+ @Test
+ public void writeSparkReadCPMultiColBlockReblockUp() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(50, 124, 1, 3, 1.0, 2514));
+ testWriteSparkReadCP(mb, 100, 150);
+ }
+
+ @Test
+ public void writeSparkReadCPMultiRowBlockReblockUp() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(1322, 33, 1, 3, 1.0, 2514));
+ testWriteSparkReadCP(mb, 100, 150);
+ }
+
+ @Test
+ public void writeSparkReadCPSingleBlockReblockUp() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(50, 99, 1, 3, 1.0, 33));
+ testWriteSparkReadCP(mb, 100, 150);
+ }
+
+ @Test
+ public void writeSparkReadCPMultiBlockReblockUp() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(580, 244, 1, 3, 1.0, 33));
+ testWriteSparkReadCP(mb, 100, 150);
+ }
+
+ @Test
+ public void writeSparkReadCPMultiColBlockReblockDown() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(50, 124, 1, 3, 1.0, 2514));
+ testWriteSparkReadCP(mb, 100, 80);
+ }
+
+ @Test
+ public void writeSparkReadCPMultiRowBlockReblockDown() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(1322, 33, 1, 3, 1.0, 2514));
+ testWriteSparkReadCP(mb, 100, 80);
+ }
+
+ @Test
+ public void writeSparkReadCPSingleBlockReblockDown() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(50, 99, 1, 3, 1.0, 33));
+ testWriteSparkReadCP(mb, 100, 80);
+ }
+
+ @Test
+ public void writeSparkReadCPMultiBlockReblockDown() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(580, 244, 1, 3, 1.0, 33));
+ testWriteSparkReadCP(mb, 100, 80);
+ }
+
+ private void testWriteSparkReadCP(MatrixBlock mb, int blen1, int blen2) throws Exception {
+
+ String f1 = getName();
+ WriterCompressed.writeCompressedMatrixToHDFS(mb, f1, blen1);
+
+ // Make sure the first file is written
+ File f = new File(f1);
+ assertTrue(f.isFile() || f.isDirectory());
+ // Read in again as RDD
+ JavaPairRDD<MatrixIndexes, MatrixBlock> m = getRDD(f1);
+ String f2 = getName(); // get new name for writing RDD.
+
+ // Write RDD to disk
+ if(blen1 != blen2) {
+ DataCharacteristics mc = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), blen1);
+ WriterCompressed.writeRDDToHDFS(m, f2, blen2, mc);
}
- verifySum(read(n), sum, 0.0001);
+ else
+ WriterCompressed.writeRDDToHDFS(m, f2);
+
+ // Read locally the spark block written.
+ MatrixBlock mbr = IOCompressionTestUtils.read(f2);
+ IOCompressionTestUtils.verifyEquivalence(mb, mbr);
}
- private void verifySum(List<Tuple2<MatrixIndexes, MatrixBlock>> c, double val, double tol) {
- double sum = 0.0;
- for(Tuple2<MatrixIndexes, MatrixBlock> b : c)
- sum += b._2().sum();
- assertEquals(val, sum, tol);
+ @Test
+ public void testReblock_up() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(50, 50, 1, 3, 1.0, 2514));
+ testReblock(mb, 25, 50);
}
- private List<Tuple2<MatrixIndexes, MatrixBlock>> read(String n) {
- JavaPairRDD<MatrixIndexes, MatrixBlock> m = getRDD(n).mapValues(x -> x.get());
- return m.collect();
+ @Test
+ public void testReblock_up_2() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(50, 50, 1, 3, 1.0, 2514));
+ testReblock(mb, 25, 55);
+ }
+
+ @Test
+ public void testReblock_up_3() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(165, 110, 1, 3, 1.0, 2514));
+ testReblock(mb, 25, 55);
+ }
+
+ @Test
+ public void testReblock_up_4() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(165, 110, 1, 3, 1.0, 2514));
+ testReblock(mb, 25, 100);
+ }
+
+ @Test
+ public void testReblock_up_5() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(230, 401, 1, 3, 1.0, 2514));
+ testReblock(mb, 25, 100);
+ }
+
+ @Test
+ public void testReblock_down() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(50, 50, 1, 3, 1.0, 2514));
+ testReblock(mb, 50, 25);
+ }
+
+ @Test
+ public void testReblock_down_2() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(50, 50, 1, 3, 1.0, 2514));
+ testReblock(mb, 55, 25);
+ }
+
+ @Test
+ public void testReblock_down_3() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(165, 110, 1, 3, 1.0, 2514));
+ testReblock(mb, 55, 25);
+ }
+
+ @Test
+ public void testReblock_down_4() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(165, 110, 1, 3, 1.0, 2514));
+ testReblock(mb, 100, 25);
+ }
+
+ @Test
+ public void testReblock_down_5() throws Exception {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(230, 401, 1, 3, 1.0, 2514));
+ testReblock(mb, 100, 25);
+ }
+
+ private void testReblock(MatrixBlock mb, int blen1, int blen2) throws Exception {
+ String f1 = getName();
+ WriterCompressed.writeCompressedMatrixToHDFS(mb, f1, blen1);
+
+ // Read in again as RDD
+ JavaPairRDD<MatrixIndexes, MatrixBlock> m = getRDD(f1); // Our starting point
+ int nBlocksExpected = (1 + (mb.getNumColumns() - 1) / blen1) * (1 + (mb.getNumRows() - 1) / blen1);
+ int nBlocksActual = m.collect().size();
+ assertEquals("Expected same number of blocks ", nBlocksExpected, nBlocksActual);
+
+ DataCharacteristics mc = new MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(), blen1);
+ JavaPairRDD<MatrixIndexes, MatrixBlock> m2 = reblock(m, mc, blen2);
+ int nBlocksExpected2 = (1 + (mb.getNumColumns() - 1) / blen2) * (1 + (mb.getNumRows() - 1) / blen2);
+ int nBlocksActual2 = m2.collect().size();
+ assertEquals("Expected same number of blocks on re-blocked", nBlocksExpected2, nBlocksActual2);
+
+ double val = mb.sum();
+ verifySum(m, val, 0.0000001);
+ verifySum(m2, val, 0.0000001);
+
+ }
+
+ private static JavaPairRDD<MatrixIndexes, MatrixBlock> reblock(JavaPairRDD<MatrixIndexes, MatrixBlock> in,
+ DataCharacteristics mc, int blen) {
+ final DataCharacteristics outC = new MatrixCharacteristics(mc).setBlocksize(blen);
+ return RDDConverterUtils.binaryBlockToBinaryBlock(in, mc, outC);
+ }
+
+ private List<Tuple2<MatrixIndexes, MatrixBlock>> read(String n) {
+ return getRDD(n).collect();
}
@SuppressWarnings({"unchecked"})
- private JavaPairRDD<MatrixIndexes, CompressedWriteBlock> getRDD(String path) {
+ private JavaPairRDD<MatrixIndexes, MatrixBlock> getRDD(String path) {
InputOutputInfo inf = InputOutputInfo.CompressedInputOutputInfo;
JavaSparkContext sc = SparkExecutionContext.getSparkContextStatic();
- return (JavaPairRDD<MatrixIndexes, CompressedWriteBlock>) sc.hadoopFile(path, inf.inputFormatClass, inf.keyClass,
- inf.valueClass);
+ return ((JavaPairRDD<MatrixIndexes, CompressedWriteBlock>) sc.hadoopFile(path, inf.inputFormatClass, inf.keyClass,
+ inf.valueClass)).mapValues(new CompressUnwrap());
}
@SuppressWarnings({"unchecked"})
@@ -158,13 +327,36 @@ public class IOSpark {
JavaPairRDD<MatrixIndexes, MatrixBlock> m = (JavaPairRDD<MatrixIndexes, MatrixBlock>) ec
.getRDDHandleForMatrixObject(obj, fmt);
+ List<Tuple2<MatrixIndexes, MatrixBlock>> c = m.collect();
+ verifySum(c, mb.sum(), 0.0001);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
- verifySum(m.collect(), mb.sum(), 0.0001);
+ private void readWrite(MatrixBlock mb) {
+ double sum = mb.sum();
+ String n = getName();
+ try {
+ WriterCompressed.writeCompressedMatrixToHDFS(mb, n, 50);
}
catch(Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
+ verifySum(read(n), sum, 0.0001);
+ }
+
+ private void verifySum(JavaPairRDD<MatrixIndexes, MatrixBlock> m, double val, double tol) {
+ verifySum(m.collect(), val, tol);
}
+ private void verifySum(List<Tuple2<MatrixIndexes, MatrixBlock>> c, double val, double tol) {
+ double sum = 0.0;
+ for(Tuple2<MatrixIndexes, MatrixBlock> b : c)
+ sum += b._2().sum();
+ assertEquals(val, sum, tol);
+ }
}
diff --git a/src/test/java/org/apache/sysds/test/component/compress/io/IOTest.java b/src/test/java/org/apache/sysds/test/component/compress/io/IOTest.java
index 90b23e49ab..11f28b6fd5 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/io/IOTest.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/io/IOTest.java
@@ -41,18 +41,18 @@ public class IOTest {
+ IOTest.class.getSimpleName() + "/";
public IOTest() {
- synchronized(IOTestUtils.lock) {
+ synchronized(IOCompressionTestUtils.lock) {
new File(nameBeginning).mkdirs();
}
}
@AfterClass
public static void cleanup() {
- IOTestUtils.deleteDirectory(new File(nameBeginning));
+ IOCompressionTestUtils.deleteDirectory(new File(nameBeginning));
}
public static String getName() {
- return IOTestUtils.getName(nameBeginning);
+ return IOCompressionTestUtils.getName(nameBeginning);
}
@Test
@@ -101,8 +101,8 @@ public class IOTest {
WriterCompressed.writeCompressedMatrixToHDFS(mb, filename);
File f = new File(filename);
assertTrue(f.isFile() || f.isDirectory());
- MatrixBlock mbr = IOTestUtils.read(filename);
- IOTestUtils.verifyEquivalence(mb, mbr);
+ MatrixBlock mbr = IOCompressionTestUtils.read(filename);
+ IOCompressionTestUtils.verifyEquivalence(mb, mbr);
}
protected static void write(MatrixBlock src, String path) {
@@ -145,7 +145,7 @@ public class IOTest {
WriterCompressed.writeCompressedMatrixToHDFS(mb, filename, blen);
File f = new File(filename);
assertTrue(f.isFile() || f.isDirectory());
- MatrixBlock mbr = IOTestUtils.read(filename);
- IOTestUtils.verifyEquivalence(mb, mbr);
+ MatrixBlock mbr = IOCompressionTestUtils.read(filename);
+ IOCompressionTestUtils.verifyEquivalence(mb, mbr);
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/io/compressed/WriteCompressedTest.java b/src/test/java/org/apache/sysds/test/functions/io/compressed/WriteCompressedTest.java
new file mode 100644
index 0000000000..ba9e6f430f
--- /dev/null
+++ b/src/test/java/org/apache/sysds/test/functions/io/compressed/WriteCompressedTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.functions.io.compressed;
+
+import java.io.IOException;
+
+import org.apache.sysds.common.Types.ExecMode;
+import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+import org.apache.sysds.test.AutomatedTestBase;
+import org.apache.sysds.test.TestConfiguration;
+import org.apache.sysds.test.TestUtils;
+import org.apache.sysds.test.component.compress.io.IOCompressionTestUtils;
+import org.junit.Test;
+
+/**
+ * JUnit Test cases to evaluate the functionality of reading CSV files.
+ *
+ * Test 1: write() w/ all properties. Test 2: read(format="csv") w/o mtd file. Test 3: read() w/ complete mtd file.
+ *
+ */
+
+public class WriteCompressedTest extends AutomatedTestBase {
+
+ private final static String TEST_NAME = "WriteCompressedTest";
+ private final static String TEST_DIR = "functions/io/compressed/";
+ private final static String TEST_CLASS_DIR = TEST_DIR + WriteCompressedTest.class.getSimpleName() + "/";
+
+ private final static double eps = 1e-9;
+
+ @Override
+ public void setUp() {
+ addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"Rout"}));
+ }
+
+ @Test
+ public void testCP() throws IOException {
+ runWriteTest(ExecMode.SINGLE_NODE);
+ }
+
+ @Test
+ public void testHP() throws IOException {
+ runWriteTest(ExecMode.HYBRID);
+ }
+
+ @Test
+ public void testSP() throws IOException {
+ runWriteTest(ExecMode.SPARK);
+ }
+
+ private void runWriteTest(ExecMode platform) throws IOException {
+ runWriteTest(platform, 100, 100, 0, 0, 0.0);
+ }
+
+ private void runWriteTest(ExecMode platform, int rows, int cols, int min, int max, double sparsity)
+ throws IOException {
+
+ ExecMode oldPlatform = rtplatform;
+ rtplatform = platform;
+
+ TestConfiguration config = getTestConfiguration(TEST_NAME);
+ loadTestConfiguration(config);
+
+ String HOME = SCRIPT_DIR + TEST_DIR;
+
+ fullDMLScriptName = HOME + TEST_NAME + ".dml";
+ programArgs = new String[] {"-explain", "-args", "" + rows, "" + cols, "" + min, "" + max, "" + sparsity,
+ output("out.cla"), output("sum.scalar")};
+
+ runTest(null);
+
+ double sumDML = TestUtils.readDMLScalar(output("sum.scalar"));
+ MatrixBlock mbr = IOCompressionTestUtils.read(output("out.cla"));
+
+ TestUtils.compareScalars(sumDML, mbr.sum(), eps);
+
+ rtplatform = oldPlatform;
+ }
+}
diff --git a/src/test/scripts/functions/io/compressed/WriteCompressedTest.dml b/src/test/scripts/functions/io/compressed/WriteCompressedTest.dml
new file mode 100644
index 0000000000..6a51041016
--- /dev/null
+++ b/src/test/scripts/functions/io/compressed/WriteCompressedTest.dml
@@ -0,0 +1,26 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+m = rand(rows= $1, cols=$2, min=$3, max=$4, sparsity=$5)
+
+s = sum(m)
+write(m, $6, format="compressed")
+write(s, $7, format="csv")
[systemds] 06/08: [SYSTEMDS-3445] N Way combining Interface
Posted by ba...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 657a1bacca9a83dba1632df156979fec7bfaee5a
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Wed Oct 19 17:02:34 2022 +0200
[SYSTEMDS-3445] N Way combining Interface
---
.../sysds/runtime/compress/colgroup/AColGroup.java | 26 +++++++++++++++++++++-
.../runtime/compress/colgroup/ColGroupConst.java | 9 ++++++++
.../runtime/compress/colgroup/ColGroupDDC.java | 6 +++++
.../runtime/compress/colgroup/ColGroupDDCFOR.java | 5 +++++
.../runtime/compress/colgroup/ColGroupEmpty.java | 8 +++++++
.../colgroup/ColGroupLinearFunctional.java | 5 +++++
.../runtime/compress/colgroup/ColGroupOLE.java | 5 +++++
.../runtime/compress/colgroup/ColGroupRLE.java | 5 +++++
.../runtime/compress/colgroup/ColGroupSDC.java | 5 +++++
.../runtime/compress/colgroup/ColGroupSDCFOR.java | 5 +++++
.../compress/colgroup/ColGroupSDCSingle.java | 5 +++++
.../compress/colgroup/ColGroupSDCSingleZeros.java | 5 +++++
.../compress/colgroup/ColGroupSDCZeros.java | 5 +++++
.../compress/colgroup/ColGroupUncompressed.java | 5 +++++
.../sysds/runtime/compress/lib/CLALibCombine.java | 5 +----
.../compress/colgroup/ColGroupNegativeTests.java | 12 ++++++++++
16 files changed, 111 insertions(+), 5 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java
index efaef22cde..10c374b29f 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java
@@ -611,10 +611,34 @@ public abstract class AColGroup implements Serializable {
* If it is not possible or very inefficient null is returned.
*
* @param g The other column group
- * @return A combined column group
+ * @return A combined column group or null
*/
public abstract AColGroup append(AColGroup g);
+ /**
+ * Append all column groups in the list provided together in one go allocating the output once.
+ *
+ * If it is not possible or very inefficient null is returned.
+ *
+ * @param groups The groups to combine.
+ * @return A combined column group or null
+ */
+ public static AColGroup appendN(AColGroup[] groups) {
+ return groups[0].appendNInternal(groups);
+ }
+
+ /**
+ * Append all column groups in the list provided together with this.
+ *
+ * A Important detail is the first entry in the group == this, and should not be appended twice.
+ *
+ * If it is not possible or very inefficient null is returned.
+ *
+ * @param groups The groups to combine.
+ * @return A combined column group or null
+ */
+ protected abstract AColGroup appendNInternal(AColGroup[] groups);
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java
index aa1b5e229e..91f4ff4003 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupConst.java
@@ -21,6 +21,7 @@ package org.apache.sysds.runtime.compress.colgroup;
import java.io.DataInput;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.sysds.runtime.compress.DMLCompressionException;
import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
@@ -527,6 +528,14 @@ public class ColGroupConst extends ADictBasedColGroup {
return null;
}
+ @Override
+ public AColGroup appendNInternal(AColGroup[] g) {
+ for(int i = 0; i < g.length; i++)
+ if(!Arrays.equals(_colIndexes, g[i]._colIndexes) || !this._dict.eq(((ColGroupConst) g[i])._dict))
+ return null;
+ return this;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
index ce2c691cca..9ef46554d0 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
@@ -506,6 +506,12 @@ public class ColGroupDDC extends APreAgg {
return null;
}
+
+ @Override
+ public AColGroup appendNInternal(AColGroup[] g) {
+ return null;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java
index b2f3ff1dd5..effc025de1 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java
@@ -445,6 +445,11 @@ public class ColGroupDDCFOR extends AMorphingMMColGroup {
return null;
}
+ @Override
+ public AColGroup appendNInternal(AColGroup[] g) {
+ return null;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java
index 27230957ec..b5133c73e3 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java
@@ -319,4 +319,12 @@ public class ColGroupEmpty extends AColGroupCompressed {
return null;
}
+ @Override
+ public AColGroup appendNInternal(AColGroup[] g) {
+ for(int i = 0; i < g.length; i++)
+ if(!Arrays.equals(_colIndexes, g[i]._colIndexes))
+ return null;
+ return this;
+ }
+
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java
index 45e474e3b8..4482dfe6ed 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupLinearFunctional.java
@@ -670,4 +670,9 @@ public class ColGroupLinearFunctional extends AColGroupCompressed {
return null;
}
+ @Override
+ public AColGroup appendNInternal(AColGroup[] g) {
+ return null;
+ }
+
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java
index 04fb526513..d92fda137a 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java
@@ -660,4 +660,9 @@ public class ColGroupOLE extends AColGroupOffset {
return null;
}
+ @Override
+ public AColGroup appendNInternal(AColGroup[] g) {
+ return null;
+ }
+
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java
index bda8175267..426ad96493 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java
@@ -972,6 +972,11 @@ public class ColGroupRLE extends AColGroupOffset {
return null;
}
+ @Override
+ public AColGroup appendNInternal(AColGroup[] g) {
+ return null;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
index f266f9a8fb..d36cd0289b 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
@@ -582,6 +582,11 @@ public class ColGroupSDC extends ASDC {
return null;
}
+ @Override
+ public AColGroup appendNInternal(AColGroup[] g) {
+ return null;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
index a47edbbf72..30f0e7568e 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
@@ -444,6 +444,11 @@ public class ColGroupSDCFOR extends ASDC {
return null;
}
+ @Override
+ public AColGroup appendNInternal(AColGroup[] g) {
+ return null;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java
index af397f5049..6a2851e753 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java
@@ -581,6 +581,11 @@ public class ColGroupSDCSingle extends ASDC {
return null;
}
+ @Override
+ public AColGroup appendNInternal(AColGroup[] g) {
+ return null;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java
index 24eb000398..e044446b33 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java
@@ -814,6 +814,11 @@ public class ColGroupSDCSingleZeros extends ASDCZero {
return null;
}
+ @Override
+ public AColGroup appendNInternal(AColGroup[] g) {
+ return null;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java
index 5d9e2b4ddb..2d4fc037e8 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java
@@ -725,6 +725,11 @@ public class ColGroupSDCZeros extends ASDCZero {
return null;
}
+ @Override
+ public AColGroup appendNInternal(AColGroup[] g) {
+ return null;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
index 7c173cb6d1..a56d681d31 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
@@ -775,6 +775,11 @@ public class ColGroupUncompressed extends AColGroup {
return null;
}
+ @Override
+ public AColGroup appendNInternal(AColGroup[] g) {
+ return null;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombine.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombine.java
index ca54f7d181..1e01a3b423 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombine.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombine.java
@@ -187,9 +187,6 @@ public class CLALibCombine {
}
private static AColGroup combineN(AColGroup[] groups) {
- AColGroup r = groups[0];
- for(int i = 1; i < groups.length && r != null; i++)
- r = r.append(groups[i]);
- return r;
+ return AColGroup.appendN(groups);
}
}
diff --git a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java
index 135228ffdd..02bd763e8b 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupNegativeTests.java
@@ -360,6 +360,12 @@ public class ColGroupNegativeTests {
// TODO Auto-generated method stub
return null;
}
+
+ @Override
+ protected AColGroup appendNInternal(AColGroup[] groups) {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
private class FakeDictBasedColGroup extends ADictBasedColGroup {
@@ -572,5 +578,11 @@ public class ColGroupNegativeTests {
// TODO Auto-generated method stub
return null;
}
+
+ @Override
+ protected AColGroup appendNInternal(AColGroup[] groups) {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
}
[systemds] 02/08: [SYSTEMDS-3446] DDC Append
Posted by ba...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 07dd88aba39547b2ba00ae44f86d9a0f83444b0e
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Sat Oct 15 16:09:12 2022 +0200
[SYSTEMDS-3446] DDC Append
---
.../sysds/runtime/compress/colgroup/AColGroup.java | 8 +++++
.../compress/colgroup/AColGroupCompressed.java | 7 +++++
.../runtime/compress/colgroup/ColGroupDDC.java | 8 +++++
.../compress/colgroup/ColGroupUncompressed.java | 5 ++++
.../compress/colgroup/dictionary/Dictionary.java | 1 -
.../compress/colgroup/mapping/AMapToData.java | 3 ++
.../compress/colgroup/mapping/MapToBit.java | 19 ++++++++++++
.../compress/colgroup/mapping/MapToByte.java | 26 +++++++++++++++-
.../compress/colgroup/mapping/MapToChar.java | 35 +++++++++++++++++-----
.../compress/colgroup/mapping/MapToCharPByte.java | 24 +++++++++++++++
.../compress/colgroup/mapping/MapToInt.java | 20 +++++++++++++
.../compress/colgroup/mapping/MapToZero.java | 9 ++++++
.../component/compress/combine/CombineTest.java | 28 ++++++++++++++++-
13 files changed, 182 insertions(+), 11 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java
index 785a0d3087..efaef22cde 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java
@@ -516,6 +516,14 @@ public abstract class AColGroup implements Serializable {
*/
public abstract double getMax();
+ /**
+ * Short hand method for getting the sum of this column group
+ *
+ * @param nRows The number of rows in the column group
+ * @return The sum of this column group
+ */
+ public abstract double getSum(int nRows);
+
/**
* Detect if the column group contains a specific value.
*
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupCompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupCompressed.java
index cd0fa77c6d..83cdafe595 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupCompressed.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroupCompressed.java
@@ -115,6 +115,13 @@ public abstract class AColGroupCompressed extends AColGroup {
return computeMxx(Double.NEGATIVE_INFINITY, Builtin.getBuiltinFnObject(BuiltinCode.MAX));
}
+ @Override
+ public double getSum(int nRows) {
+ double[] ret = new double[1];
+ computeSum(ret, nRows);
+ return ret[0];
+ }
+
@Override
public final void unaryAggregateOperations(AggregateUnaryOperator op, double[] c, int nRows, int rl, int ru) {
unaryAggregateOperations(op, c, nRows, rl, ru,
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
index 6f5346ad4d..008d66ce1b 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
@@ -22,6 +22,7 @@ package org.apache.sysds.runtime.compress.colgroup;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.commons.lang.NotImplementedException;
import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
@@ -485,6 +486,13 @@ public class ColGroupDDC extends APreAgg {
@Override
public AColGroup append(AColGroup g) {
+ if(g instanceof ColGroupDDC && Arrays.equals(g.getColIndices(), _colIndexes)) {
+ ColGroupDDC gDDC = (ColGroupDDC) g;
+ if(gDDC._dict.eq(_dict)){
+ AMapToData nd = _data.append(gDDC._data);
+ return create(_colIndexes, _dict, nd, null);
+ }
+ }
return null;
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
index 8cccf580c2..fa888261f1 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java
@@ -477,6 +477,11 @@ public class ColGroupUncompressed extends AColGroup {
return _data.max();
}
+ @Override
+ public double getSum(int nRows) {
+ return _data.sum();
+ }
+
@Override
public final void tsmm(MatrixBlock ret, int nRows) {
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java
index 530c622f26..7cfc49e32b 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/Dictionary.java
@@ -1077,7 +1077,6 @@ public class Dictionary extends ADictionary {
final double[] dv = mb.getDenseBlockValues();
return Arrays.equals(_values, dv);
}
-
return false;
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java
index 943e8c7027..b9bec0b742 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java
@@ -815,6 +815,9 @@ public abstract class AMapToData implements Serializable {
*/
public abstract AMapToData slice(int l, int u);
+
+ public abstract AMapToData append(AMapToData t);
+
@Override
public String toString() {
final int sz = size();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToBit.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToBit.java
index 7bcb462fb6..d2f9f49cae 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToBit.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToBit.java
@@ -24,6 +24,7 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.BitSet;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
import org.apache.sysds.utils.MemoryEstimates;
@@ -324,4 +325,22 @@ public class MapToBit extends AMapToData {
public AMapToData slice(int l, int u) {
return new MapToBit(getUnique(), _data.get(l,u), u - l);
}
+
+ @Override
+ public AMapToData append(AMapToData t) {
+ if(t instanceof MapToBit){
+ MapToBit tb = (MapToBit) t;
+ BitSet tbb = tb._data;
+ final int newSize = _size + t.size();
+ BitSet ret = new BitSet(newSize);
+ ret.xor(_data);
+
+ tbb.stream().forEach(x -> ret.set(x + _size, true));
+ return new MapToBit(2, ret, newSize);
+ }
+ else{
+ throw new NotImplementedException("Not implemented append on Bit map different type");
+
+ }
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToByte.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToByte.java
index 7ae3e9448d..e0e13bf1e7 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToByte.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToByte.java
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.BitSet;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
import org.apache.sysds.utils.MemoryEstimates;
@@ -147,7 +148,7 @@ public class MapToByte extends AMapToData {
public int[] getCounts(int[] ret) {
for(int i = 0; i < _data.length; i++)
ret[_data[i] & 0xFF]++;
- return ret;
+ return ret;
}
@Override
@@ -209,4 +210,27 @@ public class MapToByte extends AMapToData {
public AMapToData slice(int l, int u) {
return new MapToByte(getUnique(), Arrays.copyOfRange(_data, l, u));
}
+
+ @Override
+ public AMapToData append(AMapToData t) {
+ if(t instanceof MapToByte) {
+ MapToByte tb = (MapToByte) t;
+ byte[] tbb = tb._data;
+ final int newSize = _data.length + t.size();
+ final int newDistinct = Math.max(getUnique(), t.getUnique());
+
+ // copy
+ byte[] ret = Arrays.copyOf(_data, newSize);
+ System.arraycopy(tbb, 0, ret, _data.length, t.size());
+
+ // return
+ if(newDistinct < 127)
+ return new MapToUByte(newDistinct, ret);
+ else
+ return new MapToByte(newDistinct, ret);
+ }
+ else {
+ throw new NotImplementedException("Not implemented append on Bit map different type");
+ }
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToChar.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToChar.java
index 693cee52ec..b309861ee2 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToChar.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToChar.java
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.BitSet;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
import org.apache.sysds.utils.MemoryEstimates;
@@ -171,16 +172,15 @@ public class MapToChar extends AMapToData {
}
}
-
@Override
public int getUpperBoundValue() {
return Character.MAX_VALUE;
}
@Override
- public void copyInt(int[] d){
+ public void copyInt(int[] d) {
for(int i = 0; i < _data.length; i++)
- _data[i] = (char)d[i];
+ _data[i] = (char) d[i];
}
@Override
@@ -193,23 +193,23 @@ public class MapToChar extends AMapToData {
@Override
public int[] getCounts(int[] ret) {
for(int i = 0; i < _data.length; i++)
- ret[_data[i]]++;
+ ret[_data[i]]++;
return ret;
}
@Override
- public AMapToData resize(int unique){
+ public AMapToData resize(int unique) {
final int size = _data.length;
AMapToData ret;
if(unique <= 1)
return new MapToZero(size);
else if(unique == 2 && size > 32)
ret = new MapToBit(unique, size);
- else if (unique <= 127)
+ else if(unique <= 127)
ret = new MapToUByte(unique, size);
else if(unique < 256)
ret = new MapToByte(unique, size);
- else{
+ else {
setUnique(unique);
return this;
}
@@ -221,7 +221,7 @@ public class MapToChar extends AMapToData {
public int countRuns() {
int c = 1;
char prev = _data[0];
- for(int i = 1; i <_data.length; i++){
+ for(int i = 1; i < _data.length; i++) {
c += prev == _data[i] ? 0 : 1;
prev = _data[i];
}
@@ -232,4 +232,23 @@ public class MapToChar extends AMapToData {
public AMapToData slice(int l, int u) {
return new MapToChar(getUnique(), Arrays.copyOfRange(_data, l, u));
}
+
+ @Override
+ public AMapToData append(AMapToData t) {
+ if(t instanceof MapToChar) {
+ MapToChar tb = (MapToChar) t;
+ char[] tbb = tb._data;
+ final int newSize = _data.length + t.size();
+ final int newDistinct = Math.max(getUnique(), t.getUnique());
+
+ // copy
+ char[] ret = Arrays.copyOf(_data, newSize);
+ System.arraycopy(tbb, 0, ret, _data.length, t.size());
+
+ return new MapToChar(newDistinct, ret);
+ }
+ else {
+ throw new NotImplementedException("Not implemented append on Bit map different type");
+ }
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToCharPByte.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToCharPByte.java
index 6109024bc4..88b2e435e1 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToCharPByte.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToCharPByte.java
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.BitSet;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
import org.apache.sysds.utils.MemoryEstimates;
@@ -217,4 +218,27 @@ public class MapToCharPByte extends AMapToData {
public AMapToData slice(int l, int u) {
return new MapToCharPByte(getUnique(), Arrays.copyOfRange(_data_c, l, u), Arrays.copyOfRange(_data_b, l, u));
}
+
+ @Override
+ public AMapToData append(AMapToData t) {
+ if(t instanceof MapToCharPByte) {
+ MapToCharPByte tb = (MapToCharPByte) t;
+ char[] tbb = tb._data_c;
+ byte[] tbbb = tb._data_b;
+ final int newSize = _data_c.length + t.size();
+ final int newDistinct = Math.max(getUnique(), t.getUnique());
+
+ // copy
+ char[] ret_c = Arrays.copyOf(_data_c, newSize);
+ System.arraycopy(tbb, 0, ret_c, _data_c.length, t.size());
+ byte[] ret_b = Arrays.copyOf(_data_b, newSize);
+ System.arraycopy(tbbb, 0, ret_b, _data_b.length, t.size());
+
+
+ return new MapToCharPByte(newDistinct, ret_c, ret_b);
+ }
+ else {
+ throw new NotImplementedException("Not implemented append on Bit map different type");
+ }
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToInt.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToInt.java
index d40a7a5ccb..b4117cd5b8 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToInt.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToInt.java
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.BitSet;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
import org.apache.sysds.utils.MemoryEstimates;
@@ -232,4 +233,23 @@ public class MapToInt extends AMapToData {
public AMapToData slice(int l, int u) {
return new MapToInt(getUnique(), Arrays.copyOfRange(_data, l, u));
}
+
+ @Override
+ public AMapToData append(AMapToData t) {
+ if(t instanceof MapToInt) {
+ MapToInt tb = (MapToInt) t;
+ int[] tbb = tb._data;
+ final int newSize = _data.length + t.size();
+ final int newDistinct = Math.max(getUnique(), t.getUnique());
+
+ // copy
+ int[] ret = Arrays.copyOf(_data, newSize);
+ System.arraycopy(tbb, 0, ret, _data.length, t.size());
+
+ return new MapToInt(newDistinct, ret);
+ }
+ else {
+ throw new NotImplementedException("Not implemented append on Bit map different type");
+ }
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToZero.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToZero.java
index 5fc2c51178..c3cd14afc4 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToZero.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToZero.java
@@ -24,6 +24,7 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.BitSet;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
@@ -149,4 +150,12 @@ public class MapToZero extends AMapToData {
public AMapToData slice(int l, int u) {
return new MapToZero(u - l);
}
+
+ @Override
+ public AMapToData append(AMapToData t) {
+ if(t instanceof MapToZero)
+ return new MapToZero(_size + t.size());
+ else
+ throw new NotImplementedException("Not implemented append on Bit map different type");
+ }
}
diff --git a/src/test/java/org/apache/sysds/test/component/compress/combine/CombineTest.java b/src/test/java/org/apache/sysds/test/component/compress/combine/CombineTest.java
index 86736b99f3..8db991ef9f 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/combine/CombineTest.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/combine/CombineTest.java
@@ -26,15 +26,23 @@ import static org.junit.Assert.fail;
import java.util.HashMap;
import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory;
+import org.apache.sysds.runtime.compress.CompressionSettingsBuilder;
+import org.apache.sysds.runtime.compress.colgroup.AColGroup;
+import org.apache.sysds.runtime.compress.colgroup.AColGroup.CompressionType;
import org.apache.sysds.runtime.compress.lib.CLALibCombine;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.matrix.data.MatrixIndexes;
+import org.apache.sysds.test.TestUtils;
import org.junit.Test;
public class CombineTest {
+ protected static final Log LOG = LogFactory.getLog(CombineTest.class.getName());
+
@Test
public void combineEmpty() {
CompressedMatrixBlock m1 = CompressedMatrixBlockFactory.createConstant(100, 10, 0.0);
@@ -56,7 +64,6 @@ public class CombineTest {
}
-
@Test
public void combineConst() {
CompressedMatrixBlock m1 = CompressedMatrixBlockFactory.createConstant(100, 10, 1.0);
@@ -78,4 +85,23 @@ public class CombineTest {
}
+ @Test
+ public void combineDDC() {
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(165, 2, 1, 3, 1.0, 2514));
+ CompressedMatrixBlock csb = (CompressedMatrixBlock) CompressedMatrixBlockFactory
+ .compress(mb,
+ new CompressionSettingsBuilder().clearValidCompression().addValidCompression(CompressionType.DDC))
+ .getLeft();
+
+ AColGroup g = csb.getColGroups().get(0);
+ double sum = g.getSum(165);
+ AColGroup ret = g.append(g);
+ double sum2 = ret.getSum(165 * 2);
+ assertEquals(sum * 2, sum2, 0.001);
+ AColGroup ret2 = ret.append(g);
+ double sum3 = ret2.getSum(165 * 3);
+ assertEquals(sum * 3, sum3, 0.001);
+
+ }
+
}
[systemds] 07/08: [SYSTEMDS-3445] N Way combining Mapping
Posted by ba...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit ead3c5e9fbbfa6884d0a226abf029b2ab7157424
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Wed Oct 19 18:23:08 2022 +0200
[SYSTEMDS-3445] N Way combining Mapping
This commit adds support for n way combining / appending AMapToData objects.
This enables for instance DDC to do n-way-combining as well.
---
.../runtime/compress/colgroup/AMapToDataGroup.java | 26 ++++++
.../runtime/compress/colgroup/ColGroupDDC.java | 31 ++++++-
.../runtime/compress/colgroup/ColGroupDDCFOR.java | 2 +-
.../runtime/compress/colgroup/ColGroupOLE.java | 2 +-
.../runtime/compress/colgroup/ColGroupRLE.java | 2 +-
.../runtime/compress/colgroup/ColGroupSDC.java | 2 +-
.../runtime/compress/colgroup/ColGroupSDCFOR.java | 2 +-
.../compress/colgroup/ColGroupSDCSingle.java | 2 +-
.../compress/colgroup/ColGroupSDCSingleZeros.java | 2 +-
.../compress/colgroup/ColGroupSDCZeros.java | 2 +-
.../compress/colgroup/mapping/AMapToData.java | 3 +
.../compress/colgroup/mapping/MapToBit.java | 60 +++++++++++++-
.../compress/colgroup/mapping/MapToByte.java | 28 ++++++-
.../compress/colgroup/mapping/MapToChar.java | 19 +++++
.../compress/colgroup/mapping/MapToCharPByte.java | 6 ++
.../compress/colgroup/mapping/MapToInt.java | 19 +++++
.../compress/colgroup/mapping/MapToZero.java | 13 ++-
.../component/compress/combine/CombineTest.java | 16 ++--
.../component/compress/mapping/MappingTests.java | 96 +++++++++++++++++++++-
19 files changed, 302 insertions(+), 31 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AMapToDataGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AMapToDataGroup.java
new file mode 100644
index 0000000000..50b65898a1
--- /dev/null
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AMapToDataGroup.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.compress.colgroup;
+
+import org.apache.sysds.runtime.compress.colgroup.mapping.AMapToData;
+
+public interface AMapToDataGroup {
+ public AMapToData getMapToData();
+}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
index 9ef46554d0..ba642e7df9 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDC.java
@@ -46,7 +46,7 @@ import org.apache.sysds.runtime.matrix.operators.UnaryOperator;
/**
* Class to encapsulate information about a column group that is encoded with dense dictionary encoding (DDC).
*/
-public class ColGroupDDC extends APreAgg {
+public class ColGroupDDC extends APreAgg implements AMapToDataGroup {
private static final long serialVersionUID = -5769772089913918987L;
protected final AMapToData _data;
@@ -122,6 +122,11 @@ public class ColGroupDDC extends APreAgg {
}
+ @Override
+ public AMapToData getMapToData(){
+ return _data;
+ }
+
private void decompressToDenseBlockDenseDictSingleColOutContiguous(DenseBlock db, int rl, int ru, int offR, int offC,
double[] values) {
final double[] c = db.values(0);
@@ -481,7 +486,7 @@ public class ColGroupDDC extends APreAgg {
@Override
protected AColGroup copyAndSet(int[] colIndexes, ADictionary newDictionary) {
- return create(colIndexes, newDictionary, _data, getCounts());
+ return create(colIndexes, newDictionary, _data, getCachedCounts());
}
@Override
@@ -506,10 +511,28 @@ public class ColGroupDDC extends APreAgg {
return null;
}
-
@Override
public AColGroup appendNInternal(AColGroup[] g) {
- return null;
+ for(int i = 1; i < g.length; i++) {
+ if(!Arrays.equals(_colIndexes, g[i]._colIndexes)) {
+ LOG.warn("Not same columns therefore not appending DDC\n" + Arrays.toString(_colIndexes) + "\n\n"
+ + Arrays.toString(g[i]._colIndexes));
+ return null;
+ }
+
+ if(!(g[i] instanceof ColGroupDDC)) {
+ LOG.warn("Not DDC but " + g[i].getClass().getSimpleName() + ", therefore not appending DDC");
+ return null;
+ }
+
+ final ColGroupDDC gDDC = (ColGroupDDC) g[i];
+ if(!gDDC._dict.eq(_dict)) {
+ LOG.warn("Not same Dictionaries therefore not appending DDC\n" + _dict + "\n\n" + gDDC._dict);
+ return null;
+ }
+ }
+ AMapToData nd = _data.appendN(Arrays.copyOf(g, g.length, AMapToDataGroup[].class));
+ return create(_colIndexes, _dict, nd, null);
}
@Override
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java
index effc025de1..ca0b2b46f8 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDDCFOR.java
@@ -430,7 +430,7 @@ public class ColGroupDDCFOR extends AMorphingMMColGroup {
@Override
protected AColGroup copyAndSet(int[] colIndexes, ADictionary newDictionary) {
- return create(colIndexes, newDictionary, _data, getCounts(), _reference);
+ return create(colIndexes, newDictionary, _data, getCachedCounts(), _reference);
}
@Override
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java
index d92fda137a..c966f585fe 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupOLE.java
@@ -652,7 +652,7 @@ public class ColGroupOLE extends AColGroupOffset {
@Override
protected AColGroup copyAndSet(int[] colIndexes, ADictionary newDictionary) {
- return create(colIndexes, _numRows, _zeros, newDictionary, _data, _ptr, getCounts());
+ return create(colIndexes, _numRows, _zeros, newDictionary, _data, _ptr, getCachedCounts());
}
@Override
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java
index 426ad96493..b923ba7c2c 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupRLE.java
@@ -964,7 +964,7 @@ public class ColGroupRLE extends AColGroupOffset {
@Override
protected AColGroup copyAndSet(int[] colIndexes, ADictionary newDictionary) {
- return create(colIndexes, _numRows, _zeros, newDictionary, _data, _ptr, getCounts());
+ return create(colIndexes, _numRows, _zeros, newDictionary, _data, _ptr, getCachedCounts());
}
@Override
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
index d36cd0289b..d055ade891 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDC.java
@@ -566,7 +566,7 @@ public class ColGroupSDC extends ASDC {
@Override
protected AColGroup copyAndSet(int[] colIndexes, ADictionary newDictionary) {
- return create(colIndexes, _numRows, newDictionary, _defaultTuple, _indexes, _data, getCounts());
+ return create(colIndexes, _numRows, newDictionary, _defaultTuple, _indexes, _data, getCachedCounts());
}
@Override
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
index 30f0e7568e..5a1b9212df 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCFOR.java
@@ -436,7 +436,7 @@ public class ColGroupSDCFOR extends ASDC {
@Override
protected AColGroup copyAndSet(int[] colIndexes, ADictionary newDictionary) {
- return create(colIndexes, _numRows, newDictionary, _indexes, _data, getCounts(), _reference);
+ return create(colIndexes, _numRows, newDictionary, _indexes, _data, getCachedCounts(), _reference);
}
@Override
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java
index 6a2851e753..55372423cc 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingle.java
@@ -573,7 +573,7 @@ public class ColGroupSDCSingle extends ASDC {
@Override
protected AColGroup copyAndSet(int[] colIndexes, ADictionary newDictionary) {
- return create(colIndexes, _numRows, newDictionary, _defaultTuple, _indexes, getCounts());
+ return create(colIndexes, _numRows, newDictionary, _defaultTuple, _indexes, getCachedCounts());
}
@Override
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java
index e044446b33..1731c7799b 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java
@@ -806,7 +806,7 @@ public class ColGroupSDCSingleZeros extends ASDCZero {
@Override
protected AColGroup copyAndSet(int[] colIndexes, ADictionary newDictionary) {
- return create(colIndexes, _numRows, newDictionary, _indexes, getCounts());
+ return create(colIndexes, _numRows, newDictionary, _indexes, getCachedCounts());
}
@Override
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java
index 2d4fc037e8..961e63570e 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCZeros.java
@@ -717,7 +717,7 @@ public class ColGroupSDCZeros extends ASDCZero {
@Override
protected AColGroup copyAndSet(int[] colIndexes, ADictionary newDictionary) {
- return create(colIndexes, _numRows, newDictionary, _indexes, _data, getCounts());
+ return create(colIndexes, _numRows, newDictionary, _indexes, _data, getCachedCounts());
}
@Override
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java
index b9bec0b742..4e781cf5bc 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/AMapToData.java
@@ -27,6 +27,7 @@ import java.util.BitSet;
import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.sysds.runtime.compress.colgroup.AMapToDataGroup;
import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
import org.apache.sysds.runtime.compress.colgroup.dictionary.Dictionary;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
@@ -818,6 +819,8 @@ public abstract class AMapToData implements Serializable {
public abstract AMapToData append(AMapToData t);
+ public abstract AMapToData appendN(AMapToDataGroup[] d);
+
@Override
public String toString() {
final int sz = size();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToBit.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToBit.java
index d2f9f49cae..4d06e1efd6 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToBit.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToBit.java
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.util.BitSet;
import org.apache.commons.lang.NotImplementedException;
+import org.apache.sysds.runtime.compress.colgroup.AMapToDataGroup;
import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
import org.apache.sysds.utils.MemoryEstimates;
@@ -323,12 +324,12 @@ public class MapToBit extends AMapToData {
@Override
public AMapToData slice(int l, int u) {
- return new MapToBit(getUnique(), _data.get(l,u), u - l);
+ return new MapToBit(getUnique(), _data.get(l, u), u - l);
}
@Override
public AMapToData append(AMapToData t) {
- if(t instanceof MapToBit){
+ if(t instanceof MapToBit) {
MapToBit tb = (MapToBit) t;
BitSet tbb = tb._data;
final int newSize = _size + t.size();
@@ -338,9 +339,62 @@ public class MapToBit extends AMapToData {
tbb.stream().forEach(x -> ret.set(x + _size, true));
return new MapToBit(2, ret, newSize);
}
- else{
+ else {
throw new NotImplementedException("Not implemented append on Bit map different type");
}
}
+
+ @Override
+ public AMapToData appendN(AMapToDataGroup[] d) {
+ int p = 0; // pointer
+ for(AMapToDataGroup gd : d)
+ p += gd.getMapToData().size();
+ final long[] ret = new long[(p - 1) / 64 + 1];
+ long[] or = _data.toLongArray();
+ System.arraycopy(or, 0, ret, 0, or.length);
+
+ p = size();
+ for(int i = 1; i < d.length; i++) {
+ final MapToBit mm = (MapToBit) d[i].getMapToData();
+ final int ms = mm.size();
+ or = mm._data.toLongArray();
+ final int remainder = p % 64;
+ int retLp = p / 64;
+ if(remainder == 0)// Easy lining up
+ System.arraycopy(or, 0, ret, retLp, or.length);
+ else { // Not Lining up
+ // all but last
+ for(int j = 0; j < or.length - 1; j++) {
+ long v = or[j];
+ ret[retLp] = ret[retLp] ^ (v << remainder);
+ retLp++;
+ ret[retLp] = v >>> (64 - remainder);
+ }
+ // last
+ long v = or[or.length - 1];
+ ret[retLp] = ret[retLp] ^ (v << remainder);
+ retLp++;
+ if(retLp < ret.length)
+ ret[retLp] = v >>> (64 - remainder);
+ }
+ p += ms;
+ }
+
+ BitSet retBS = BitSet.valueOf(ret);
+ return new MapToBit(getUnique(), retBS, p);
+
+ }
+
+ private static String bl(long l) {
+ int lead = Long.numberOfLeadingZeros(l);
+ if(lead == 64)
+ return "0000000000000000000000000000000000000000000000000000000000000000";
+ StringBuilder sb = new StringBuilder(64);
+ for(int i = 0; i < lead; i++) {
+ sb.append('0');
+ }
+ sb.append(Long.toBinaryString(l));
+ return sb.toString();
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToByte.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToByte.java
index e0e13bf1e7..81eb974490 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToByte.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToByte.java
@@ -26,6 +26,7 @@ import java.util.Arrays;
import java.util.BitSet;
import org.apache.commons.lang.NotImplementedException;
+import org.apache.sysds.runtime.compress.colgroup.AMapToDataGroup;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
import org.apache.sysds.utils.MemoryEstimates;
@@ -214,13 +215,13 @@ public class MapToByte extends AMapToData {
@Override
public AMapToData append(AMapToData t) {
if(t instanceof MapToByte) {
- MapToByte tb = (MapToByte) t;
- byte[] tbb = tb._data;
+ final MapToByte tb = (MapToByte) t;
+ final byte[] tbb = tb._data;
final int newSize = _data.length + t.size();
final int newDistinct = Math.max(getUnique(), t.getUnique());
// copy
- byte[] ret = Arrays.copyOf(_data, newSize);
+ final byte[] ret = Arrays.copyOf(_data, newSize);
System.arraycopy(tbb, 0, ret, _data.length, t.size());
// return
@@ -233,4 +234,25 @@ public class MapToByte extends AMapToData {
throw new NotImplementedException("Not implemented append on Bit map different type");
}
}
+
+ @Override
+ public AMapToData appendN(AMapToDataGroup[] d) {
+ int p = 0; // pointer
+ for(AMapToDataGroup gd : d)
+ p += gd.getMapToData().size();
+ final byte[] ret = Arrays.copyOf(_data, p);
+
+ p = size();
+ for(int i = 1; i < d.length; i++) {
+ final MapToByte mm = (MapToByte) d[i].getMapToData();
+ final int ms = mm.size();
+ System.arraycopy(mm._data, 0, ret, p, ms);
+ p += ms;
+ }
+
+ if(getUnique() < 127)
+ return new MapToUByte(getUnique(), ret);
+ else
+ return new MapToByte(getUnique(), ret);
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToChar.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToChar.java
index b309861ee2..f24c8a72e8 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToChar.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToChar.java
@@ -26,6 +26,7 @@ import java.util.Arrays;
import java.util.BitSet;
import org.apache.commons.lang.NotImplementedException;
+import org.apache.sysds.runtime.compress.colgroup.AMapToDataGroup;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
import org.apache.sysds.utils.MemoryEstimates;
@@ -251,4 +252,22 @@ public class MapToChar extends AMapToData {
throw new NotImplementedException("Not implemented append on Bit map different type");
}
}
+
+ @Override
+ public AMapToData appendN(AMapToDataGroup[] d) {
+ int p = 0; // pointer
+ for(AMapToDataGroup gd : d)
+ p += gd.getMapToData().size();
+ final char[] ret = Arrays.copyOf(_data, p);
+
+ p = size();
+ for(int i = 1; i < d.length; i++) {
+ final MapToChar mm = (MapToChar) d[i].getMapToData();
+ final int ms = mm.size();
+ System.arraycopy(mm._data, 0, ret, p, ms);
+ p += ms;
+ }
+
+ return new MapToChar(getUnique(), ret);
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToCharPByte.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToCharPByte.java
index 88b2e435e1..f1483a2bcd 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToCharPByte.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToCharPByte.java
@@ -26,6 +26,7 @@ import java.util.Arrays;
import java.util.BitSet;
import org.apache.commons.lang.NotImplementedException;
+import org.apache.sysds.runtime.compress.colgroup.AMapToDataGroup;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
import org.apache.sysds.utils.MemoryEstimates;
@@ -241,4 +242,9 @@ public class MapToCharPByte extends AMapToData {
throw new NotImplementedException("Not implemented append on Bit map different type");
}
}
+
+ @Override
+ public AMapToData appendN(AMapToDataGroup[] d){
+ throw new NotImplementedException();
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToInt.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToInt.java
index b4117cd5b8..2324b2bac0 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToInt.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToInt.java
@@ -26,6 +26,7 @@ import java.util.Arrays;
import java.util.BitSet;
import org.apache.commons.lang.NotImplementedException;
+import org.apache.sysds.runtime.compress.colgroup.AMapToDataGroup;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
import org.apache.sysds.utils.MemoryEstimates;
@@ -252,4 +253,22 @@ public class MapToInt extends AMapToData {
throw new NotImplementedException("Not implemented append on Bit map different type");
}
}
+
+ @Override
+ public AMapToData appendN(AMapToDataGroup[] d) {
+ int p = 0; // pointer
+ for(AMapToDataGroup gd : d)
+ p += gd.getMapToData().size();
+ final int[] ret = Arrays.copyOf(_data, p);
+
+ p = size();
+ for(int i = 1; i < d.length; i++) {
+ final MapToInt mm = (MapToInt) d[i].getMapToData();
+ final int ms = mm.size();
+ System.arraycopy(mm._data, 0, ret, p, ms);
+ p += ms;
+ }
+
+ return new MapToInt(getUnique(), ret);
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToZero.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToZero.java
index c3cd14afc4..8e2115e5b8 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToZero.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/mapping/MapToZero.java
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.util.BitSet;
import org.apache.commons.lang.NotImplementedException;
+import org.apache.sysds.runtime.compress.colgroup.AMapToDataGroup;
import org.apache.sysds.runtime.compress.colgroup.dictionary.ADictionary;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory.MAP_TYPE;
@@ -153,9 +154,17 @@ public class MapToZero extends AMapToData {
@Override
public AMapToData append(AMapToData t) {
- if(t instanceof MapToZero)
+ if(t instanceof MapToZero)
return new MapToZero(_size + t.size());
- else
+ else
throw new NotImplementedException("Not implemented append on Bit map different type");
}
+
+ @Override
+ public AMapToData appendN(AMapToDataGroup[] d) {
+ int p = 0; // pointer
+ for(AMapToDataGroup gd : d)
+ p += gd.getMapToData().size();
+ return new MapToZero(p);
+ }
}
diff --git a/src/test/java/org/apache/sysds/test/component/compress/combine/CombineTest.java b/src/test/java/org/apache/sysds/test/component/compress/combine/CombineTest.java
index 8db991ef9f..4223e5c773 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/combine/CombineTest.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/combine/CombineTest.java
@@ -87,13 +87,7 @@ public class CombineTest {
@Test
public void combineDDC() {
- MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(165, 2, 1, 3, 1.0, 2514));
- CompressedMatrixBlock csb = (CompressedMatrixBlock) CompressedMatrixBlockFactory
- .compress(mb,
- new CompressionSettingsBuilder().clearValidCompression().addValidCompression(CompressionType.DDC))
- .getLeft();
-
- AColGroup g = csb.getColGroups().get(0);
+ AColGroup g = getDDC();
double sum = g.getSum(165);
AColGroup ret = g.append(g);
double sum2 = ret.getSum(165 * 2);
@@ -101,7 +95,15 @@ public class CombineTest {
AColGroup ret2 = ret.append(g);
double sum3 = ret2.getSum(165 * 3);
assertEquals(sum * 3, sum3, 0.001);
+ }
+ private static AColGroup getDDC(){
+ MatrixBlock mb = TestUtils.ceil(TestUtils.generateTestMatrixBlock(165, 2, 1, 3, 1.0, 2514));
+ CompressedMatrixBlock csb = (CompressedMatrixBlock) CompressedMatrixBlockFactory
+ .compress(mb,
+ new CompressionSettingsBuilder().clearValidCompression().addValidCompression(CompressionType.DDC))
+ .getLeft();
+ return csb.getColGroups().get(0);
}
}
diff --git a/src/test/java/org/apache/sysds/test/component/compress/mapping/MappingTests.java b/src/test/java/org/apache/sysds/test/component/compress/mapping/MappingTests.java
index 4d9712efcb..c1e9046830 100644
--- a/src/test/java/org/apache/sysds/test/component/compress/mapping/MappingTests.java
+++ b/src/test/java/org/apache/sysds/test/component/compress/mapping/MappingTests.java
@@ -32,8 +32,10 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Random;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.sysds.runtime.compress.colgroup.AMapToDataGroup;
import org.apache.sysds.runtime.compress.colgroup.mapping.AMapToData;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToCharPByte;
import org.apache.sysds.runtime.compress.colgroup.mapping.MapToFactory;
@@ -68,8 +70,14 @@ public class MappingTests {
tests.add(new Object[] {6, t, 63, false});
tests.add(new Object[] {4, t, 63, false});
tests.add(new Object[] {3, t, 64, false});
+ tests.add(new Object[] {4, t, 64, false});
tests.add(new Object[] {3, t, 65, false});
tests.add(new Object[] {5, t, 64 + 63, false});
+ tests.add(new Object[] {5, t, 127, false});
+ tests.add(new Object[] {5, t, 128, false});
+ tests.add(new Object[] {5, t, 129, false});
+ tests.add(new Object[] {7, t, 255, false});
+ tests.add(new Object[] {8, t, 256, false});
tests.add(new Object[] {5, t, 1234, false});
tests.add(new Object[] {5, t, 13, true});
}
@@ -80,7 +88,7 @@ public class MappingTests {
this.seed = seed;
this.type = type;
this.size = size;
- this.max =Math.min(Math.min(MappingTestUtil.getUpperBoundValue(type), fictiveMax) + 1, size);
+ this.max = Math.min(Math.min(MappingTestUtil.getUpperBoundValue(type), fictiveMax) + 1, size);
expected = new int[size];
m = genMap(MapToFactory.create(size, max), expected, max, fill, seed);
}
@@ -113,7 +121,7 @@ public class MappingTests {
}
// to make sure that the bit set is actually filled.
- for(int i = 0; i < max; i++){
+ for(int i = 0; i < max; i++) {
m.set(i, i);
expected[i] = i;
@@ -126,7 +134,6 @@ public class MappingTests {
for(int i = 0; i < size; i++)
if(expected[i] != m.getIndex(i))
fail("Expected equals " + Arrays.toString(expected) + "\nbut got: " + m);
-
}
@Test
@@ -214,7 +221,7 @@ public class MappingTests {
@Test
public void replaceMax() {
- m.replace(max-1, 0);
+ m.replace(max - 1, 0);
for(int i = 0; i < size; i++) {
expected[i] = expected[i] == max - 1 ? 0 : expected[i];
@@ -271,4 +278,85 @@ public class MappingTests {
+ m.getType() + " " + type + " " + max + " " + m);
}
+ @Test
+ public void testAppend() {
+ int nVal = m.getUnique();
+ int[] counts = m.getCounts(new int[nVal]);
+
+ AMapToData m2 = m.append(m);
+ assertEquals(m.size() * 2, m2.size());
+ assertEquals(m.getUnique(), m2.getUnique());
+ int[] counts2 = m2.getCounts(new int[nVal]);
+
+ for(int i = 0; i < nVal; i++)
+ assertEquals(counts[i] * 2, counts2[i]);
+ }
+
+ @Test
+ public void testAppendN() {
+ int nVal = m.getUnique();
+ int[] counts = m.getCounts(new int[nVal]);
+
+ try {
+
+ AMapToData m2 = m.appendN(new AMapToDataGroup[] {//
+ new Holder(m), new Holder(m), new Holder(m)});
+ try {
+ assertEquals(m.size() * 3, m2.size());
+ assertEquals(m.getUnique(), m2.getUnique());
+ int[] counts2 = m2.getCounts(new int[nVal]);
+
+ for(int i = 0; i < nVal; i++)
+ assertEquals(counts[i] * 3, counts2[i]);
+ }
+ catch(AssertionError e) {
+ fail(e.getMessage() + "\nFailed appendN with in: \n" + m + "\ncomp:\n" + m2);
+ }
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail("Failed " + e.getMessage());
+ }
+
+ }
+
+ @Test
+ public void testAppendVSAppendN() {
+ final AMapToData m2 = m.append(m).append(m);
+ final AMapToData m3 = m.appendN(new AMapToDataGroup[] {//
+ new Holder(m), new Holder(m), new Holder(m)});
+ compare(m2, m3);
+ }
+
+ @Test(expected = NotImplementedException.class)
+ public void testAppendNotSame() {
+ AMapToData mm;
+ switch(type) {
+ case INT:
+ mm = MapToFactory.create(size, MAP_TYPE.CHAR_BYTE);
+ mm.copy(m);
+ m.append(mm);
+ break;
+ default:
+ mm = MapToFactory.create(size, MAP_TYPE.INT);
+ mm.copy(m);
+ m.append(mm);
+ }
+ LOG.error("Did not throw exception with: " + m);
+ }
+
+ private static class Holder implements AMapToDataGroup {
+
+ AMapToData d;
+
+ protected Holder(AMapToData d) {
+ this.d = d;
+ }
+
+ @Override
+ public AMapToData getMapToData() {
+ return d;
+ }
+
+ }
}
[systemds] 05/08: [MINOR] Disable Hadoop Write compression
Posted by ba...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 678b28a2210dcdf11841d9486d1dac4fdacb036d
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Wed Oct 19 16:38:16 2022 +0200
[MINOR] Disable Hadoop Write compression
---
.../sysds/runtime/compress/io/CompressWrap.java | 1 -
.../runtime/compress/io/WriterCompressed.java | 3 +-
.../sysds/runtime/compress/lib/CLALibCombine.java | 93 +++++++++++-----------
3 files changed, 48 insertions(+), 49 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/compress/io/CompressWrap.java b/src/main/java/org/apache/sysds/runtime/compress/io/CompressWrap.java
index e3e370b694..c17edc2b6d 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/io/CompressWrap.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/io/CompressWrap.java
@@ -20,7 +20,6 @@
package org.apache.sysds.runtime.compress.io;
import org.apache.spark.api.java.function.Function;
-import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
public class CompressWrap implements Function<MatrixBlock, CompressedWriteBlock> {
diff --git a/src/main/java/org/apache/sysds/runtime/compress/io/WriterCompressed.java b/src/main/java/org/apache/sysds/runtime/compress/io/WriterCompressed.java
index d2194b98f2..82f2be59e4 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/io/WriterCompressed.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/io/WriterCompressed.java
@@ -124,7 +124,8 @@ public final class WriterCompressed extends MatrixWriter {
// Make Writer (New interface)
final Writer w = SequenceFile.createWriter(job, Writer.file(path), Writer.bufferSize(4096),
Writer.blockSize(4096), Writer.keyClass(MatrixIndexes.class), Writer.valueClass(CompressedWriteBlock.class),
- Writer.compression(SequenceFile.CompressionType.RECORD), Writer.replication((short) 1));
+ Writer.compression(SequenceFile.CompressionType.NONE), // No Compression type on disk
+ Writer.replication((short) 1));
final int rlen = src.getNumRows();
final int clen = src.getNumColumns();
diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombine.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombine.java
index 7600167250..ca54f7d181 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombine.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCombine.java
@@ -32,13 +32,14 @@ import org.apache.sysds.runtime.compress.colgroup.AColGroup.CompressionType;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.matrix.data.MatrixIndexes;
+import edu.emory.mathcs.backport.java.util.Arrays;
+
public class CLALibCombine {
protected static final Log LOG = LogFactory.getLog(CLALibCombine.class.getName());
public static MatrixBlock combine(Map<MatrixIndexes, MatrixBlock> m) {
// Dynamically find rlen, clen and blen;
-
// assume that the blen is the same in all blocks.
// assume that all blocks are there ...
final MatrixIndexes lookup = new MatrixIndexes(1, 1);
@@ -57,15 +58,21 @@ public class CLALibCombine {
lookup.setIndexes(1, lookup.getColumnIndex() + 1);
}
- return combine(m, (int) rows, (int) cols, blen);
+ return combine(m, lookup, (int) rows, (int) cols, blen);
}
public static MatrixBlock combine(Map<MatrixIndexes, MatrixBlock> m, int rlen, int clen, int blen) {
+ final MatrixIndexes lookup = new MatrixIndexes();
+ return combine(m, lookup, rlen, clen, blen);
+ }
+
+ private static MatrixBlock combine(final Map<MatrixIndexes, MatrixBlock> m, final MatrixIndexes lookup,
+ final int rlen, final int clen, final int blen) {
if(rlen < blen) // Shortcut, in case file only contains one block in r length.
- return CombiningColumnGroups(m, rlen, clen, blen);
+ return CombiningColumnGroups(m, lookup, rlen, clen, blen);
+
final CompressionType[] colTypes = new CompressionType[clen];
- final MatrixIndexes lookup = new MatrixIndexes();
// Look through the first blocks in to the top.
for(int bc = 0; bc * blen < clen; bc++) {
lookup.setIndexes(1, bc + 1); // get first blocks
@@ -119,10 +126,11 @@ public class CLALibCombine {
}
}
- return CombiningColumnGroups(m, rlen, clen, blen);
+ return CombiningColumnGroups(m, lookup, rlen, clen, blen);
}
- private static MatrixBlock combineViaDecompression(Map<MatrixIndexes, MatrixBlock> m, int rlen, int clen, int blen) {
+ private static MatrixBlock combineViaDecompression(final Map<MatrixIndexes, MatrixBlock> m, final int rlen,
+ final int clen, final int blen) {
final MatrixBlock out = new MatrixBlock(rlen, clen, false);
out.allocateDenseBlock();
for(Entry<MatrixIndexes, MatrixBlock> e : m.entrySet()) {
@@ -140,57 +148,48 @@ public class CLALibCombine {
}
// It is known all of the matrices are Compressed and they are non overlapping.
- private static MatrixBlock CombiningColumnGroups(Map<MatrixIndexes, MatrixBlock> m, int rlen, int clen, int blen) {
+ private static MatrixBlock CombiningColumnGroups(final Map<MatrixIndexes, MatrixBlock> m, final MatrixIndexes lookup,
+ final int rlen, final int clen, final int blen) {
- final AColGroup[] finalCols = new AColGroup[clen];
- final MatrixIndexes lookup = new MatrixIndexes();
- for(int bc = 0; bc * blen < clen; bc++) {
- lookup.setIndexes(1, bc + 1); // get first blocks
- final MatrixBlock b = m.get(lookup);
- final CompressedMatrixBlock cmb = (CompressedMatrixBlock) b;
-
- final List<AColGroup> gs = cmb.getColGroups();
- for(AColGroup g : gs) {
- AColGroup gc = g;
- if(bc > 0)
- gc = g.shiftColIndices(bc * blen);
- final int[] cols = gc.getColIndices();
- finalCols[cols[0]] = gc; // only assign first column of each group.
- }
- }
+ final AColGroup[][] finalCols = new AColGroup[clen][]; // temp array for combining
+ final int blocksInColumn = (rlen - 1) / blen + 1;
+ final int nGroups = m.size() / blocksInColumn;
- for(int br = 1; br * blen < rlen; br++) {
+ // Add all the blocks into linear structure.
+ for(int br = 0; br * blen < rlen; br++) {
for(int bc = 0; bc * blen < clen; bc++) {
- lookup.setIndexes(br + 1, bc + 1); // get first blocks
- final MatrixBlock b = m.get(lookup);
-
- final CompressedMatrixBlock cmb = (CompressedMatrixBlock) b;
-
- final List<AColGroup> gs = cmb.getColGroups();
- for(AColGroup g : gs) {
- AColGroup gc = g;
- if(bc > 0)
- gc = g.shiftColIndices(bc * blen);
+ lookup.setIndexes(br + 1, bc + 1);
+ final CompressedMatrixBlock cmb = (CompressedMatrixBlock) m.get(lookup);
+ for(AColGroup g : cmb.getColGroups()) {
+ final AColGroup gc = bc > 0 ? g.shiftColIndices(bc * blen) : g;
final int[] cols = gc.getColIndices();
- AColGroup prev = finalCols[cols[0]];
- AColGroup comb = prev.append(gc);
- if(comb == null) {
- LOG.warn("Combining of columns from group: " + prev.getClass().getSimpleName() + " and "
- + gc.getClass().getSimpleName() + " was non trivial, therefore falling back to decompression");
- return combineViaDecompression(m, rlen, clen, blen);
- }
- finalCols[cols[0]] = comb;
+ if(br == 0)
+ finalCols[cols[0]] = new AColGroup[blocksInColumn];
+ finalCols[cols[0]][br] = gc;
}
}
}
- List<AColGroup> finalGroups = new ArrayList<>();
-
- for(AColGroup g : finalCols)
- if(g != null)
- finalGroups.add(g);
+ final List<AColGroup> finalGroups = new ArrayList<>(nGroups);
+ for(AColGroup[] colOfGroups : finalCols) {
+ if(colOfGroups != null) { // skip null entries
+ final AColGroup combined = combineN(colOfGroups);
+ if(combined == null) {
+ LOG.warn("Combining of columns from group failed: " + Arrays.toString(colOfGroups));
+ return combineViaDecompression(m, rlen, clen, blen);
+ }
+ finalGroups.add(combined);
+ }
+ }
return new CompressedMatrixBlock(rlen, clen, -1, false, finalGroups);
}
+
+ private static AColGroup combineN(AColGroup[] groups) {
+ AColGroup r = groups[0];
+ for(int i = 1; i < groups.length && r != null; i++)
+ r = r.append(groups[i]);
+ return r;
+ }
}