You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by mb...@apache.org on 2024/08/16 13:18:51 UTC
(systemds) branch main updated: [MINOR] Improved code coverage bufferpool and related components
This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new b054bd8cce [MINOR] Improved code coverage bufferpool and related components
b054bd8cce is described below
commit b054bd8cce515a413b23bd91dd8ebb054736f166
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Fri Aug 16 15:18:17 2024 +0200
[MINOR] Improved code coverage bufferpool and related components
---
.../runtime/controlprogram/caching/ByteBuffer.java | 6 +-
.../caching/CacheMaintenanceService.java | 19 +--
.../controlprogram/caching/CacheStatistics.java | 17 ---
.../controlprogram/caching/CacheableData.java | 4 +-
.../runtime/controlprogram/caching/PageCache.java | 5 +-
.../controlprogram/parfor/RemoteParForUtils.java | 40 -------
.../sysds/runtime/frame/data/FrameBlock.java | 5 +-
.../sysds/runtime/matrix/data/MatrixBlock.java | 4 +
.../sysds/runtime/meta/DataCharacteristics.java | 131 +++++---------------
.../sysds/runtime/meta/MatrixCharacteristics.java | 30 ++++-
.../sysds/runtime/meta/TensorCharacteristics.java | 86 +++++++++++++
.../test/functions/caching/BufferpoolLeakTest.java | 22 +++-
.../functions/caching/CachingPWriteExportTest.java | 11 +-
.../functions/caching/KeyBufferPoolComponents.java | 133 +++++++++++++++++++++
.../sysds/test/functions/caching/UMMTest.java | 2 +
15 files changed, 318 insertions(+), 197 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/ByteBuffer.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/ByteBuffer.java
index 8c74de792f..48da215f88 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/ByteBuffer.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/ByteBuffer.java
@@ -25,6 +25,7 @@ import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.IOException;
+import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.data.DenseBlockLDRB;
import org.apache.sysds.runtime.frame.data.FrameBlock;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
@@ -51,8 +52,7 @@ public class ByteBuffer
}
public void serializeBlock( CacheBlock<?> cb )
- throws IOException
- {
+ {
_shallow = cb.isShallowSerialize(true);
_matrix = (cb instanceof MatrixBlock);
@@ -77,7 +77,7 @@ public class ByteBuffer
}
}
catch(Exception ex) {
- throw new IOException("Failed to serialize cache block.", ex);
+ throw new DMLRuntimeException("Failed to serialize cache block.", ex);
}
_serialized = true;
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheMaintenanceService.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheMaintenanceService.java
index f4454e66d9..ca68183653 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheMaintenanceService.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheMaintenanceService.java
@@ -19,10 +19,8 @@
package org.apache.sysds.runtime.controlprogram.caching;
-import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.util.LocalFileUtils;
-import java.io.IOException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -48,14 +46,8 @@ public class CacheMaintenanceService
//sync or async file delete
if( CacheableData.CACHING_ASYNC_SERIALIZE )
_pool.submit(new CacheMaintenanceService.DataSerializerTask(bbuff, cb));
- else {
- try {
- bbuff.serializeBlock(cb);
- }
- catch(IOException ex) {
- throw new DMLRuntimeException(ex);
- }
- }
+ else
+ bbuff.serializeBlock(cb);
}
public void close() {
@@ -94,12 +86,7 @@ public class CacheMaintenanceService
@Override
public void run() {
- try {
- _bbuff.serializeBlock(_cb);
- }
- catch(IOException ex) {
- throw new DMLRuntimeException(ex);
- }
+ _bbuff.serializeBlock(_cb);
}
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheStatistics.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheStatistics.java
index fe525be9cd..2e8f8a9353 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheStatistics.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheStatistics.java
@@ -34,23 +34,6 @@ import java.util.concurrent.atomic.LongAdder;
*/
public class CacheStatistics
{
- //enum used for MR counters
- public enum Stat {
- CACHE_HITS_MEM,
- CACHE_HITS_FSBUFF,
- CACHE_HITS_FS,
- CACHE_HITS_HDFS,
- CACHE_HITS_LIN,
- CACHE_WRITES_FSBUFF,
- CACHE_WRITES_FS,
- CACHE_WRITES_HDFS,
- CACHE_WRITES_LIN,
- CACHE_TIME_ACQR, //acquire read
- CACHE_TIME_ACQM, //acquire read
- CACHE_TIME_RLS, //release
- CACHE_TIME_EXP, //export
- }
-
//hit statistics (for acquire read)
private static final LongAdder _numHitsMem = new LongAdder();
private static final LongAdder _numHitsFSBuff = new LongAdder();
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java
index b70e583765..8e37dc29a2 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java
@@ -83,11 +83,11 @@ public abstract class CacheableData<T extends CacheBlock<?>> extends Data
// global constant configuration parameters
public static final long CACHING_THRESHOLD = (long)Math.max(4*1024, //obj not s.t. caching
1e-5 * InfrastructureAnalyzer.getLocalMaxMemory()); //if below threshold [in bytes]
- public static final RPolicy CACHING_BUFFER_POLICY = RPolicy.FIFO;
+ public static RPolicy CACHING_BUFFER_POLICY = RPolicy.FIFO;
public static final String CACHING_COUNTER_GROUP_NAME = "SystemDS Caching Counters";
public static final String CACHING_EVICTION_FILEEXTENSION = ".dat";
public static final boolean CACHING_ASYNC_FILECLEANUP = true;
- public static final boolean CACHING_ASYNC_SERIALIZE = false;
+ public static boolean CACHING_ASYNC_SERIALIZE = false;
//NOTE CACHING_ASYNC_SERIALIZE:
// The serialization of matrices and frames (ultra-sparse matrices or
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/PageCache.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/PageCache.java
index 70878bc6ba..e0d11ed2b8 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/PageCache.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/PageCache.java
@@ -32,8 +32,7 @@ public class PageCache
_pool = new HashMap<>();
}
- public static void clear()
- {
+ public static void clear() {
_pool = null;
}
@@ -48,7 +47,7 @@ public class PageCache
list = new LinkedList<>();
_pool.put(data.length, list);
}
- list.addLast(new SoftReference<>(data));
+ list.addLast(new SoftReference<>(data));
}
public static byte[] getPage( int size )
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/parfor/RemoteParForUtils.java b/src/main/java/org/apache/sysds/runtime/controlprogram/parfor/RemoteParForUtils.java
index 4a5a898c5d..b0c1c6eb45 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/parfor/RemoteParForUtils.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/parfor/RemoteParForUtils.java
@@ -26,21 +26,14 @@ import java.util.List;
import java.util.Map.Entry;
import org.apache.commons.logging.Log;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.sysds.api.DMLScript;
import org.apache.sysds.conf.ConfigurationManager;
import org.apache.sysds.hops.OptimizerUtils;
import org.apache.sysds.common.Types.DataType;
import org.apache.sysds.parser.ParForStatementBlock.ResultVar;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.controlprogram.LocalVariableMap;
-import org.apache.sysds.runtime.controlprogram.ParForProgramBlock;
-import org.apache.sysds.runtime.controlprogram.caching.CacheStatistics;
import org.apache.sysds.runtime.controlprogram.caching.CacheableData;
import org.apache.sysds.runtime.controlprogram.caching.MatrixObject;
-import org.apache.sysds.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
-import org.apache.sysds.runtime.controlprogram.parfor.stat.Stat;
import org.apache.sysds.runtime.controlprogram.parfor.util.IDHandler;
import org.apache.sysds.runtime.instructions.cp.Data;
import org.apache.sysds.runtime.instructions.cp.ListObject;
@@ -54,7 +47,6 @@ import static org.apache.sysds.utils.Explain.explain;
import org.apache.sysds.runtime.util.HDFSTool;
import org.apache.sysds.runtime.util.LocalFileUtils;
import org.apache.sysds.runtime.util.ProgramConverter;
-import org.apache.sysds.utils.Statistics;
import scala.Tuple2;
@@ -65,38 +57,6 @@ import scala.Tuple2;
*/
public class RemoteParForUtils
{
- public static void incrementParForMRCounters(Reporter reporter, long deltaTasks, long deltaIterations)
- {
- //report parfor counters
- if( deltaTasks>0 )
- reporter.incrCounter(ParForProgramBlock.PARFOR_COUNTER_GROUP_NAME, Stat.PARFOR_NUMTASKS.toString(), deltaTasks);
- if( deltaIterations>0 )
- reporter.incrCounter(ParForProgramBlock.PARFOR_COUNTER_GROUP_NAME, Stat.PARFOR_NUMITERS.toString(), deltaIterations);
-
- JobConf job = ConfigurationManager.getCachedJobConf();
- if( DMLScript.STATISTICS && !InfrastructureAnalyzer.isLocalMode(job) )
- {
- //report cache statistics
- reporter.incrCounter( ParForProgramBlock.PARFOR_COUNTER_GROUP_NAME, Stat.PARFOR_JITCOMPILE.toString(), Statistics.getJITCompileTime());
- reporter.incrCounter( ParForProgramBlock.PARFOR_COUNTER_GROUP_NAME, Stat.PARFOR_JVMGC_COUNT.toString(), Statistics.getJVMgcCount());
- reporter.incrCounter( ParForProgramBlock.PARFOR_COUNTER_GROUP_NAME, Stat.PARFOR_JVMGC_TIME.toString(), Statistics.getJVMgcTime());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_HITS_MEM.toString(), CacheStatistics.getMemHits());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_HITS_FSBUFF.toString(), CacheStatistics.getFSBuffHits());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_HITS_FS.toString(), CacheStatistics.getFSHits());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_HITS_HDFS.toString(), CacheStatistics.getHDFSHits());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_WRITES_FSBUFF.toString(), CacheStatistics.getFSBuffWrites());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_WRITES_FS.toString(), CacheStatistics.getFSWrites());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_WRITES_HDFS.toString(), CacheStatistics.getHDFSWrites());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_TIME_ACQR.toString(), CacheStatistics.getAcquireRTime());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_TIME_ACQM.toString(), CacheStatistics.getAcquireMTime());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_TIME_RLS.toString(), CacheStatistics.getReleaseTime());
- reporter.incrCounter( CacheableData.CACHING_COUNTER_GROUP_NAME, CacheStatistics.Stat.CACHE_TIME_EXP.toString(), CacheStatistics.getExportTime());
-
- //reset cache statistics to prevent overlapping reporting
- CacheStatistics.reset();
- }
- }
-
/**
* For remote Spark parfor workers. This is a simplified version compared to MR.
*
diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java b/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
index e1f2625cb0..4896d82f28 100644
--- a/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
@@ -846,8 +846,9 @@ public class FrameBlock implements CacheBlock<FrameBlock>, Externalizable {
// meta data array (overhead and entries)
size += MemoryEstimates.objectArrayCost(clen);
- for(ColumnMetadata mtd : _colmeta)
- size += mtd == null ? 8 : mtd.getInMemorySize();
+ if( _colmeta != null )
+ for(ColumnMetadata mtd : _colmeta)
+ size += mtd == null ? 8 : mtd.getInMemorySize();
// data array
size += MemoryEstimates.objectArrayCost(clen);
diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
index cbf9485c34..16a9503699 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java
@@ -5526,6 +5526,10 @@ public class MatrixBlock extends MatrixValue implements CacheBlock<MatrixBlock>,
// (rand, sequence)
+ public static MatrixBlock randOperations(int rows, int cols, double sparsity) {
+ return randOperations(rows, cols, sparsity, 0, 1, "uniform", -1);
+ }
+
/**
* Function to generate the random matrix with specified dimensions (block sizes are not specified).
*
diff --git a/src/main/java/org/apache/sysds/runtime/meta/DataCharacteristics.java b/src/main/java/org/apache/sysds/runtime/meta/DataCharacteristics.java
index 17900cbf3c..073434ad81 100644
--- a/src/main/java/org/apache/sysds/runtime/meta/DataCharacteristics.java
+++ b/src/main/java/org/apache/sysds/runtime/meta/DataCharacteristics.java
@@ -19,7 +19,6 @@
package org.apache.sysds.runtime.meta;
-import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.matrix.operators.AggregateBinaryOperator;
import org.apache.sysds.runtime.matrix.operators.AggregateUnaryOperator;
import org.apache.sysds.runtime.matrix.operators.ReorgOperator;
@@ -32,45 +31,25 @@ public abstract class DataCharacteristics implements Serializable {
protected int _blocksize; // squared block size
protected boolean _noEmptyBlocks = false; // does not materialize empty blocks
- public DataCharacteristics set(long nr, long nc, int blen) {
- throw new DMLRuntimeException("DataCharacteristics.set(long, long, int): should never get called in the base class");
- }
+ public abstract DataCharacteristics set(long nr, long nc, int blen);
- public DataCharacteristics set(long nr, long nc, int blen, long nnz) {
- throw new DMLRuntimeException("DataCharacteristics.set(long, long, int, long): should never get called in the base class");
- }
+ public abstract DataCharacteristics set(long nr, long nc, int blen, long nnz);
- public DataCharacteristics set(long[] dims, int blocksize) {
- throw new DMLRuntimeException("DataCharacteristics.set(long[], int): should never get called in the base class");
- }
+ public abstract DataCharacteristics set(long[] dims, int blocksize);
- public DataCharacteristics set(long[] dims, int blocksize, long nnz) {
- throw new DMLRuntimeException("DataCharacteristics.set(long[], int, long): should never get called in the base class");
- }
+ public abstract DataCharacteristics set(long[] dims, int blocksize, long nnz);
- public DataCharacteristics set(DataCharacteristics that) {
- throw new DMLRuntimeException("DataCharacteristics.set(DataCharacteristics): should never get called in the base class");
- }
+ public abstract DataCharacteristics set(DataCharacteristics that);
- public long getRows() {
- throw new DMLRuntimeException("DataCharacteristics.getRows(): should never get called in the base class");
- }
+ public abstract long getRows();
- public DataCharacteristics setRows(long rlen) {
- throw new DMLRuntimeException("DataCharacteristics.setRows(long): should never get called in the base class");
- }
+ public abstract DataCharacteristics setRows(long rlen);
- public long getCols() {
- throw new DMLRuntimeException("DataCharacteristics.getCols(): should never get called in the base class");
- }
+ public abstract long getCols();
- public DataCharacteristics setCols(long clen) {
- throw new DMLRuntimeException("DataCharacteristics.setCols(long): should never get called in the base class");
- }
+ public abstract DataCharacteristics setCols(long clen);
- public long getLength() {
- throw new DMLRuntimeException("DataCharacteristics.getLength(): should never get called in the base class");
- }
+ public abstract long getLength();
public int getBlocksize() {
return _blocksize;
@@ -92,100 +71,56 @@ public abstract class DataCharacteristics implements Serializable {
}
public long getNumBlocks() {
- throw new DMLRuntimeException("DataCharacteristics.getNumBlocks(int): should never get called in the base class");
+ return getNumRowBlocks() * getNumColBlocks();
}
- public long getNumRowBlocks() {
- throw new DMLRuntimeException("DataCharacteristics.getNumRowBlocks(): should never get called in the base class");
- }
+ public abstract long getNumRowBlocks();
- public long getNumColBlocks() {
- throw new DMLRuntimeException("DataCharacteristics.getNumColBlocks(): should never get called in the base class");
- }
+ public abstract long getNumColBlocks();
- public DataCharacteristics setDimension(long nr, long nc) {
- throw new DMLRuntimeException("DataCharacteristics.setDimension(long, long): should never get called in the base class");
- }
+ public abstract DataCharacteristics setDimension(long nr, long nc);
- public int getNumDims() {
- throw new DMLRuntimeException("DataCharacteristics.getNumDims(): should never get called in the base class");
- }
+ public abstract int getNumDims();
- public long getDim(int i) {
- throw new DMLRuntimeException("DataCharacteristics.getDim(int): should never get called in the base class");
- }
+ public abstract long getDim(int i);
public long[] getDims() {
return getLongDims();
}
- public long[] getLongDims() {
- throw new DMLRuntimeException("DataCharacteristics.getLongDims(): should never get called in the base class");
- }
+ public abstract long[] getLongDims();
- public int[] getIntDims() {
- throw new DMLRuntimeException("DataCharacteristics.getIntDims(): should never get called in the base class");
- }
+ public abstract int[] getIntDims();
- public DataCharacteristics setDim(int i, long dim) {
- throw new DMLRuntimeException("DataCharacteristics.setDim(int, long): should never get called in the base class");
- }
+ public abstract DataCharacteristics setDim(int i, long dim);
- public DataCharacteristics setDims(long[] dims) {
- throw new DMLRuntimeException("DataCharacteristics.setDims(long[]): should never get called in the base class");
- }
+ public abstract DataCharacteristics setDims(long[] dims);
- public long getNumBlocks(int i) {
- throw new DMLRuntimeException("DataCharacteristics.getNumBlocks(i): should never get called in the base class");
- }
+ public abstract long getNumBlocks(int i);
- public DataCharacteristics setNonZeros(long nnz) {
- throw new DMLRuntimeException("DataCharacteristics.setNonZeros(long): should never get called in the base class");
- }
+ public abstract DataCharacteristics setNonZeros(long nnz);
- public long getNonZeros() {
- throw new DMLRuntimeException("DataCharacteristics.getNonZeros(): should never get called in the base class");
- }
+ public abstract long getNonZeros();
- public DataCharacteristics setNonZerosBound(long nnz) {
- throw new DMLRuntimeException("DataCharacteristics.setNonZerosBound(long): should never get called in the base class");
- }
+ public abstract DataCharacteristics setNonZerosBound(long nnz);
- public long getNonZerosBound() {
- throw new DMLRuntimeException("DataCharacteristics.getNonZerosBound(): should never get called in the base class");
- }
+ public abstract long getNonZerosBound();
- public double getSparsity() {
- throw new DMLRuntimeException("DataCharacteristics.getSparsity(): should never get called in the base class");
- }
+ public abstract double getSparsity();
- public boolean dimsKnown() {
- throw new DMLRuntimeException("DataCharacteristics.dimsKnown(): should never get called in the base class");
- }
+ public abstract boolean dimsKnown();
- public boolean dimsKnown(boolean includeNnz) {
- throw new DMLRuntimeException("DataCharacteristics.dimsKnown(boolean): should never get called in the base class");
- }
+ public abstract boolean dimsKnown(boolean includeNnz);
- public boolean rowsKnown() {
- throw new DMLRuntimeException("DataCharacteristics.rowsKnown(): should never get called in the base class");
- }
+ public abstract boolean rowsKnown();
- public boolean colsKnown() {
- throw new DMLRuntimeException("DataCharacteristics.colsKnown(): should never get called in the base class");
- }
+ public abstract boolean colsKnown();
- public boolean nnzKnown() {
- throw new DMLRuntimeException("DataCharacteristics.nnzKnown(): should never get called in the base class");
- }
+ public abstract boolean nnzKnown();
- public boolean isUltraSparse() {
- throw new DMLRuntimeException("DataCharacteristics.isUltraSparse(): should never get called in the base class");
- }
+ public abstract boolean isUltraSparse();
- public boolean mightHaveEmptyBlocks() {
- throw new DMLRuntimeException("DataCharacteristics.mightHaveEmptyBlocks(): should never get called in the base class");
- }
+ public abstract boolean mightHaveEmptyBlocks();
public static void reorg(DataCharacteristics dim, ReorgOperator op, DataCharacteristics dimOut) {
op.fn.computeDimension(dim, dimOut);
diff --git a/src/main/java/org/apache/sysds/runtime/meta/MatrixCharacteristics.java b/src/main/java/org/apache/sysds/runtime/meta/MatrixCharacteristics.java
index 82c3aba08d..ff769df44a 100644
--- a/src/main/java/org/apache/sysds/runtime/meta/MatrixCharacteristics.java
+++ b/src/main/java/org/apache/sysds/runtime/meta/MatrixCharacteristics.java
@@ -112,11 +112,6 @@ public class MatrixCharacteristics extends DataCharacteristics
return numRows * numColumns;
}
- @Override
- public long getNumBlocks() {
- return getNumRowBlocks() * getNumColBlocks();
- }
-
@Override
public long getNumRowBlocks() {
//number of row blocks w/ awareness of zero rows
@@ -274,4 +269,29 @@ public class MatrixCharacteristics extends DataCharacteristics
sb.append(")]");
return sb.toString();
}
+
+ @Override
+ public DataCharacteristics set(long[] dims, int blocksize) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DataCharacteristics set(long[] dims, int blocksize, long nnz) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DataCharacteristics setDim(int i, long dim) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DataCharacteristics setDims(long[] dims) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getNumBlocks(int i) {
+ throw new UnsupportedOperationException();
+ }
}
diff --git a/src/main/java/org/apache/sysds/runtime/meta/TensorCharacteristics.java b/src/main/java/org/apache/sysds/runtime/meta/TensorCharacteristics.java
index 20e11d955f..42b34d8c28 100644
--- a/src/main/java/org/apache/sysds/runtime/meta/TensorCharacteristics.java
+++ b/src/main/java/org/apache/sysds/runtime/meta/TensorCharacteristics.java
@@ -190,4 +190,90 @@ public class TensorCharacteristics extends DataCharacteristics
return UtilFunctions.intHashCode(UtilFunctions.intHashCode(
Arrays.hashCode(_dims), _blocksize), Long.hashCode(_nnz));
}
+
+ @Override
+ public DataCharacteristics set(long nr, long nc, int blen) {
+ _dims = new long[] {nr, nc};
+ _blocksize = blen;
+ return this;
+ }
+
+ @Override
+ public DataCharacteristics set(long nr, long nc, int blen, long nnz) {
+ set(nr, nc, blen);
+ _nnz = nnz;
+ return this;
+ }
+
+ @Override
+ public DataCharacteristics setRows(long rlen) {
+ _dims[0] = rlen;
+ return this;
+ }
+
+ @Override
+ public DataCharacteristics setCols(long clen) {
+ _dims[1] = clen;
+ return this;
+ }
+
+ @Override
+ public long getNumRowBlocks() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public long getNumColBlocks() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public DataCharacteristics setDimension(long nr, long nc) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public DataCharacteristics setNonZerosBound(long nnz) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public long getNonZerosBound() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public double getSparsity() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public boolean rowsKnown() {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public boolean colsKnown() {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public boolean isUltraSparse() {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public boolean mightHaveEmptyBlocks() {
+ // TODO Auto-generated method stub
+ return false;
+ }
}
diff --git a/src/test/java/org/apache/sysds/test/functions/caching/BufferpoolLeakTest.java b/src/test/java/org/apache/sysds/test/functions/caching/BufferpoolLeakTest.java
index e652c5c6f6..0eca06730f 100644
--- a/src/test/java/org/apache/sysds/test/functions/caching/BufferpoolLeakTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/caching/BufferpoolLeakTest.java
@@ -22,6 +22,9 @@ package org.apache.sysds.test.functions.caching;
import org.junit.Assert;
import org.junit.Test;
import org.apache.sysds.runtime.controlprogram.caching.CacheStatistics;
+import org.apache.sysds.runtime.controlprogram.caching.CacheableData;
+import org.apache.sysds.runtime.controlprogram.caching.LazyWriteBuffer;
+import org.apache.sysds.runtime.controlprogram.caching.LazyWriteBuffer.RPolicy;
import org.apache.sysds.test.AutomatedTestBase;
import org.apache.sysds.test.TestConfiguration;
@@ -38,15 +41,27 @@ public class BufferpoolLeakTest extends AutomatedTestBase
}
@Test
- public void testLeak1() {
- runTestBufferpoolLeak(10000, 15);
+ public void testLeak1_FIFO() {
+ runTestBufferpoolLeak(10000, 15, RPolicy.FIFO, false);
}
- private void runTestBufferpoolLeak(int rows, int cols) {
+ @Test
+ public void testLeak1_LRU() {
+ runTestBufferpoolLeak(10000, 15, RPolicy.LRU, false);
+ }
+
+ @Test
+ public void testLeak1_FIFO_Async() {
+ runTestBufferpoolLeak(10000, 15, RPolicy.FIFO, true);
+ }
+
+ private void runTestBufferpoolLeak(int rows, int cols, RPolicy policy, boolean asyncSerialize) {
TestConfiguration config = getTestConfiguration(TEST_NAME);
config.addVariable("rows", rows);
config.addVariable("cols", cols);
loadTestConfiguration(config);
+ CacheableData.CACHING_BUFFER_POLICY = policy;
+ CacheableData.CACHING_ASYNC_SERIALIZE = asyncSerialize;
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + TEST_NAME + ".dml";
@@ -55,6 +70,7 @@ public class BufferpoolLeakTest extends AutomatedTestBase
//run test and check no evictions
runTest(true, false, null, -1);
+ LazyWriteBuffer.printStatus("tests");
Assert.assertEquals(0, CacheStatistics.getFSWrites());
}
}
diff --git a/src/test/java/org/apache/sysds/test/functions/caching/CachingPWriteExportTest.java b/src/test/java/org/apache/sysds/test/functions/caching/CachingPWriteExportTest.java
index 6ebca6ba3a..3131e5f46d 100644
--- a/src/test/java/org/apache/sysds/test/functions/caching/CachingPWriteExportTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/caching/CachingPWriteExportTest.java
@@ -19,7 +19,6 @@
package org.apache.sysds.test.functions.caching;
-import org.junit.Assert;
import org.junit.Test;
import org.apache.sysds.common.Types.FileFormat;
import org.apache.sysds.hops.Hop;
@@ -28,10 +27,10 @@ import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.util.DataConverter;
import org.apache.sysds.test.AutomatedTestBase;
import org.apache.sysds.test.TestConfiguration;
+import org.apache.sysds.test.TestUtils;
public class CachingPWriteExportTest extends AutomatedTestBase
{
-
private final static String TEST_NAME = "export";
private final static String TEST_DIR = "functions/caching/";
private final static String TEST_CLASS_DIR = TEST_DIR + CachingPWriteExportTest.class.getSimpleName() + "/";
@@ -94,16 +93,12 @@ public class CachingPWriteExportTest extends AutomatedTestBase
ii, rows, cols, OptimizerUtils.DEFAULT_BLOCKSIZE, nnz);
Vp = DataConverter.convertToDoubleMatrix(mb);
}
- catch(Exception ex)
- {
+ catch(Exception ex) {
ex.printStackTrace();
throw new RuntimeException(ex);
}
//compare
- for( int i=0; i<rows; i++ )
- for( int j=0; j<cols; j++ )
- if( V[i][j]!=Vp[i][j] )
- Assert.fail("Wrong value i="+i+", j="+j+", value1="+V[i][j]+", value2="+Vp[i][j]);
+ TestUtils.compareMatrices(V, Vp, 1e-14);
}
}
\ No newline at end of file
diff --git a/src/test/java/org/apache/sysds/test/functions/caching/KeyBufferPoolComponents.java b/src/test/java/org/apache/sysds/test/functions/caching/KeyBufferPoolComponents.java
new file mode 100644
index 0000000000..79b2a6b793
--- /dev/null
+++ b/src/test/java/org/apache/sysds/test/functions/caching/KeyBufferPoolComponents.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.functions.caching;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+import org.apache.sysds.common.Warnings;
+import org.apache.sysds.runtime.controlprogram.caching.ByteBuffer;
+import org.apache.sysds.runtime.controlprogram.caching.CacheBlockFactory;
+import org.apache.sysds.runtime.controlprogram.caching.CacheDataInput;
+import org.apache.sysds.runtime.controlprogram.caching.CacheDataOutput;
+import org.apache.sysds.runtime.controlprogram.caching.PageCache;
+import org.apache.sysds.runtime.data.TensorBlock;
+import org.apache.sysds.runtime.frame.data.FrameBlock;
+import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+import org.apache.sysds.runtime.util.CommonThreadPool;
+import org.apache.sysds.test.AutomatedTestBase;
+import org.apache.sysds.test.TestUtils;
+
+public class KeyBufferPoolComponents extends AutomatedTestBase
+{
+ @Override
+ public void setUp() {}
+
+ @Test
+ public void testDataStreamsDense() {
+ testSerialization(100, 100, 0.7);
+ }
+
+ @Test
+ public void testDataStreamsSparse() {
+ testSerialization(100, 100, 0.07);
+ }
+
+ @Test
+ public void testBufferDense() {
+ testBufferSerialization(100, 100, 0.7);
+ }
+
+ @Test
+ public void testBufferSparse() {
+ testBufferSerialization(100, 100, 0.007);
+ }
+
+ @Test
+ public void testCacheBlockFactory() {
+ Assert.assertEquals(new MatrixBlock(), CacheBlockFactory.newInstance(0));
+ Assert.assertEquals(
+ new FrameBlock().getInMemorySize(), CacheBlockFactory.newInstance(1).getInMemorySize());
+ Assert.assertEquals(
+ new TensorBlock().getInMemorySize(), CacheBlockFactory.newInstance(2).getInMemorySize());
+ Assert.assertThrows(RuntimeException.class, ()->CacheBlockFactory.newInstance(3));
+
+ Assert.assertEquals(new MatrixBlock(), CacheBlockFactory.newInstance(new MatrixBlock()));
+ Assert.assertEquals(
+ new FrameBlock().getInMemorySize(), CacheBlockFactory.newInstance(new FrameBlock()).getInMemorySize());
+ Assert.assertEquals(
+ new TensorBlock().getInMemorySize(), CacheBlockFactory.newInstance(new TensorBlock()).getInMemorySize());
+ Assert.assertThrows(RuntimeException.class, ()->CacheBlockFactory.newInstance(null));
+
+ Assert.assertThrows(RuntimeException.class, ()->CacheBlockFactory.getCode(null));
+ Assert.assertThrows(RuntimeException.class, ()->CacheBlockFactory.getPairList(null));
+ }
+
+ @Test
+ public void testPageCache() {
+ //coverage for classes w/ only static methods
+ new Warnings();
+ new PageCache();
+
+ PageCache.init();
+ for(int i=7; i<256; i++) {
+ PageCache.putPage(new byte[i]);
+ PageCache.putPage(new byte[i]);
+ }
+ int count = 0;
+ for(int i=7; i<256; i++)
+ count += PageCache.getPage(i)!=null ? 1: 0;
+ System.out.println("Found "+count+" pages for reuse."); //120
+ PageCache.clear();
+ }
+
+ private void testSerialization(int rows, int cols, double sparsity) {
+ try {
+ MatrixBlock mb = MatrixBlock.randOperations(rows, cols, sparsity);
+ byte[] barr = new byte[(int)mb.getExactSizeOnDisk()];
+ CacheDataOutput dos = new CacheDataOutput(barr);
+ mb.write(dos);
+ CacheDataInput dis = new CacheDataInput(barr);
+ MatrixBlock mb2 = new MatrixBlock();
+ mb2.readFields(dis);
+ TestUtils.compareMatrices(mb, mb2, 1e-14);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void testBufferSerialization(int rows, int cols, double sparsity) {
+ try {
+ MatrixBlock mb = MatrixBlock.randOperations(rows, cols, sparsity);
+ ByteBuffer buff = new ByteBuffer((int)mb.getExactSizeOnDisk());
+ Future<?> check = CommonThreadPool.get().submit(()->buff.checkSerialized());
+ buff.serializeBlock(mb);
+ check.get(); //check non-blocking after serialization
+ MatrixBlock mb2 = (MatrixBlock) buff.deserializeBlock();
+ TestUtils.compareMatrices(mb, mb2, 1e-14);
+ } catch (IOException | InterruptedException | ExecutionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/src/test/java/org/apache/sysds/test/functions/caching/UMMTest.java b/src/test/java/org/apache/sysds/test/functions/caching/UMMTest.java
index 0bb61a83dd..00361f79b1 100644
--- a/src/test/java/org/apache/sysds/test/functions/caching/UMMTest.java
+++ b/src/test/java/org/apache/sysds/test/functions/caching/UMMTest.java
@@ -91,6 +91,8 @@ public class UMMTest extends AutomatedTestBase {
// Compare FS write counts (#unified FS writes always smaller than #static FS writes)
Assert.assertTrue("Violated buffer pool eviction counts: "+FSwrites_unified+" <= "+FSwrites_static,
FSwrites_unified <= FSwrites_static);
+
+ UnifiedMemoryManager.printStatus("tests");
}
finally {
Recompiler.reinitRecompiler();