You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2020/04/25 16:38:40 UTC
[systemml] branch master updated: [SYSTEMDS-208] Fix buffer pool
leak and cleanup robustness
This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemml.git
The following commit(s) were added to refs/heads/master by this push:
new beb4840 [SYSTEMDS-208] Fix buffer pool leak and cleanup robustness
beb4840 is described below
commit beb4840439ce6ca027470ce0cf3d2c903c1fa40d
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Sat Apr 25 16:49:45 2020 +0200
[SYSTEMDS-208] Fix buffer pool leak and cleanup robustness
This patch fixes a buffer pool eviction leak, where each calls to mice
added 3 uncleaned objects to the buffer pool and thus eventually ran
into severe eviction (up to 'no space left on device').
A closer investigation revealed missing rmVar instructions in complex
control flow programs. Specifically, we now reintroduced the notion of
exit instructions for while/for/parfor/if and derive and add a packed
rmVar instruction if necessary (based on livein and liveout sets).
To make the mentioned exit instructions more effective, this patch also
introduces a best-effort cleanup of liveout variable sets, which are too
conservative for nested control flow. However, this cleanup is only done
where it is guaranteed to be safe, i.e., the top-level of statement
blocks at the main program and individual functions.
Finally, the memory leak was due to creatvar instructions overwriting
existing objects in the symbol table without proper cleanup. This is a
consequence of missing rmvar instructions, but in order to guard against
all cases, we now check this condition and perform a proper cleanup
which guards against such unknown leaks.
---
docs/Tasks.txt | 1 +
.../org/apache/sysds/parser/DMLTranslator.java | 41 +++++++++++++--
.../runtime/controlprogram/BasicProgramBlock.java | 4 +-
.../runtime/controlprogram/ForProgramBlock.java | 3 ++
.../runtime/controlprogram/IfProgramBlock.java | 27 +++-------
.../runtime/controlprogram/ParForProgramBlock.java | 3 ++
.../sysds/runtime/controlprogram/ProgramBlock.java | 27 +++++++++-
.../runtime/controlprogram/WhileProgramBlock.java | 3 ++
.../controlprogram/caching/LazyWriteBuffer.java | 7 ++-
.../instructions/cp/VariableCPInstruction.java | 4 ++
.../sysds/runtime/util/ProgramConverter.java | 5 ++
src/main/java/org/apache/sysds/utils/Explain.java | 33 +++++++-----
.../test/functions/caching/BufferpoolLeakTest.java | 60 ++++++++++++++++++++++
.../scripts/functions/caching/BufferpoolLeak.dml | 28 ++++++++++
14 files changed, 204 insertions(+), 42 deletions(-)
diff --git a/docs/Tasks.txt b/docs/Tasks.txt
index d1e30c0..e63544c 100644
--- a/docs/Tasks.txt
+++ b/docs/Tasks.txt
@@ -166,6 +166,7 @@ SYSTEMDS-200 Various Fixes
* 205 Fix scoping of builtin dml-bodied functions (vs user-defined)
* 206 Fix codegen outer template compilation (tsmm) OK
* 207 Fix builtin function call hoisting from expressions OK
+ * 208 Fix bufferpool leak (live var analysis and createvar) OK
SYSTEMDS-210 Extended lists Operations
* 211 Cbind and Rbind over lists of matrices OK
diff --git a/src/main/java/org/apache/sysds/parser/DMLTranslator.java b/src/main/java/org/apache/sysds/parser/DMLTranslator.java
index f1f64c1..789ea9d 100644
--- a/src/main/java/org/apache/sysds/parser/DMLTranslator.java
+++ b/src/main/java/org/apache/sysds/parser/DMLTranslator.java
@@ -25,6 +25,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.logging.Log;
@@ -88,6 +89,7 @@ import org.apache.sysds.runtime.controlprogram.Program;
import org.apache.sysds.runtime.controlprogram.ProgramBlock;
import org.apache.sysds.runtime.controlprogram.WhileProgramBlock;
import org.apache.sysds.runtime.instructions.Instruction;
+import org.apache.sysds.runtime.instructions.cp.VariableCPInstruction;
public class DMLTranslator
@@ -200,6 +202,8 @@ public class DMLTranslator
currentLiveOut = sb.analyze(currentLiveOut);
}
}
+
+ cleanupLiveOutVariables(dmlp.getStatementBlocks(), new VariableSet());
}
public void liveVariableAnalysisFunction(DMLProgram dmlp, FunctionStatementBlock fsb) {
@@ -218,15 +222,32 @@ public class DMLTranslator
//STEP 2: backward direction
VariableSet currentLiveOut = new VariableSet();
VariableSet currentLiveIn = new VariableSet();
+ VariableSet unionLiveIn = new VariableSet();
for (DataIdentifier id : fstmt.getInputParams())
currentLiveIn.addVariable(id.getName(), id);
- for (DataIdentifier id : fstmt.getOutputParams())
+ for (DataIdentifier id : fstmt.getOutputParams()) {
currentLiveOut.addVariable(id.getName(), id);
+ unionLiveIn.addVariable(id.getName(), id);
+ }
fsb._liveOut = currentLiveOut;
fsb.analyze(currentLiveIn, currentLiveOut);
+ cleanupLiveOutVariables(fstmt.getBody(), unionLiveIn);
+ }
+
+ public void cleanupLiveOutVariables(List<StatementBlock> sbs, VariableSet unionLiveIn) {
+ //backwards pass to collect union of livein variables of all successors
+ //and cleanup unnecessary liveout variables
+ for(int i=sbs.size()-1; i>=0; i--) {
+ StatementBlock sb = sbs.get(i);
+ //remove liveout variables that are not in unionLivein
+ sb.liveOut().removeVariables(
+ VariableSet.minus(sb.liveOut(), unionLiveIn));
+ //collect all livein information
+ unionLiveIn.addVariables(sb.liveIn());
+ }
}
public void constructHops(DMLProgram dmlp) {
@@ -482,6 +503,9 @@ public class DMLTranslator
retPB = rtpb;
+ //post processing for generating missing instructions
+ retPB.setExitInstruction(deriveExitInstruction(sb));
+
// add statement block
retPB.setStatementBlock(sb);
@@ -525,7 +549,7 @@ public class DMLTranslator
retPB = rtpb;
//post processing for generating missing instructions
- //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
+ retPB.setExitInstruction(deriveExitInstruction(sb));
// add statement block
retPB.setStatementBlock(sb);
@@ -583,6 +607,9 @@ public class DMLTranslator
retPB = rtpb;
+ //post processing for generating missing instructions
+ retPB.setExitInstruction(deriveExitInstruction(sb));
+
// add statement block
retPB.setStatementBlock(sb);
@@ -641,7 +668,7 @@ public class DMLTranslator
retPB = rtpb;
//post processing for generating missing instructions
- //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
+ //retPB.setExitInstruction(deriveExitInstruction(sb));
// add statement block
retPB.setStatementBlock(sb);
@@ -670,6 +697,14 @@ public class DMLTranslator
}
}
+ private static Instruction deriveExitInstruction(StatementBlock sb) {
+ Set<String> rmVars = VariableSet.union(
+ VariableSet.minus(sb.liveIn(), sb.liveOut()),
+ VariableSet.minus(sb.getKill(), sb.liveOut())).getVariableNames();
+ return rmVars.isEmpty() ? null :
+ VariableCPInstruction.prepareRemoveInstruction(rmVars.toArray(new String[0]));
+ }
+
public static void refreshMemEstimates(StatementBlock current) {
MemoTable memo = new MemoTable();
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/BasicProgramBlock.java b/src/main/java/org/apache/sysds/runtime/controlprogram/BasicProgramBlock.java
index 4590f0e..3b18b8b 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/BasicProgramBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/BasicProgramBlock.java
@@ -126,7 +126,7 @@ public class BasicProgramBlock extends ProgramBlock
//statement-block-level, lineage-based caching
if (_sb != null && liInputs != null)
- LineageCache.putValue(_sb.getOutputsofSB(), liInputs, _sb.getName(),
- ec, System.nanoTime()-t0);
+ LineageCache.putValue(_sb.getOutputsofSB(),
+ liInputs, _sb.getName(), ec, System.nanoTime()-t0);
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/ForProgramBlock.java b/src/main/java/org/apache/sysds/runtime/controlprogram/ForProgramBlock.java
index 297d953..d27d4c6 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/ForProgramBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/ForProgramBlock.java
@@ -170,6 +170,9 @@ public class ForProgramBlock extends ProgramBlock
catch (Exception e) {
throw new DMLRuntimeException(printBlockErrorLocation() + "Error evaluating for program block", e);
}
+
+ //execute exit instructions
+ executeExitInstructions(_exitInstruction, "for", ec);
}
protected IntObject executePredicateInstructions( int pos, ArrayList<Instruction> instructions, ExecutionContext ec )
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/IfProgramBlock.java b/src/main/java/org/apache/sysds/runtime/controlprogram/IfProgramBlock.java
index c2737f4..9d0d58e 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/IfProgramBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/IfProgramBlock.java
@@ -34,7 +34,6 @@ import org.apache.sysds.runtime.instructions.cp.BooleanObject;
public class IfProgramBlock extends ProgramBlock
{
private ArrayList<Instruction> _predicate;
- private ArrayList <Instruction> _exitInstructions ;
private ArrayList<ProgramBlock> _childBlocksIfBody;
private ArrayList<ProgramBlock> _childBlocksElseBody;
@@ -43,7 +42,6 @@ public class IfProgramBlock extends ProgramBlock
_childBlocksIfBody = new ArrayList<>();
_childBlocksElseBody = new ArrayList<>();
_predicate = predicate;
- _exitInstructions = new ArrayList<>();
}
public ArrayList<ProgramBlock> getChildBlocksIfBody() {
@@ -115,8 +113,7 @@ public class IfProgramBlock extends ProgramBlock
}
else
{
- try
- {
+ try {
for (int i=0 ; i < _childBlocksElseBody.size() ; i++) {
_childBlocksElseBody.get(i).execute(ec);
}
@@ -124,32 +121,20 @@ public class IfProgramBlock extends ProgramBlock
catch(DMLScriptException e) {
throw e;
}
- catch(Exception e)
- {
+ catch(Exception e) {
throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error evaluating else statement body ", e);
- }
+ }
}
//execute exit instructions
- try {
- executeInstructions(_exitInstructions, ec);
- }
- catch(DMLScriptException e) {
- throw e;
- }
- catch (Exception e){
-
- throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error evaluating if exit instructions ", e);
- }
+ executeExitInstructions(_exitInstruction, "if", ec);
}
private BooleanObject executePredicate(ExecutionContext ec)
{
BooleanObject result = null;
- try
- {
- if( _sb != null )
- {
+ try {
+ if( _sb != null ) {
IfStatementBlock isb = (IfStatementBlock)_sb;
result = (BooleanObject) executePredicate(_predicate, isb.getPredicateHops(),
isb.requiresPredicateRecompilation(), ValueType.BOOLEAN, ec);
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java
index 7e3db23..5fe1860 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java
@@ -658,6 +658,9 @@ public class ParForProgramBlock extends ForProgramBlock
ec.setVariable(var, mo);
}
+ //execute exit instructions
+ executeExitInstructions(_exitInstruction, "parfor", ec);
+
///////
//end PARALLEL EXECUTION of (PAR)FOR body
///////
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/ProgramBlock.java b/src/main/java/org/apache/sysds/runtime/controlprogram/ProgramBlock.java
index 8859d39..8bb7bfc 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/ProgramBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/ProgramBlock.java
@@ -58,7 +58,11 @@ public abstract class ProgramBlock implements ParseInfo
private static final boolean CHECK_MATRIX_SPARSITY = false;
protected Program _prog; // pointer to Program this ProgramBlock is part of
-
+
+ //optional exit instructions, necessary for proper cleanup in while/for/if
+ //in case a variable needs to be removed (via rmvar) after the control block
+ protected Instruction _exitInstruction = null; //single packed rmvar
+
//additional attributes for recompile
protected StatementBlock _sb = null;
protected long _tid = 0; //by default _t0
@@ -103,6 +107,13 @@ public abstract class ProgramBlock implements ParseInfo
return _tid;
}
+ public void setExitInstruction(Instruction rmVar) {
+ _exitInstruction = rmVar;
+ }
+
+ public Instruction getExitInstruction() {
+ return _exitInstruction;
+ }
/**
* Get the list of child program blocks if nested;
@@ -171,6 +182,20 @@ public abstract class ProgramBlock implements ParseInfo
return executePredicateInstructions(tmp, retType, ec);
}
+ protected void executeExitInstructions(Instruction inst, String ctx, ExecutionContext ec) {
+ try {
+ if( _exitInstruction != null )
+ executeSingleInstruction(_exitInstruction, ec);
+ }
+ catch(DMLScriptException e) {
+ throw e;
+ }
+ catch (Exception e) {
+ throw new DMLRuntimeException(printBlockErrorLocation()
+ + "Error evaluating "+ctx+" exit instructions ", e);
+ }
+ }
+
protected void executeInstructions(ArrayList<Instruction> inst, ExecutionContext ec) {
for (int i = 0; i < inst.size(); i++) {
//indexed access required due to dynamic add
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/WhileProgramBlock.java b/src/main/java/org/apache/sysds/runtime/controlprogram/WhileProgramBlock.java
index 18c1033..e8853be 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/WhileProgramBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/WhileProgramBlock.java
@@ -115,6 +115,9 @@ public class WhileProgramBlock extends ProgramBlock
catch (Exception e) {
throw new DMLRuntimeException(printBlockErrorLocation() + "Error evaluating while program block", e);
}
+
+ //execute exit instructions
+ executeExitInstructions(_exitInstruction, "while", ec);
}
public void setChildBlocks(ArrayList<ProgramBlock> childs) {
diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/LazyWriteBuffer.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/LazyWriteBuffer.java
index 23cf3fc..cf6bf0e 100644
--- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/LazyWriteBuffer.java
+++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/LazyWriteBuffer.java
@@ -211,6 +211,10 @@ public class LazyWriteBuffer
return _limit - _size; }
}
+ public static int getQueueSize() {
+ return _mQueue.size();
+ }
+
public static long getCacheBlockSize(CacheBlock cb) {
return cb.isShallowSerialize() ?
cb.getInMemorySize() : cb.getExactSerializedSize();
@@ -239,7 +243,8 @@ public class LazyWriteBuffer
String fname = entry.getKey();
ByteBuffer bbuff = entry.getValue();
System.out.println("\tWB: buffer element ("+count+"): "
- +fname+", "+bbuff.getSize()+", "+bbuff.isShallow());
+ +fname+", "+(bbuff.isShallow()?bbuff._cdata.getClass().getSimpleName():"?")
+ +", "+bbuff.getSize()+", "+bbuff.isShallow());
count--;
}
}
diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java
index 456b999..65b856d 100644
--- a/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java
+++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java
@@ -514,6 +514,10 @@ public class VariableCPInstruction extends CPInstruction implements LineageTrace
switch ( opcode )
{
case CreateVariable:
+ //PRE: for robustness we cleanup existing variables, because a setVariable
+ //would cause a buffer pool memory leak as these objects would never be removed
+ if(ec.containsVariable(getInput1()))
+ processRemoveVariableInstruction(ec, getInput1().getName());
if ( getInput1().getDataType() == DataType.MATRIX ) {
//create new variable for symbol table and cache
diff --git a/src/main/java/org/apache/sysds/runtime/util/ProgramConverter.java b/src/main/java/org/apache/sysds/runtime/util/ProgramConverter.java
index db3924d..ce3cc74 100644
--- a/src/main/java/org/apache/sysds/runtime/util/ProgramConverter.java
+++ b/src/main/java/org/apache/sysds/runtime/util/ProgramConverter.java
@@ -275,6 +275,7 @@ public class ProgramConverter
tmpPB.setStatementBlock( createWhileStatementBlockCopy((WhileStatementBlock) wpb.getStatementBlock(), pid, plain, forceDeepCopy) );
tmpPB.setThreadID(pid);
tmpPB.setChildBlocks(rcreateDeepCopyProgramBlocks(wpb.getChildBlocks(), pid, IDPrefix, fnStack, fnCreated, plain, forceDeepCopy));
+ tmpPB.setExitInstruction(wpb.getExitInstruction());
return tmpPB;
}
@@ -285,6 +286,7 @@ public class ProgramConverter
tmpPB.setThreadID(pid);
tmpPB.setChildBlocksIfBody(rcreateDeepCopyProgramBlocks(ipb.getChildBlocksIfBody(), pid, IDPrefix, fnStack, fnCreated, plain, forceDeepCopy));
tmpPB.setChildBlocksElseBody(rcreateDeepCopyProgramBlocks(ipb.getChildBlocksElseBody(), pid, IDPrefix, fnStack, fnCreated, plain, forceDeepCopy));
+ tmpPB.setExitInstruction(ipb.getExitInstruction());
return tmpPB;
}
@@ -296,6 +298,7 @@ public class ProgramConverter
tmpPB.setToInstructions( createDeepCopyInstructionSet(fpb.getToInstructions(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true) );
tmpPB.setIncrementInstructions( createDeepCopyInstructionSet(fpb.getIncrementInstructions(), pid, IDPrefix, prog, fnStack, fnCreated, plain, true) );
tmpPB.setChildBlocks( rcreateDeepCopyProgramBlocks(fpb.getChildBlocks(), pid, IDPrefix, fnStack, fnCreated, plain, forceDeepCopy) );
+ tmpPB.setExitInstruction(fpb.getExitInstruction());
return tmpPB;
}
@@ -305,6 +308,7 @@ public class ProgramConverter
tmpPB.setToInstructions( fpb.getToInstructions() );
tmpPB.setIncrementInstructions( fpb.getIncrementInstructions() );
tmpPB.setChildBlocks( fpb.getChildBlocks() );
+ tmpPB.setExitInstruction(fpb.getExitInstruction());
return tmpPB;
}
@@ -332,6 +336,7 @@ public class ProgramConverter
tmpPB.setChildBlocks( rcreateDeepCopyProgramBlocks(pfpb.getChildBlocks(), pid, IDPrefix, fnStack, fnCreated, plain, forceDeepCopy) );
else
tmpPB.setChildBlocks( pfpb.getChildBlocks() );
+ tmpPB.setExitInstruction(pfpb.getExitInstruction());
return tmpPB;
}
diff --git a/src/main/java/org/apache/sysds/utils/Explain.java b/src/main/java/org/apache/sysds/utils/Explain.java
index 29dd940..8d6124e 100644
--- a/src/main/java/org/apache/sysds/utils/Explain.java
+++ b/src/main/java/org/apache/sysds/utils/Explain.java
@@ -287,18 +287,15 @@ public class Explain
return sb.toString();
}
- public static String explain( ProgramBlock pb )
- {
+ public static String explain( ProgramBlock pb ) {
return explainProgramBlock(pb, 0);
}
- public static String explain( ArrayList<Instruction> inst )
- {
+ public static String explain( ArrayList<Instruction> inst ) {
return explainInstructions(inst, 0);
}
- public static String explain( ArrayList<Instruction> inst, int level )
- {
+ public static String explain( ArrayList<Instruction> inst, int level ) {
return explainInstructions(inst, level);
}
@@ -693,6 +690,8 @@ public class Explain
sb.append(explainInstructions(wpb.getPredicate(), level+1));
for( ProgramBlock pbc : wpb.getChildBlocks() )
sb.append( explainProgramBlock( pbc, level+1) );
+ if( wpb.getExitInstruction() != null )
+ sb.append(explainInstructions(wpb.getExitInstruction(), level+1));
}
else if (pb instanceof IfProgramBlock) {
IfProgramBlock ipb = (IfProgramBlock) pb;
@@ -707,6 +706,8 @@ public class Explain
for( ProgramBlock pbc : ipb.getChildBlocksElseBody() )
sb.append( explainProgramBlock( pbc, level+1) );
}
+ if( ipb.getExitInstruction() != null )
+ sb.append(explainInstructions(ipb.getExitInstruction(), level+1));
}
else if (pb instanceof ForProgramBlock) { //incl parfor
ForProgramBlock fpb = (ForProgramBlock) pb;
@@ -725,7 +726,8 @@ public class Explain
sb.append(explainInstructions(fpb.getIncrementInstructions(), level+1));
for( ProgramBlock pbc : fpb.getChildBlocks() )
sb.append( explainProgramBlock( pbc, level+1) );
-
+ if( fpb.getExitInstruction() != null )
+ sb.append(explainInstructions(fpb.getExitInstruction(), level+1));
}
else if( pb instanceof BasicProgramBlock ) {
BasicProgramBlock bpb = (BasicProgramBlock) pb;
@@ -740,24 +742,27 @@ public class Explain
return sb.toString();
}
- private static String explainInstructions( ArrayList<Instruction> instSet, int level )
- {
+ private static String explainInstructions( ArrayList<Instruction> instSet, int level ) {
StringBuilder sb = new StringBuilder();
String offsetInst = createOffset(level);
-
- for( Instruction inst : instSet )
- {
+ for( Instruction inst : instSet ) {
String tmp = explainGenericInstruction(inst, level);
-
sb.append( offsetInst );
sb.append( tmp );
-
sb.append( '\n' );
}
return sb.toString();
}
+ private static String explainInstructions( Instruction inst, int level ) {
+ StringBuilder sb = new StringBuilder();
+ sb.append( createOffset(level) );
+ sb.append( explainGenericInstruction(inst, level) );
+ sb.append( '\n' );
+ return sb.toString();
+ }
+
private static String explainGenericInstruction( Instruction inst, int level )
{
String tmp = null;
diff --git a/src/test/java/org/apache/sysds/test/functions/caching/BufferpoolLeakTest.java b/src/test/java/org/apache/sysds/test/functions/caching/BufferpoolLeakTest.java
new file mode 100644
index 0000000..e652c5c
--- /dev/null
+++ b/src/test/java/org/apache/sysds/test/functions/caching/BufferpoolLeakTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.functions.caching;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.sysds.runtime.controlprogram.caching.CacheStatistics;
+import org.apache.sysds.test.AutomatedTestBase;
+import org.apache.sysds.test.TestConfiguration;
+
+public class BufferpoolLeakTest extends AutomatedTestBase
+{
+ private final static String TEST_NAME = "BufferpoolLeak";
+ private final static String TEST_DIR = "functions/caching/";
+ private final static String TEST_CLASS_DIR = TEST_DIR + BufferpoolLeakTest.class.getSimpleName() + "/";
+
+ @Override
+ public void setUp() {
+ addTestConfiguration(TEST_NAME,
+ new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] { "V" }) );
+ }
+
+ @Test
+ public void testLeak1() {
+ runTestBufferpoolLeak(10000, 15);
+ }
+
+ private void runTestBufferpoolLeak(int rows, int cols) {
+ TestConfiguration config = getTestConfiguration(TEST_NAME);
+ config.addVariable("rows", rows);
+ config.addVariable("cols", cols);
+ loadTestConfiguration(config);
+
+ String HOME = SCRIPT_DIR + TEST_DIR;
+ fullDMLScriptName = HOME + TEST_NAME + ".dml";
+ programArgs = new String[]{"-stats", "-args",
+ Integer.toString(rows), Integer.toString(cols)};
+
+ //run test and check no evictions
+ runTest(true, false, null, -1);
+ Assert.assertEquals(0, CacheStatistics.getFSWrites());
+ }
+}
diff --git a/src/test/scripts/functions/caching/BufferpoolLeak.dml b/src/test/scripts/functions/caching/BufferpoolLeak.dml
new file mode 100644
index 0000000..b7c2ab4
--- /dev/null
+++ b/src/test/scripts/functions/caching/BufferpoolLeak.dml
@@ -0,0 +1,28 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+X = rand(rows=$1, cols=$2, min=1, max=10);
+for(i in 1:500) {
+ #print("executed iteration "+i)
+ [m1,m2] = mice(as.frame(X), matrix(0,1,ncol(X)),3,3)
+}
+if( ncol(X) > $2 )
+ print(toString(m1));