You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/11/16 08:10:41 UTC
[1/3] systemml git commit: [SYSTEMML-2015] Performance sparse diagV2M
and removeEmpty (CSR output)
Repository: systemml
Updated Branches:
refs/heads/master 3b4656f5e -> 7dc61c05b
[SYSTEMML-2015] Performance sparse diagV2M and removeEmpty (CSR output)
This patch makes major performance improvements to diag V2M (ultra
sparse output) and sparse removeEmpty, which follows the diag for
creating permutation matrices. Both operations are now able to output in
CSR format, which is more memory-efficient for such ultra-sparse
matrices because it avoids sparse row overheads (object and array
headers). In detail, (1) diagV2M directly constructs the row points,
column index and value arrays, and (2) removeEmpty for sparse CSR inputs
uses a shallow copy of index and value arrays and only reconstructs a
new row pointer array.
Over 100 iterations of removeEmpty(diag(V)), where V is of size 10M with
sparsity 0.1, this patch improved performance as follows:
rdiag: 14.8s -> 4.6s
rmempty: 11.7s -> 3.9s
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/9e599cd1
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/9e599cd1
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/9e599cd1
Branch: refs/heads/master
Commit: 9e599cd16bef4ff9b7a56dd4634756c7fefecc6d
Parents: 3b4656f
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed Nov 15 21:55:34 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Nov 16 00:11:51 2017 -0800
----------------------------------------------------------------------
.../runtime/matrix/data/LibMatrixReorg.java | 98 +++++++++++++++-----
1 file changed, 76 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/9e599cd1/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
index b86dc96..80b3285 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java
@@ -71,8 +71,8 @@ public class LibMatrixReorg
//allow reuse of temporary blocks for certain operations
public static final boolean ALLOW_BLOCK_REUSE = false;
- //use csr instead of mcsr sparse block for rexpand columns
- public static final boolean REXPAND_COLS_IN_CSR = true;
+ //use csr instead of mcsr sparse block for rexpand columns / diag v2m
+ public static final boolean SPARSE_OUTPUTS_IN_CSR = true;
private enum ReorgType {
TRANSPOSE,
@@ -108,15 +108,15 @@ public class LibMatrixReorg
return transpose(in, out, op.getNumThreads());
else
return transpose(in, out);
- case REV:
+ case REV:
return rev(in, out);
- case DIAG:
+ case DIAG:
return diag(in, out);
- case SORT:
+ case SORT:
SortIndex ix = (SortIndex) op.fn;
return sort(in, out, ix.getCols(), ix.getDecreasing(), ix.getIndexReturn());
- default:
+ default:
throw new DMLRuntimeException("Unsupported reorg operator: "+op.fn);
}
}
@@ -1029,7 +1029,7 @@ public class LibMatrixReorg
}
/**
- * Generic implementation diagV2M (non-performance critical)
+ * Generic implementation diagV2M
* (in most-likely DENSE, out most likely SPARSE)
*
* @param in input matrix
@@ -1037,15 +1037,47 @@ public class LibMatrixReorg
*/
private static void diagV2M( MatrixBlock in, MatrixBlock out )
{
- int rlen = in.rlen;
+ final int rlen = in.rlen;
//CASE column vector
- for( int i=0; i<rlen; i++ )
- {
- double val = in.quickGetValue(i, 0);
- if( val != 0 )
- out.appendValue(i, i, val);
+ if( out.sparse ) { //SPARSE
+ if( SPARSE_OUTPUTS_IN_CSR ) {
+ int[] rptr = new int[in.rlen+1];
+ int[] cix = new int[(int)in.nonZeros];
+ double[] vals = new double[(int)in.nonZeros];
+ for( int i=0, pos=0; i<rlen; i++ ) {
+ double val = in.quickGetValue(i, 0);
+ if( val != 0 ) {
+ cix[pos] = i;
+ vals[pos] = val;
+ pos++;
+ }
+ rptr[i+1]=pos;
+ }
+ out.sparseBlock = new SparseBlockCSR(
+ rptr, cix, vals, (int)in.nonZeros);
+ }
+ else {
+ out.allocateBlock();
+ SparseBlock sblock = out.sparseBlock;
+ for(int i=0; i<rlen; i++) {
+ double val = in.quickGetValue(i, 0);
+ if( val != 0 ) {
+ sblock.allocate(i, 1);
+ sblock.append(i, i, val);
+ }
+ }
+ }
}
+ else { //DENSE
+ for( int i=0; i<rlen; i++ ) {
+ double val = in.quickGetValue(i, 0);
+ if( val != 0 )
+ out.appendValue(i, i, val);
+ }
+ }
+
+ out.setNonZeros(in.nonZeros);
}
/**
@@ -1647,19 +1679,41 @@ public class LibMatrixReorg
boolean[] flags = null;
int rlen2 = 0;
+ //Step 0: special case handling
+ if( SHALLOW_COPY_REORG && SPARSE_OUTPUTS_IN_CSR
+ && in.sparse && !in.isEmptyBlock(false)
+ && select==null && in.sparseBlock instanceof SparseBlockCSR
+ && in.nonZeros < Integer.MAX_VALUE )
+ {
+ //create the output in csr format with a shallow copy of arrays for column
+ //indexes and values (heuristic: shallow copy better than copy to dense)
+ SparseBlockCSR sblock = (SparseBlockCSR) in.sparseBlock;
+ int lrlen = 0;
+ for( int i=0; i<m; i++ )
+ lrlen += sblock.isEmpty(i) ? 0 : 1;
+ int[] rptr = new int[lrlen+1];
+ for( int i=0, j=0, pos=0; i<m; i++ )
+ if( !sblock.isEmpty(i) ) {
+ pos += sblock.size(i);
+ rptr[++j] = pos;
+ }
+ ret.reset(lrlen, in.clen, true);
+ ret.sparseBlock = new SparseBlockCSR(rptr,
+ sblock.indexes(), sblock.values(), (int)in.nonZeros);
+ ret.nonZeros = in.nonZeros;
+ return ret;
+ }
+
+ //Step 1: scan block and determine non-empty rows
if(select == null)
{
flags = new boolean[ m ]; //false
- //Step 1: scan block and determine non-empty rows
-
- if( in.sparse ) //SPARSE
- {
+ if( in.sparse ) { //SPARSE
SparseBlock a = in.sparseBlock;
for ( int i=0; i < m; i++ )
rlen2 += (flags[i] = !a.isEmpty(i)) ? 1 : 0;
}
- else //DENSE
- {
+ else { //DENSE
double[] a = in.denseBlock;
for(int i=0, aix=0; i<m; i++, aix+=n)
for(int j=0; j<n; j++)
@@ -1671,7 +1725,7 @@ public class LibMatrixReorg
}
}
}
- else {
+ else {
flags = DataConverter.convertToBooleanVector(select);
rlen2 = (int)select.getNonZeros();
}
@@ -1918,7 +1972,7 @@ public class LibMatrixReorg
//execute rexpand columns
long rnnz = 0; //real nnz (due to cutoff max)
if( k <= 1 || in.getNumRows() <= PAR_NUMCELL_THRESHOLD
- || (sp && REXPAND_COLS_IN_CSR) ) {
+ || (sp && SPARSE_OUTPUTS_IN_CSR) ) {
rnnz = rexpandColumns(in, ret, max, cast, ignore, 0, rlen);
}
else {
@@ -1951,7 +2005,7 @@ public class LibMatrixReorg
//initialize auxiliary data structures
int lnnz = 0;
int[] cix = null;
- if( ret.sparse && REXPAND_COLS_IN_CSR ) {
+ if( ret.sparse && SPARSE_OUTPUTS_IN_CSR ) {
cix = new int[in.rlen];
Arrays.fill(cix, -1);
}
[2/3] systemml git commit: [SYSTEMML-2016] Performance frame cbind
and vector frame casts (shallow)
Posted by mb...@apache.org.
[SYSTEMML-2016] Performance frame cbind and vector frame casts (shallow)
This patch makes major performance improvements to the frame primitives
cbind and matrix (special case of vector) to frame casts, which both can
be done via shallow copies. This is safe due to copy-on-write semantics
of operations.
On a scenario of 20 iterations of cbind(as.frame(X), cbind(as.frame(Y),
as.frame(Z))) over 10M x 1 dense inputs, this patch improved end-to-end
runtime from 26.1s (15s GC) to 3.9s (0.2s GC). Note that 90% of
remaining execution time is spent in buffer pool evictions, which are
unnecessary and will be addressed in a subsequent patch.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/1e0a4151
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/1e0a4151
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/1e0a4151
Branch: refs/heads/master
Commit: 1e0a4151d21d0821668dd7d0de34ac6b75c33358
Parents: 9e599cd
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed Nov 15 22:38:24 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Nov 16 00:11:52 2017 -0800
----------------------------------------------------------------------
.../RewriteAlgebraicSimplificationStatic.java | 2 +-
.../apache/sysml/runtime/matrix/data/FrameBlock.java | 8 +++-----
.../org/apache/sysml/runtime/util/DataConverter.java | 15 +++++++++++----
3 files changed, 15 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/1e0a4151/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
index d71c4e0..cc2fe88 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationStatic.java
@@ -522,7 +522,7 @@ public class RewriteAlgebraicSimplificationStatic extends HopRewriteRule
private static Hop foldMultipleAppendOperations(Hop hi)
throws HopsException
{
- if( hi.getDataType().isMatrix() //no string appends
+ if( hi.getDataType().isMatrix() //no string appends or frames
&& (HopRewriteUtils.isBinary(hi, OpOp2.CBIND, OpOp2.RBIND)
|| HopRewriteUtils.isNary(hi, OpOpN.CBIND, OpOpN.RBIND))
&& !OptimizerUtils.isHadoopExecutionMode() )
http://git-wip-us.apache.org/repos/asf/systemml/blob/1e0a4151/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
index a56fd6a..e83b362 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
@@ -471,9 +471,9 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
for( int j=0; j<ncol; j++ )
tmpData[j] = new DoubleArray(cols[j]);
_colnames = empty ? null : (String[]) ArrayUtils.addAll(getColumnNames(),
- createColNames(getNumColumns(), ncol)); //before schema modification
+ createColNames(getNumColumns(), ncol)); //before schema modification
_schema = empty ? tmpSchema : (ValueType[]) ArrayUtils.addAll(_schema, tmpSchema);
- _coldata = empty ? tmpData : (Array[]) ArrayUtils.addAll(_coldata, tmpData);
+ _coldata = empty ? tmpData : (Array[]) ArrayUtils.addAll(_coldata, tmpData);
_numRows = cols[0].length;
}
@@ -988,10 +988,8 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable
ret._colnames = (String[]) ArrayUtils.addAll(getColumnNames(), that.getColumnNames());
ret._colmeta = (ColumnMetadata[]) ArrayUtils.addAll(_colmeta, that._colmeta);
- //concatenate column data (w/ deep copy to prevent side effects)
+ //concatenate column data (w/ shallow copy which is safe due to copy on write semantics)
ret._coldata = (Array[]) ArrayUtils.addAll(_coldata, that._coldata);
- for( int i=0; i<ret.getNumColumns(); i++ )
- ret._coldata[i] = ret._coldata[i].clone();
}
else //ROW APPEND
{
http://git-wip-us.apache.org/repos/asf/systemml/blob/1e0a4151/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
index af69e81..bfc07ba 100644
--- a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java
@@ -654,7 +654,7 @@ public class DataConverter
if( mb.isInSparseFormat() ) //SPARSE
{
- SparseBlock sblock = mb.getSparseBlock();
+ SparseBlock sblock = mb.getSparseBlock();
for( int i=0; i<mb.getNumRows(); i++ ) {
Arrays.fill(row, null); //reset
if( sblock != null && !sblock.isEmpty(i) ) {
@@ -664,7 +664,7 @@ public class DataConverter
double[] aval = sblock.values(i);
for( int j=apos; j<apos+alen; j++ ) {
row[aix[j]] = UtilFunctions.doubleToObject(
- schema[aix[j]], aval[j]);
+ schema[aix[j]], aval[j]);
}
}
frame.appendRow(row);
@@ -673,8 +673,15 @@ public class DataConverter
else //DENSE
{
int dFreq = UtilFunctions.frequency(schema, ValueType.DOUBLE);
-
- if( dFreq == schema.length ) {
+
+ if( schema.length==1 && dFreq==1 && mb.isAllocated() ) {
+ // special case double schema and single columns which
+ // allows for a shallow copy since the physical representation
+ // of row-major matrix and column-major frame match exactly
+ frame.reset();
+ frame.appendColumns(new double[][]{mb.getDenseBlock()});
+ }
+ else if( dFreq == schema.length ) {
// special case double schema (without cell-object creation,
// col pre-allocation, and cache-friendly row-column copy)
int m = mb.getNumRows();
[3/3] systemml git commit: [SYSTEMML-2017] Fix missing bufferpool
cleanup of frame intermediates
Posted by mb...@apache.org.
[SYSTEMML-2017] Fix missing bufferpool cleanup of frame intermediates
So far any rmvar, cpvar, mvvar instructions only trigger the cleanup of
buffer pool in-memory objects, evicted files and hdfs files for matrices
but not frames. This creates a leak of objects and files that causes
severe performance issues if frame intermediates are creates in
iterative algorithms. This patch now generalizes all related code paths
to CacheableData which is the super class of MatrixObjects and
FrameObjects.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/7dc61c05
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/7dc61c05
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/7dc61c05
Branch: refs/heads/master
Commit: 7dc61c05b609aec4afc5a408cc8fe5463bdeb7aa
Parents: 1e0a415
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Nov 16 00:04:10 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Nov 16 00:11:52 2017 -0800
----------------------------------------------------------------------
src/main/java/org/apache/sysml/parser/Expression.java | 3 +++
.../runtime/controlprogram/ParForProgramBlock.java | 4 ++--
.../controlprogram/context/ExecutionContext.java | 2 +-
.../controlprogram/context/SparkExecutionContext.java | 2 +-
.../controlprogram/parfor/opt/OptimizerRuleBased.java | 2 +-
.../instructions/cp/FunctionCallCPInstruction.java | 10 +++++-----
.../instructions/cp/VariableCPInstruction.java | 14 +++++++-------
7 files changed, 20 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/7dc61c05/src/main/java/org/apache/sysml/parser/Expression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/Expression.java b/src/main/java/org/apache/sysml/parser/Expression.java
index 1d7303e..fa68870 100644
--- a/src/main/java/org/apache/sysml/parser/Expression.java
+++ b/src/main/java/org/apache/sysml/parser/Expression.java
@@ -173,6 +173,9 @@ public abstract class Expression implements ParseInfo
public boolean isMatrix() {
return (this == MATRIX);
}
+ public boolean isFrame() {
+ return (this == FRAME);
+ }
public boolean isScalar() {
return (this == SCALAR);
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/7dc61c05/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
index 4775494..d534fe0 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
@@ -1239,7 +1239,7 @@ public class ParForProgramBlock extends ForProgramBlock
for( MatrixObject tmp : in ) {
//check for empty inputs (no iterations executed)
if( tmp != null && tmp != out )
- ec.cleanupMatrixObject(tmp);
+ ec.cleanupCacheableData(tmp);
}
}
@@ -1705,7 +1705,7 @@ public class ParForProgramBlock extends ForProgramBlock
//cleanup existing var
Data exdata = ec.removeVariable(var);
if( exdata != null && exdata != outNew && exdata instanceof MatrixObject )
- ec.cleanupMatrixObject((MatrixObject)exdata);
+ ec.cleanupCacheableData((MatrixObject)exdata);
//cleanup of intermediate result variables
cleanWorkerResultVariables( ec, out, in );
http://git-wip-us.apache.org/repos/asf/systemml/blob/7dc61c05/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
index 67e91b0..af8c8b9 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
@@ -608,7 +608,7 @@ public class ExecutionContext {
return ret;
}
- public void cleanupMatrixObject(MatrixObject mo)
+ public void cleanupCacheableData(CacheableData<?> mo)
throws DMLRuntimeException
{
//early abort w/o scan of symbol table if no cleanup required
http://git-wip-us.apache.org/repos/asf/systemml/blob/7dc61c05/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
index ff47b3a..39e52f5 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
@@ -1087,7 +1087,7 @@ public class SparkExecutionContext extends ExecutionContext
}
@Override
- public void cleanupMatrixObject( MatrixObject mo )
+ public void cleanupCacheableData( CacheableData<?> mo )
throws DMLRuntimeException
{
//NOTE: this method overwrites the default behavior of cleanupMatrixObject
http://git-wip-us.apache.org/repos/asf/systemml/blob/7dc61c05/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index eca87da..3e99128 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -2078,7 +2078,7 @@ public class OptimizerRuleBased extends Optimizer
{
//replace existing matrix object with empty matrix
MatrixObject mo = (MatrixObject)dat;
- ec.cleanupMatrixObject(mo);
+ ec.cleanupCacheableData(mo);
ec.setMatrixOutput(rvar, new MatrixBlock((int)mo.getNumRows(), (int)mo.getNumColumns(),false), null);
//keep track of cleaned result variables
http://git-wip-us.apache.org/repos/asf/systemml/blob/7dc61c05/src/main/java/org/apache/sysml/runtime/instructions/cp/FunctionCallCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/FunctionCallCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/FunctionCallCPInstruction.java
index b36365c..953c365 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/FunctionCallCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/FunctionCallCPInstruction.java
@@ -32,7 +32,7 @@ import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.DMLScriptException;
import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
-import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
import org.apache.sysml.runtime.instructions.Instruction;
@@ -180,8 +180,8 @@ public class FunctionCallCPInstruction extends CPInstruction {
if( expectRetVars.contains(var.getKey()) )
continue;
//cleanup unexpected return values to avoid leaks
- if( var.getValue() instanceof MatrixObject )
- fn_ec.cleanupMatrixObject((MatrixObject)var.getValue());
+ if( var.getValue() instanceof CacheableData )
+ fn_ec.cleanupCacheableData((CacheableData<?>)var.getValue());
}
// Unpin the pinned variables
@@ -196,8 +196,8 @@ public class FunctionCallCPInstruction extends CPInstruction {
//cleanup existing data bound to output variable name
Data exdata = ec.removeVariable(boundVarName);
- if ( exdata != null && exdata instanceof MatrixObject && exdata != boundValue ) {
- ec.cleanupMatrixObject( (MatrixObject)exdata );
+ if ( exdata != null && exdata instanceof CacheableData && exdata != boundValue ) {
+ ec.cleanupCacheableData( (CacheableData<?>)exdata );
}
//add/replace data in symbol table
http://git-wip-us.apache.org/repos/asf/systemml/blob/7dc61c05/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
index 92750f7..9e81b50 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java
@@ -698,13 +698,13 @@ public class VariableCPInstruction extends CPInstruction {
+ "for variable name:" + getInput1().getName() + ", while processing instruction ");
}
- if( getInput2().getDataType().isMatrix() ) {
+ if( getInput2().getDataType().isMatrix() || getInput2().getDataType().isFrame() ) {
// remove existing variable bound to target name
Data tgt = ec.removeVariable(getInput2().getName());
//cleanup matrix data on fs/hdfs (if necessary)
- if ( tgt != null && tgt instanceof MatrixObject ) {
- ec.cleanupMatrixObject((MatrixObject) tgt);
+ if ( tgt != null && tgt instanceof CacheableData ) {
+ ec.cleanupCacheableData((CacheableData<?>) tgt);
}
}
@@ -754,8 +754,8 @@ public class VariableCPInstruction extends CPInstruction {
Data input2_data = ec.removeVariable(getInput2().getName());
//cleanup matrix data on fs/hdfs (if necessary)
- if ( input2_data != null && input2_data instanceof MatrixObject ) {
- ec.cleanupMatrixObject((MatrixObject) input2_data);
+ if ( input2_data != null && input2_data instanceof CacheableData ) {
+ ec.cleanupCacheableData((CacheableData<?>) input2_data);
}
// do the actual copy!
@@ -820,8 +820,8 @@ public class VariableCPInstruction extends CPInstruction {
throw new DMLRuntimeException("Unexpected error: could not find a data object for variable name:" + varname + ", while processing rmvar instruction.");
//cleanup matrix data on fs/hdfs (if necessary)
- if ( input1_data instanceof MatrixObject ) {
- ec.cleanupMatrixObject( (MatrixObject) input1_data );
+ if ( input1_data instanceof CacheableData ) {
+ ec.cleanupCacheableData( (CacheableData<?>) input1_data );
}
}