You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/02/24 20:27:37 UTC
[5/6] incubator-systemml git commit: [SYSTEMML-1302] Remove parfor
perftesttool, cleanup heuristic optimizer
[SYSTEMML-1302] Remove parfor perftesttool, cleanup heuristic optimizer
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/35da413a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/35da413a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/35da413a
Branch: refs/heads/master
Commit: 35da413a6a23245902fae145a5e9a80981154161
Parents: 1fe1a02
Author: Matthias Boehm <mb...@gmail.com>
Authored: Wed Feb 22 22:26:39 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Feb 24 12:27:28 2017 -0800
----------------------------------------------------------------------
.../sysml/parser/ParForStatementBlock.java | 15 +-
.../controlprogram/ParForProgramBlock.java | 8 +-
.../parfor/opt/CostEstimator.java | 23 +-
.../parfor/opt/CostEstimatorHops.java | 3 +-
.../parfor/opt/CostEstimatorRuntime.java | 187 +--
.../controlprogram/parfor/opt/CostFunction.java | 151 --
.../parfor/opt/OptTreeConverter.java | 104 +-
.../parfor/opt/OptTreePlanMappingAbstract.java | 10 +-
.../parfor/opt/OptTreePlanMappingRuntime.java | 3 +-
.../parfor/opt/OptimizationWrapper.java | 27 +-
.../controlprogram/parfor/opt/Optimizer.java | 9 +-
.../parfor/opt/OptimizerConstrained.java | 3 +-
.../parfor/opt/OptimizerHeuristic.java | 19 +-
.../controlprogram/parfor/opt/PerfTestDef.java | 99 --
.../parfor/opt/PerfTestExtFunctCP.java | 94 --
.../parfor/opt/PerfTestMemoryObserver.java | 105 --
.../controlprogram/parfor/opt/PerfTestTool.java | 1411 ------------------
.../parfor/opt/PerfTestToolRegression.dml | 59 -
.../parfor/ParForRulebasedOptimizerTest.java | 149 +-
.../functions/parfor/parfor_optimizer1b.dml | 53 +
.../functions/parfor/parfor_optimizer2b.dml | 277 ++++
.../functions/parfor/parfor_optimizer3b.dml | 52 +
22 files changed, 551 insertions(+), 2310 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/35da413a/src/main/java/org/apache/sysml/parser/ParForStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ParForStatementBlock.java b/src/main/java/org/apache/sysml/parser/ParForStatementBlock.java
index bd92a15..5ca0cbc 100644
--- a/src/main/java/org/apache/sysml/parser/ParForStatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/ParForStatementBlock.java
@@ -241,20 +241,7 @@ public class ParForStatementBlock extends ForStatementBlock
}
else //default case
params.put(key, _paramDefaults.get(key));
- }
-
- //check for disabled parameters values
- if( params.containsKey( OPT_MODE ) )
- {
- String optStr = params.get( OPT_MODE );
- if( optStr.equals(POptMode.HEURISTIC.toString())
- || optStr.equals(POptMode.GREEDY.toString())
- || optStr.equals(POptMode.FULL_DP.toString()) )
- { //always unconditional
- raiseValidateError("Sorry, parfor optimization mode '"+optStr+"' is disabled for external usage.", false);
- }
- }
-
+ }
}
else
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/35da413a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
index f6c90f3..af3a0d1 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
@@ -195,12 +195,10 @@ public class ParForProgramBlock extends ForProgramBlock
//optimizer
public enum POptMode{
- NONE, //no optimization, use defaults and specified parameters
- RULEBASED, //some simple rule-based rewritings (affects only parfor PB) - similar to HEURISTIC but no exec time estimates
+ NONE, //no optimization, use defaults and specified parameters
+ RULEBASED, //rule-based rewritings with memory constraints
CONSTRAINED, //same as rule-based but with given params as constraints
- HEURISTIC, //some simple cost-based rewritings (affects only parfor PB)
- GREEDY, //greedy cost-based optimization algorithm (potentially local optimum, affects all instructions)
- FULL_DP, //full cost-based optimization algorithm (global optimum, affects all instructions)
+ HEURISTIC, //smae as rule-based but with time-based cost estimates
}
// internal parameters
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/35da413a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java
index 6e428d1..bb3ca88 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimator.java
@@ -184,17 +184,12 @@ public abstract class CostEstimator
return val;
}
- protected double getDefaultEstimate(TestMeasure measure)
- {
- double val = -1;
-
- switch( measure )
- {
- case EXEC_TIME: val = DEFAULT_TIME_ESTIMATE; break;
- case MEMORY_USAGE: val = DEFAULT_MEM_ESTIMATE_CP; break;
- }
-
- return val;
+ protected double getDefaultEstimate(TestMeasure measure) {
+ switch( measure ) {
+ case EXEC_TIME: return DEFAULT_TIME_ESTIMATE;
+ case MEMORY_USAGE: return DEFAULT_MEM_ESTIMATE_CP;
+ }
+ return -1;
}
protected double getMaxEstimate( TestMeasure measure, ArrayList<OptNode> nodes, ExecType et )
@@ -202,11 +197,7 @@ public abstract class CostEstimator
{
double max = Double.MIN_VALUE; //smallest positive value
for( OptNode n : nodes )
- {
- double tmp = getEstimate( measure, n, et );
- if( tmp > max )
- max = tmp;
- }
+ max = Math.max(max, getEstimate(measure, n, et));
return max;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/35da413a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorHops.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorHops.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorHops.java
index 02ba9ed..b65d80f 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorHops.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorHops.java
@@ -42,8 +42,7 @@ public class CostEstimatorHops extends CostEstimator
}
- public CostEstimatorHops( OptTreePlanMappingAbstract map )
- {
+ public CostEstimatorHops( OptTreePlanMappingAbstract map ) {
_map = map;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/35da413a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorRuntime.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorRuntime.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorRuntime.java
index cbb8260..442ad48 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorRuntime.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostEstimatorRuntime.java
@@ -20,178 +20,67 @@
package org.apache.sysml.runtime.controlprogram.parfor.opt;
-import org.apache.sysml.lops.Lop;
+import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.hops.cost.CostEstimationWrapper;
import org.apache.sysml.lops.LopProperties.ExecType;
import org.apache.sysml.runtime.DMLRuntimeException;
-import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.TestVariable;
+import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
/**
- * Cost Estimator for runtime programs. It uses a offline created performance profile
- * (see PerfTestTool) in order to estimate execution time, memory consumption etc of
- * instructions and program blocks with regard to given data characteristics (e.g.,
- * dimensionality, data format, sparsity) and program parameters (e.g., degree of parallelism).
- * If no performance profile cost function exists for a given TestVariables, TestMeasures, and
- * instructions combination, default values are used. Furthermore, the cost estimator provides
- * basic functionalities for estimation of cardinality and sparsity of intermediate results.
+ * Cost estimator for runtime programs. Previously this estimator used an offline created
+ * performance profile. Since SystemML 1.0, this estimator uses a time-based cost model
+ * that relies on floating operations and I/O, which does not require explicit profiling.
*
- * TODO: inst names as constants in perftesttool
- * TODO: complexity corrections for sparse matrices
*/
-@SuppressWarnings("deprecation")
public class CostEstimatorRuntime extends CostEstimator
{
+ private final CostEstimatorHops _costMem;
+ private final OptTreePlanMappingAbstract _map;
+ private final ExecutionContext _ec;
- //internal estimation parameters
- public static final boolean COMBINE_ESTIMATION_PATHS = true;
+ public CostEstimatorRuntime(OptTreePlanMappingAbstract map, LocalVariableMap vars ) {
+ _costMem = new CostEstimatorHops(map);
+ _map = map;
+ //construct execution context as wrapper to hand over
+ //deep copied symbol table to cost estimator
+ _ec = ExecutionContextFactory.createContext();
+ _ec.setVariables(vars);
+ }
+
@Override
public double getLeafNodeEstimate( TestMeasure measure, OptNode node )
throws DMLRuntimeException
{
- double val = -1;
-
- String str = node.getInstructionName();//node.getParam(ParamType.OPSTRING);
- OptNodeStatistics stats = node.getStatistics();
- DataFormat df = stats.getDataFormat();
-
- double dim1 = stats.getDim1();
- double dim2 = Math.max(stats.getDim2(), stats.getDim3()); //using max useful if just one side known
- double dim3 = stats.getDim4();
- double sparsity = stats.getSparsity();
- val = getEstimate(measure, str, dim1, dim2, dim3, sparsity, df);
-
- //FIXME just for test until cost functions for MR are trained
- if( node.getExecType() == OptNode.ExecType.MR )
- val = 60000; //1min or 60k
+ //use CostEstimatorHops to get the memory estimate
+ if( measure == TestMeasure.MEMORY_USAGE )
+ return _costMem.getLeafNodeEstimate(measure, node);
- //System.out.println("measure="+measure+", operation="+str+", val="+val);
-
- return val;
+ //redirect to exec-type-specific estimate
+ ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
+ return getLeafNodeEstimate(measure, node, node.isCPOnly() ? ExecType.CP : REMOTE);
}
@Override
public double getLeafNodeEstimate( TestMeasure measure, OptNode node, ExecType et )
throws DMLRuntimeException
{
- //TODO for the moment invariant of et
-
- return getLeafNodeEstimate(measure, node);
- }
-
- public double getEstimate( TestMeasure measure, String instName, double dim1, double dim2, double dim3, double sparsity, DataFormat dataformat )
- throws DMLRuntimeException
- {
- return getEstimate(measure, instName, dim1, dim2, dim3, sparsity, DEFAULT_EST_PARALLELISM, dataformat);
- }
-
- public double getEstimate( TestMeasure measure, String instName, double dim1, double dim2, double dim3, double sparsity, double parallelism, DataFormat dataformat )
- throws DMLRuntimeException
- {
- double ret = -1;
- double datasize = -1;
-
- if( instName.equals("CP"+Lop.OPERAND_DELIMITOR+"ba+*") )
- datasize = (dim1*dim2 + dim2*dim3 + dim1*dim3)/3;
- else
- datasize = dim1*dim2;
-
- //get basic cost functions
- CostFunction df = PerfTestTool.getCostFunction(instName, measure, TestVariable.DATA_SIZE, dataformat);
- CostFunction sf = PerfTestTool.getCostFunction(instName, measure, TestVariable.SPARSITY, dataformat);
-
- if( df == null || sf == null )
- {
- return getDefaultEstimate( measure );
- }
-
- //core merge datasize and sparsity
+ //use CostEstimatorHops to get the memory estimate
+ if( measure == TestMeasure.MEMORY_USAGE )
+ return _costMem.getLeafNodeEstimate(measure, node, et);
- //complexity corrections (inc. offset correction)
- if( !df.isMultiDim() )
- {
-
- ret = aggregate( df, sf,
- datasize, PerfTestTool.DEFAULT_DATASIZE,
- sparsity, PerfTestTool.DEFAULT_SPARSITY );
-
- //System.out.println("before correction = "+ret);
-
- double offset = df.estimate(0);
- double ddim = Math.sqrt(datasize);
- double assumedC = -1;
- double realC = -1;
-
- if( instName.equals("CP"+Lop.OPERAND_DELIMITOR+"ba+*") )
- {
- switch( measure )
- {
- case EXEC_TIME:
- assumedC = 2*ddim * ddim * ddim + ddim * ddim;
- if( dataformat==DataFormat.DENSE )
- realC = 2*dim1 * dim2 * dim3 + dim1 * dim3;
- else if( dataformat==DataFormat.SPARSE )
- realC = 2*dim1 * dim2 * dim3 + dim1 * dim3;
- break;
- case MEMORY_USAGE:
- assumedC = 3*ddim*ddim;
- if( dataformat==DataFormat.DENSE )
- realC = dim1 * dim2 + dim2 * dim3 + dim1 * dim3;
- else if( dataformat==DataFormat.SPARSE )
- realC = dim1 * dim2 + dim2 * dim3 + dim1 * dim3;
- break;
- }
- //actual correction (without offset)
- ret = (ret-offset) * realC/assumedC + offset;
- }
-
- /*NEW COMPLEXITY CORRECTIONS GO HERE*/
- }
- else
- {
- double ddim = Math.sqrt(PerfTestTool.DEFAULT_DATASIZE);
-
- ret = aggregate( df, sf,
- new double[]{dim1,dim2,dim3}, new double[]{ddim,ddim,ddim},
- sparsity, PerfTestTool.DEFAULT_SPARSITY );
-
+ //use static cost estimator based on floating point operations
+ //(currently only called for entire parfor program in order to
+ //decide for LOCAL vs REMOTE parfor execution)
+ double ret = DEFAULT_TIME_ESTIMATE;
+ boolean isCP = (et == ExecType.CP || et == null);
+ if( !node.isLeaf() && isCP ) {
+ ProgramBlock pb = (ProgramBlock)_map.getMappedProg(node.getID())[1];
+ ret = CostEstimationWrapper.getTimeEstimate(pb, _ec, true);
}
-
- return ret;
- }
-
- private static double aggregate( CostFunction f1, CostFunction f2, double x1, double d1, double x2, double d2 )
- {
- double val11 = f1.estimate(x1);
- double val12 = f1.estimate(d1);
- double val21 = f2.estimate(x2);
- double val22 = f2.estimate(d2);
-
- //estimate combined measure
- double ret;
- if( COMBINE_ESTIMATION_PATHS )
- ret = ((val11 * val21 / val22) + (val21 * val11 / val12)) / 2;
- else
- ret = (val11 * val21 / val22);
-
return ret;
}
-
- private static double aggregate( CostFunction f1, CostFunction f2, double[] x1, double[] d1, double x2, double d2 )
- {
- double val11 = f1.estimate(x1);
- double val12 = f1.estimate(d1);
- double val21 = f2.estimate(x2);
- double val22 = f2.estimate(d2);
-
- //estimate combined measure
- double ret;
- if( COMBINE_ESTIMATION_PATHS )
- ret = ((val11 * val21 / val22) + (val21 * val11 / val12)) / 2;
- else
- ret = (val11 * val21 / val22);
-
- return ret;
- }
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/35da413a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostFunction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostFunction.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostFunction.java
deleted file mode 100644
index 4bf2bd7..0000000
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/CostFunction.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.controlprogram.parfor.opt;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-
-/**
- * Generic cost function of the form <code>y = f( X )</code>, where y is a TestMeasure
- * (e.g., execution time, memory consumption) and X is a vector of input TestVariable values.
- *
- * This is used for two different use cases (1) polynomial function, for one input parameter
- * (e.g., y=f(x): y = f_0 + f_1*x^1 + f_2*x^2 + ...) and (2) multidimensional functions
- * (e.g., y=f(x1, x2, ...) with poynomials for all involved input parameters.
- *
- */
-public class CostFunction
-{
-
-
-
- protected static final Log LOG = LogFactory.getLog(CostFunction.class.getName());
-
- public static final boolean PREVENT_NEGATIVE_ESTIMATES = true;
-
- private double[] _params = null;
- private boolean _multiDim = false;
-
- public CostFunction( double[] params, boolean multiDim )
- {
- _params = params;
- _multiDim = multiDim;
- }
-
- public boolean isMultiDim()
- {
- return _multiDim;
- }
-
- public double estimate( double in )
- {
- double costs = 0;
-
- //compute the estimate for arbitrary orders of F
- if( _params != null )
- for( int i = 0; i<_params.length; i++ )
- {
- //test
- double v1 = in;
- double v2 = Math.pow(in, i);
- if( i>1 && Math.abs(Math.sqrt( v2 ) - v1) > 1.0 ) //issue if larger than 1ms or 1byte
- {
- LOG.error("Numerical stability issue: " + v1 + " vs " + v2 );
- continue;
- }
- //end test
-
- costs += _params[i] * Math.pow(in, i);
- }
-
- costs = correctEstimate(costs);
-
- return costs;
- }
-
- public double estimate( double[] in )
- {
- double costs = 0;
- int len = in.length;
-
- if( _params != null )
- {
- costs = _params[0]; //intercept
-
- for( int i=0; i<len; i++ )
- costs += _params[i+1] * in[ i ];
-
- for( int i=0; i<len; i++ )
- costs += _params[len+i+1] * Math.pow(in[ i ],2);
-
- int ix=0;
- for( int j=0; j<len-1; j++ )
- for( int k=j+1; k<len; k++, ix++ )
- costs += _params[1+2*len+ix] * in[j]*in[k];
-
- //double tmp=1;
- //for( int i=0; i<len; i++ )
- // tmp*=in[i];
- //costs += _params[_params.length-1]*tmp;
- }
-
- costs = correctEstimate(costs);
-
- return costs;
- }
-
- public double[] getParams()
- {
- return _params;
- }
-
- @Override
- public String toString()
- {
- StringBuilder sb = new StringBuilder( "f(x) = " );
-
- //compute the estimate for arbitrary orders of F
- if( _params != null )
- for( int i = 0; i<_params.length; i++ )
- {
- if( i > 0 )
- sb.append( " + " );
- sb.append( _params[i] );
- sb.append( " * x^" );
- sb.append( i );
- }
-
- return sb.toString();
- }
-
- private double correctEstimate( double cost )
- {
- double ret = cost;
-
- //check for invalid estimates (due to polynomial functions)
- if( PREVENT_NEGATIVE_ESTIMATES )
- {
- ret = (ret < 0) ? 0 : ret;
- }
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/35da413a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java
index 44d4de2..80badf9 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java
@@ -44,7 +44,6 @@ import org.apache.sysml.parser.ParForStatement;
import org.apache.sysml.parser.ParForStatementBlock;
import org.apache.sysml.parser.StatementBlock;
import org.apache.sysml.parser.WhileStatement;
-import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.WhileStatementBlock;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
@@ -55,25 +54,17 @@ import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
import org.apache.sysml.runtime.controlprogram.Program;
import org.apache.sysml.runtime.controlprogram.ProgramBlock;
import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
-import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
-import org.apache.sysml.runtime.controlprogram.parfor.opt.CostEstimator.DataFormat;
import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.ExecType;
import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.NodeType;
import org.apache.sysml.runtime.controlprogram.parfor.opt.OptNode.ParamType;
import org.apache.sysml.runtime.controlprogram.parfor.opt.Optimizer.PlanInputType;
import org.apache.sysml.runtime.instructions.Instruction;
import org.apache.sysml.runtime.instructions.MRJobInstruction;
-import org.apache.sysml.runtime.instructions.cp.ComputationCPInstruction;
-import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction;
-import org.apache.sysml.runtime.instructions.cp.DataGenCPInstruction;
import org.apache.sysml.runtime.instructions.cpfile.MatrixIndexingCPFileInstruction;
import org.apache.sysml.runtime.instructions.cpfile.ParameterizedBuiltinCPFileInstruction;
import org.apache.sysml.runtime.instructions.spark.SPInstruction;
-import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
-import org.apache.sysml.runtime.matrix.MatrixFormatMetaData;
-import org.apache.sysml.runtime.matrix.data.MatrixBlock;
/**
* Converter for creating an internal plan representation for a given runtime program
@@ -82,7 +73,6 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock;
* NOTE: currently only one abstract and one runtime plan at a time.
* This implies that only one parfor optimization can happen at a time.
*/
-@SuppressWarnings("deprecation")
public class OptTreeConverter
{
@@ -689,98 +679,10 @@ public class OptTreeConverter
private static OptNodeStatistics analyzeStatistics(Instruction inst, OptNode on, LocalVariableMap vars)
throws DMLRuntimeException
{
- OptNodeStatistics ret = null;
- String instName = on.getInstructionName();
+ //since the performance test tool for offline profiling has been removed,
+ //we return default values
- if( PerfTestTool.isRegisteredInstruction(instName) )
- {
- if( inst instanceof DataGenCPInstruction )
- {
- DataGenCPInstruction linst = (DataGenCPInstruction) inst;
- DataFormat df = ( MatrixBlock.evalSparseFormatInMemory(linst.getRows(), linst.getCols(), (long)(linst.getSparsity()*linst.getRows()*linst.getCols())) ?
- DataFormat.SPARSE : DataFormat.DENSE );
- ret = new OptNodeStatistics(linst.getRows(), linst.getCols(), -1, -1, linst.getSparsity(), df);
- }
- else if ( inst instanceof FunctionCallCPInstruction )
- {
- FunctionCallCPInstruction linst = (FunctionCallCPInstruction)inst;
- ArrayList<String> params = linst.getBoundInputParamNames();
- ret = new OptNodeStatistics(); //default vals
-
- double maxSize = 0;
- for( String param : params ) //use the largest input matrix
- {
- Data dat = vars.get(param);
- if( dat!=null && dat.getDataType()==DataType.MATRIX )
- {
- MatrixObject mdat1 = (MatrixObject) dat;
- MatrixCharacteristics mc1 = mdat1.getMatrixCharacteristics();
-
- if( mc1.getRows()*mc1.getCols() > maxSize )
- {
- ret.setDim1( mc1.getRows() );
- ret.setDim2( mc1.getCols() );
- ret.setSparsity( OptimizerUtils.getSparsity(ret.getDim1(), ret.getDim2(), mc1.getNonZeros()) ); //sparsity
- ret.setDataFormat( MatrixBlock.evalSparseFormatInMemory(mc1.getRows(), mc1.getCols(), mc1.getNonZeros()) ?
- DataFormat.SPARSE : DataFormat.DENSE );
- maxSize = mc1.getRows()*mc1.getCols();
- }
- }
- }
- }
- else if ( inst instanceof ComputationCPInstruction ) //needs to be last CP case
- {
- //AggregateBinaryCPInstruction, AggregateUnaryCPInstruction,
- //FunctionCallCPInstruction, ReorgCPInstruction
-
- ComputationCPInstruction linst = (ComputationCPInstruction) inst;
- ret = new OptNodeStatistics(); //default
-
- if( linst.input1 != null && linst.input2 != null ) //binary
- {
- Data dat1 = vars.get( linst.input1.getName() );
- Data dat2 = vars.get( linst.input2.getName() );
-
- if( dat1 != null )
- {
- MatrixObject mdat1 = (MatrixObject) dat1;
- MatrixCharacteristics mc1 = ((MatrixFormatMetaData)mdat1.getMetaData()).getMatrixCharacteristics();
- ret.setDim1( mc1.getRows() );
- ret.setDim2( mc1.getCols() );
- ret.setSparsity( OptimizerUtils.getSparsity(ret.getDim1(), ret.getDim2(), mc1.getNonZeros()) ); //sparsity
- ret.setDataFormat( MatrixBlock.evalSparseFormatInMemory(mc1.getRows(), mc1.getCols(), mc1.getNonZeros())? DataFormat.SPARSE : DataFormat.DENSE);
- }
- if( dat2 != null )
- {
- MatrixObject mdat2 = (MatrixObject) dat2;
- MatrixCharacteristics mc2 = ((MatrixFormatMetaData)mdat2.getMetaData()).getMatrixCharacteristics();
- ret.setDim3( mc2.getRows() );
- ret.setDim4( mc2.getCols() );
- ret.setDataFormat( MatrixBlock.evalSparseFormatInMemory(mc2.getRows(), mc2.getCols(), mc2.getNonZeros()) ? DataFormat.SPARSE : DataFormat.DENSE );
- }
- }
- else //unary
- {
- if( linst.input1 != null )
- {
- Data dat1 = vars.get( linst.input1.getName() );
- if( dat1 != null ) {
- MatrixObject mdat1 = (MatrixObject) dat1;
- MatrixCharacteristics mc1 = ((MatrixFormatMetaData)mdat1.getMetaData()).getMatrixCharacteristics();
- ret.setDim1( mc1.getRows() );
- ret.setDim2( mc1.getCols() );
- ret.setSparsity( OptimizerUtils.getSparsity(ret.getDim1(), ret.getDim2(), mc1.getNonZeros()) ); //sparsity
- ret.setDataFormat(MatrixBlock.evalSparseFormatInMemory(mc1.getRows(), mc1.getCols(), mc1.getNonZeros()) ? DataFormat.SPARSE : DataFormat.DENSE);
- }
- }
- }
- }
- }
-
- if( ret == null )
- ret = new OptNodeStatistics(); //default values
-
- return ret; //null if not reqistered for profiling
+ return new OptNodeStatistics(); //default values
}
public static void replaceProgramBlock(OptNode parent, OptNode n, ProgramBlock pbOld, ProgramBlock pbNew, boolean rtMap)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/35da413a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingAbstract.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingAbstract.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingAbstract.java
index 6d80f64..865dc9d 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingAbstract.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingAbstract.java
@@ -29,13 +29,11 @@ import org.apache.sysml.runtime.controlprogram.Program;
import org.apache.sysml.runtime.controlprogram.ProgramBlock;
public class OptTreePlanMappingAbstract extends OptTreePlanMapping
-{
-
+{
private DMLProgram _prog;
private Program _rtprog;
private Map<Long, Object> _id_hlprog;
private Map<Long, Object> _id_rtprog;
- private Map<Long, Object> _id_symb; // mapping for symbol table
public OptTreePlanMappingAbstract( )
{
@@ -46,7 +44,6 @@ public class OptTreePlanMappingAbstract extends OptTreePlanMapping
_id_hlprog = new HashMap<Long, Object>();
_id_rtprog = new HashMap<Long, Object>();
- _id_symb = new HashMap<Long, Object>();
}
public void putRootProgram( DMLProgram prog, Program rtprog )
@@ -61,7 +58,6 @@ public class OptTreePlanMappingAbstract extends OptTreePlanMapping
_id_hlprog.put(id, hops);
_id_rtprog.put(id, null);
- _id_symb.put(id, null);
_id_optnode.put(id, n);
n.setID(id);
@@ -75,7 +71,6 @@ public class OptTreePlanMappingAbstract extends OptTreePlanMapping
_id_hlprog.put(id, sb);
_id_rtprog.put(id, pb);
- _id_symb.put(id, null);
_id_optnode.put(id, n);
n.setID(id);
@@ -97,10 +92,9 @@ public class OptTreePlanMappingAbstract extends OptTreePlanMapping
public Object[] getMappedProg( long id )
{
- Object[] ret = new Object[3];
+ Object[] ret = new Object[2];
ret[0] = (StatementBlock)_id_hlprog.get( id );
ret[1] = (ProgramBlock)_id_rtprog.get( id );
- ret[2] = (ProgramBlock)_id_symb.get( id );
return ret;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/35da413a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingRuntime.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingRuntime.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingRuntime.java
index 0a7d9f6..8c131f3 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingRuntime.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMappingRuntime.java
@@ -26,8 +26,7 @@ import org.apache.sysml.runtime.controlprogram.ProgramBlock;
import org.apache.sysml.runtime.instructions.Instruction;
public class OptTreePlanMappingRuntime extends OptTreePlanMapping
-{
-
+{
private Map<Long, Object> _id_rtprog;
public OptTreePlanMappingRuntime()
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/35da413a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
index a7064da..2469005 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
@@ -76,7 +76,6 @@ public class OptimizationWrapper
//internal parameters
public static final double PAR_FACTOR_INFRASTRUCTURE = 1.0;
- private static final boolean ALLOW_RUNTIME_COSTMODEL = false;
private static final boolean CHECK_PLAN_CORRECTNESS = false;
static
@@ -147,12 +146,6 @@ public class OptimizationWrapper
CostModelType cmtype = opt.getCostModelType();
LOG.trace("ParFOR Opt: Created optimizer ("+otype+","+opt.getPlanInputType()+","+opt.getCostModelType());
- if( cmtype == CostModelType.RUNTIME_METRICS //TODO remove check when perftesttool supported
- && !ALLOW_RUNTIME_COSTMODEL )
- {
- throw new DMLRuntimeException("ParFOR Optimizer "+otype+" requires cost model "+cmtype+" that is not suported yet.");
- }
-
OptTree tree = null;
//recompile parfor body
@@ -243,7 +236,7 @@ public class OptimizationWrapper
}
//create cost estimator
- CostEstimator est = createCostEstimator( cmtype );
+ CostEstimator est = createCostEstimator( cmtype, ec.getVariables() );
LOG.trace("ParFOR Opt: Created cost estimator ("+cmtype+")");
//core optimize
@@ -296,15 +289,6 @@ public class OptimizationWrapper
case CONSTRAINED:
opt = new OptimizerConstrained();
break;
-
- //MB: removed unused and experimental prototypes
- //case FULL_DP:
- // opt = new OptimizerDPEnum();
- // break;
- //case GREEDY:
- // opt = new OptimizerGreedyEnum();
- // break;
-
default:
throw new DMLRuntimeException("Undefined optimizer: '"+otype+"'.");
}
@@ -312,7 +296,7 @@ public class OptimizationWrapper
return opt;
}
- private static CostEstimator createCostEstimator( CostModelType cmtype )
+ private static CostEstimator createCostEstimator( CostModelType cmtype, LocalVariableMap vars )
throws DMLRuntimeException
{
CostEstimator est = null;
@@ -320,10 +304,13 @@ public class OptimizationWrapper
switch( cmtype )
{
case STATIC_MEM_METRIC:
- est = new CostEstimatorHops( OptTreeConverter.getAbstractPlanMapping() );
+ est = new CostEstimatorHops(
+ OptTreeConverter.getAbstractPlanMapping() );
break;
case RUNTIME_METRICS:
- est = new CostEstimatorRuntime();
+ est = new CostEstimatorRuntime(
+ OptTreeConverter.getAbstractPlanMapping(),
+ (LocalVariableMap)vars.clone() );
break;
default:
throw new DMLRuntimeException("Undefined cost model type: '"+cmtype+"'.");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/35da413a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/Optimizer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/Optimizer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/Optimizer.java
index fdeffb8..af92b80 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/Optimizer.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/Optimizer.java
@@ -39,8 +39,6 @@ import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
*/
public abstract class Optimizer
{
-
-
protected static final Log LOG = LogFactory.getLog(Optimizer.class.getName());
protected long _numTotalPlans = -1;
@@ -86,14 +84,11 @@ public abstract class Optimizer
///////
//methods for evaluating the overall properties and costing
- public long getNumTotalPlans()
- {
+ public long getNumTotalPlans() {
return _numTotalPlans;
}
- public long getNumEvaluatedPlans()
- {
+ public long getNumEvaluatedPlans() {
return _numEvaluatedPlans;
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/35da413a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
index 9754e6f..39e742f 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
@@ -62,8 +62,7 @@ public class OptimizerConstrained extends OptimizerRuleBased
{
@Override
- public POptMode getOptMode()
- {
+ public POptMode getOptMode() {
return POptMode.CONSTRAINED;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/35da413a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerHeuristic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerHeuristic.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerHeuristic.java
index a53f7f1..88d7798 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerHeuristic.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerHeuristic.java
@@ -25,31 +25,22 @@ import org.apache.sysml.runtime.controlprogram.parfor.opt.CostEstimator.TestMeas
/**
- * Heuristic ParFor Optimizer (time: O(n)):
+ * Heuristic ParFor Optimizer: This optimizer extends the rule-based
+ * optimizer by a time-based cost estimate for execution type decisions.
*
*
*/
public class OptimizerHeuristic extends OptimizerRuleBased
{
-
-
- public static final double EXEC_TIME_THRESHOLD = 60000; //in ms
+ public static final double EXEC_TIME_THRESHOLD = 30000; //in ms
@Override
- public CostModelType getCostModelType()
- {
+ public CostModelType getCostModelType() {
return CostModelType.RUNTIME_METRICS;
}
-
- @Override
- public PlanInputType getPlanInputType()
- {
- return PlanInputType.RUNTIME_PLAN;
- }
@Override
- public POptMode getOptMode()
- {
+ public POptMode getOptMode() {
return POptMode.HEURISTIC;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/35da413a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestDef.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestDef.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestDef.java
deleted file mode 100644
index 08c02ef..0000000
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestDef.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.controlprogram.parfor.opt;
-
-import org.apache.sysml.runtime.controlprogram.parfor.opt.CostEstimator.DataFormat;
-import org.apache.sysml.runtime.controlprogram.parfor.opt.CostEstimator.TestMeasure;
-import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.InternalTestVariable;
-import org.apache.sysml.runtime.controlprogram.parfor.opt.PerfTestTool.TestVariable;
-
-/**
- * Internal representation of a test configuration consisting of a logical TestMeasure,
- * a logical TestVariable as well as a DataFormat. Note that one cost function refers
- * to a statistical model of a profiling run for a combination of such a test configuration
- * and an instruction.
- *
- */
-@Deprecated
-public class PerfTestDef
-{
- //logical properties
- private TestMeasure _measure;
- private TestVariable _lvariable;
- private DataFormat _dataformat;
-
- //physical properties
- private InternalTestVariable[] _pvariables;
- private double _min;
- private double _max;
- private double _samples;
-
- public PerfTestDef( TestMeasure m, TestVariable lv, DataFormat df, InternalTestVariable pv, double min, double max, double samples )
- {
- this( m, lv, df, new InternalTestVariable[]{pv}, min, max, samples);
- }
-
- public PerfTestDef( TestMeasure m, TestVariable lv, DataFormat df, InternalTestVariable[] pv, double min, double max, double samples )
- {
- _measure = m;
- _lvariable = lv;
- _dataformat = df;
-
- _pvariables = pv;
- _min = min;
- _max = max;
- _samples = samples;
- }
-
- public TestMeasure getMeasure()
- {
- return _measure;
- }
-
- public TestVariable getVariable()
- {
- return _lvariable;
- }
-
- public DataFormat getDataformat()
- {
- return _dataformat;
- }
-
- public InternalTestVariable[] getInternalVariables()
- {
- return _pvariables;
- }
-
- public double getMin()
- {
- return _min;
- }
-
- public double getMax()
- {
- return _max;
- }
-
- public double getNumSamples()
- {
- return _samples;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/35da413a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestExtFunctCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestExtFunctCP.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestExtFunctCP.java
deleted file mode 100644
index 5a1fac8..0000000
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestExtFunctCP.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.controlprogram.parfor.opt;
-
-import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence;
-import org.apache.sysml.runtime.matrix.data.InputInfo;
-import org.apache.sysml.runtime.matrix.data.MatrixBlock;
-import org.apache.sysml.runtime.matrix.data.OutputInfo;
-import org.apache.sysml.udf.FunctionParameter;
-import org.apache.sysml.udf.Matrix;
-import org.apache.sysml.udf.PackageFunction;
-import org.apache.sysml.udf.Matrix.ValueType;
-
-/**
- * External function (type CP) used within the PerfTestTool in order to
- * measure the general behavior of package support.
- *
- */
-@Deprecated
-public class PerfTestExtFunctCP extends PackageFunction
-{
-
- private static final long serialVersionUID = 1L;
- private static final String OUTPUT_FILE = "PerfTestExtFunctOutput";
-
- private static IDSequence _idSeq = new IDSequence();
- private Matrix _ret = null;
- private String _baseDir = null;
-
- @Override
- public int getNumFunctionOutputs()
- {
- return 1;
- }
-
- @Override
- public FunctionParameter getFunctionOutput(int pos)
- {
- return _ret;
- }
-
- public void setBaseDir(String dir)
- {
- _baseDir = dir;
- }
-
- @Override
- public void execute()
- {
- try
- {
- long id = _idSeq.getNextID();
-
- Matrix in = (Matrix) this.getFunctionInput(0);
- double [][] aIn = in.getMatrixAsDoubleArray();
-
- int rows = aIn.length;
- int cols = aIn[0].length;
-
- String dir = _baseDir + "/" + OUTPUT_FILE+id;
-
- //copy and write output data
- MatrixBlock mb = new MatrixBlock(rows,cols,false);
- for(int i=0; i < rows; i++)
- for(int j=0; j < cols; j++)
- mb.setValue(i, j, aIn[i][j]);
-
- _ret = new Matrix(dir, rows, cols, ValueType.Double);
- _ret.setMatrixDoubleArray(mb, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
- }
- catch(Exception e)
- {
- throw new RuntimeException("Error executing generic test extfunct.", e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/35da413a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestMemoryObserver.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestMemoryObserver.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestMemoryObserver.java
deleted file mode 100644
index 0698cbe..0000000
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestMemoryObserver.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.runtime.controlprogram.parfor.opt;
-
-import java.lang.ref.WeakReference;
-
-/**
- * Observer thread for asynchronously monitor the memory consumption.
- * It periodically measures the used memory with period <code>MEASURE_INTERVAL</code>,
- * by explicitly invoking the garbage collector (if required until it is really executed)
- * and afterwards obtaining the currently used memory.
- *
- * Protocol: (1) measure start, (2) start thread, (3) *do some work*, (4) join thread, (5) get max memory.
- *
- */
-@Deprecated
-public class PerfTestMemoryObserver implements Runnable
-{
-
- public static final int MEASURE_INTERVAL = 50; //in ms
-
- private long _startMem = -1;
- private long _maxMem = -1;
- private boolean _stopped = false;
-
- public PerfTestMemoryObserver()
- {
- _startMem = -1;
- _maxMem = -1;
- _stopped = false;
- }
-
- public void measureStartMem()
- {
- forceGC();
- _startMem = Runtime.getRuntime().totalMemory()
- - Runtime.getRuntime().freeMemory();
- }
-
- public long getMaxMemConsumption()
- {
- long val = _maxMem - _startMem;
- return (val < 0) ? 0 : val;
- }
-
- public void setStopped()
- {
- _stopped = true;
- }
-
- @Override
- public void run()
- {
- try
- {
- while( !_stopped )
- {
- forceGC();
- long value = Runtime.getRuntime().totalMemory()
- - Runtime.getRuntime().freeMemory();
-
- _maxMem = Math.max(value, _maxMem);
-
- Thread.sleep( MEASURE_INTERVAL );
- }
- }
- catch(Exception ex)
- {
- throw new RuntimeException("Error measuring Java memory usage", ex);
- }
- }
-
- public static double getUsedMemory()
- {
- forceGC();
- return ( Runtime.getRuntime().totalMemory()
- - Runtime.getRuntime().freeMemory() );
- }
-
- private static void forceGC()
- {
- //request gc until weak reference is eliminated by gc
- Object o = new Object();
- WeakReference<Object> ref = new WeakReference<Object>(o); //collected, everytime gc is actually invoked
- while((o=ref.get())!= null)
- System.gc(); //called on purpose, no production use.
- }
-}
\ No newline at end of file