You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/03/19 23:21:39 UTC
[1/4] incubator-systemml git commit: [SYSTEMML-585] Fix JMLC
connection to disable any multi-threaded ops
Repository: incubator-systemml
Updated Branches:
refs/heads/master a939658ee -> 257793d83
[SYSTEMML-585] Fix JMLC connection to disable any multi-threaded ops
So far we did not explicitly disabled multi-threaded operations (other
than read/write) and parfor for JMLC. This can lead to performance
problems, if JMLC itself is used in a multi-threaded manner. We now
explicitly disable all multi-threaded operations, remote parfor
operators, and local parfor multi-threading. Furthermore, we also
disable dynamic recompilation to avoid side effects from advanced
compilation steps.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/59a4a50a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/59a4a50a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/59a4a50a
Branch: refs/heads/master
Commit: 59a4a50acb9432f2cb976e5c26fd3daea323af2e
Parents: a939658
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Fri Mar 18 17:16:47 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sat Mar 19 15:13:39 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/sysml/api/jmlc/Connection.java | 8 +++++++-
.../java/org/apache/sysml/hops/OptimizerUtils.java | 15 +++++++++++----
.../parfor/opt/OptimizerConstrained.java | 4 ++--
.../parfor/opt/OptimizerRuleBased.java | 15 ++++++---------
4 files changed, 26 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/59a4a50a/src/main/java/org/apache/sysml/api/jmlc/Connection.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
index 0fe5966..43b56d0 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
@@ -77,10 +77,13 @@ public class Connection
//runtime parameters
DMLScript.rtplatform = RUNTIME_PLATFORM.SINGLE_NODE;
OptimizerUtils.ALLOW_CSE_PERSISTENT_READS = false;
+ OptimizerUtils.PARALLEL_CP_MATRIX_OPERATIONS = false;
+ OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR = false;
OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS = false;
OptimizerUtils.PARALLEL_CP_WRITE_TEXTFORMATS = false;
OptimizerUtils.PARALLEL_CP_READ_BINARYFORMATS = false;
OptimizerUtils.PARALLEL_CP_WRITE_BINARYFORMATS = false;
+ OptimizerUtils.ALLOW_DYN_RECOMPILATION = false;
CacheableData.disableCaching();
//create default configuration
@@ -167,10 +170,13 @@ public class Connection
DataExpression.REJECT_READ_WRITE_UNKNOWNS = true;
OptimizerUtils.ALLOW_CSE_PERSISTENT_READS =
OptimizerUtils.ALLOW_COMMON_SUBEXPRESSION_ELIMINATION;
+ OptimizerUtils.PARALLEL_CP_MATRIX_OPERATIONS = true;
+ OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR = true;
OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS = true;
OptimizerUtils.PARALLEL_CP_WRITE_TEXTFORMATS = true;
OptimizerUtils.PARALLEL_CP_READ_BINARYFORMATS = true;
- OptimizerUtils.PARALLEL_CP_WRITE_BINARYFORMATS = true;
+ OptimizerUtils.PARALLEL_CP_WRITE_BINARYFORMATS = true;
+ OptimizerUtils.ALLOW_DYN_RECOMPILATION = true;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/59a4a50a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
index b0a0790..66ab82b 100644
--- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
@@ -210,10 +210,17 @@ public class OptimizerUtils
public static final double PARALLEL_CP_WRITE_PARALLELISM_MULTIPLIER = 1.0;
/**
- * Enables multi-threaded matrix multiply for mm, mmchain, and tsmm.
+ * Enables multi-threaded operations for mm, mmchain, and tsmm,
+ * rand, wdivmm, wsloss, wumm, wcemm, uagg, tak, and groupedaggregate.
*
*/
- public static boolean PARALLEL_CP_MATRIX_MULTIPLY = true;
+ public static boolean PARALLEL_CP_MATRIX_OPERATIONS = true;
+
+ /**
+ * Enables multi-threaded local or distributed remote parfor operators.
+ * Otherwise parfor is restricted to parfor local with par=1.
+ */
+ public static boolean PARALLEL_LOCAL_OR_REMOTE_PARFOR = true;
/**
* Enables the use of CombineSequenceFileInputFormat with splitsize = 2x hdfs blocksize,
@@ -380,7 +387,7 @@ public class OptimizerUtils
//handle parallel matrix mult / rand configuration
if (!ConfigurationManager.getConfig().getBooleanValue(DMLConfig.CP_PARALLEL_MATRIXMULT)) {
- PARALLEL_CP_MATRIX_MULTIPLY = false;
+ PARALLEL_CP_MATRIX_OPERATIONS = false;
}
}
@@ -947,7 +954,7 @@ public class OptimizerUtils
}
//apply global multi-threading constraint
- if( !PARALLEL_CP_MATRIX_MULTIPLY ) {
+ if( !PARALLEL_CP_MATRIX_OPERATIONS ) {
ret = 1;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/59a4a50a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
index 5f3fb75..a6b7763 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
@@ -265,7 +265,7 @@ public class OptimizerConstrained extends OptimizerRuleBased
boolean ret = false;
// constraint awareness
- if( n.getExecType() != null )
+ if( n.getExecType() != null && OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR )
{
ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter
.getAbstractPlanMapping().getMappedProg(n.getID())[1];
@@ -298,7 +298,7 @@ public class OptimizerConstrained extends OptimizerRuleBased
throws DMLRuntimeException
{
// constraint awareness
- if( n.getK()>0 )
+ if( n.getK() > 0 && OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR )
{
ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter
.getAbstractPlanMapping().getMappedProg(n.getID())[1];
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/59a4a50a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index 6d17a06..4e871a1 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -958,14 +958,14 @@ public class OptimizerRuleBased extends Optimizer
boolean isCPOnly = n.isCPOnly();
boolean isCPOnlyPossible = isCPOnly || isCPOnlyPossible(n, _rm);
-
String datapartitioner = n.getParam(ParamType.DATA_PARTITIONER);
ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
PDataPartitioner REMOTE_DP = OptimizerUtils.isSparkExecutionMode() ? PDataPartitioner.REMOTE_SPARK : PDataPartitioner.REMOTE_MR;
//deciding on the execution strategy
- if( (isCPOnly && M <= _rm ) //Required: all instruction can be be executed in CP
- || (isCPOnlyPossible && M2 <= _rm) ) //Required: cp inst fit into remote JVM mem
+ if( OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR //allowed remote parfor execution
+ && ( (isCPOnly && M <= _rm ) //Required: all instruction can be be executed in CP
+ ||(isCPOnlyPossible && M2 <= _rm)) ) //Required: cp inst fit into remote JVM mem
{
//at this point all required conditions for REMOTE_MR given, now its an opt decision
int cpk = (int) Math.min( _lk, Math.floor( _lm / M ) ); //estimated local exploited par
@@ -1436,11 +1436,8 @@ public class OptimizerRuleBased extends Optimizer
if( type == ExecType.CP )
{
//determine local max parallelism constraint
- int kMax = -1;
- if( n.isCPOnly() )
- kMax = _lkmaxCP;
- else
- kMax = _lkmaxMR;
+ int kMax = OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR ?
+ (n.isCPOnly() ? _lkmaxCP : _lkmaxMR) : 1;
//ensure local memory constraint (for spark more conservative in order to
//prevent unnecessary guarded collect)
@@ -1538,7 +1535,7 @@ public class OptimizerRuleBased extends Optimizer
{
//set degree of parallelism for multi-threaded leaf nodes
Hop h = OptTreeConverter.getAbstractPlanMapping().getMappedHop(c.getID());
- if( OptimizerUtils.PARALLEL_CP_MATRIX_MULTIPLY
+ if( OptimizerUtils.PARALLEL_CP_MATRIX_OPERATIONS
&& h instanceof MultiThreadedHop //abop, datagenop, qop, paramop
&& !( h instanceof ParameterizedBuiltinOp //only paramop-grpagg
&& ((ParameterizedBuiltinOp)h).getOp()!=ParamBuiltinOp.GROUPEDAGG) )
[3/4] incubator-systemml git commit: [SYSTEMML-584] New thread-local
configuration handling (dml/compiler)
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
index a7b4776..dcee148 100644
--- a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
+++ b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
@@ -31,6 +31,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.BinaryOp;
import org.apache.sysml.hops.DataGenOp;
import org.apache.sysml.hops.DataOp;
@@ -692,7 +693,7 @@ public class InterProceduralAnalysis
MatrixObject mo = new MatrixObject(ValueType.DOUBLE, null);
MatrixCharacteristics mc = new MatrixCharacteristics(
input.getDim1(), input.getDim2(),
- DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize,
+ ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(),
inputSafeNNZ.contains(input.getHopID())?input.getNnz():-1 );
MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
mo.setMetaData(meta);
@@ -873,7 +874,7 @@ public class InterProceduralAnalysis
MatrixObject moOut = new MatrixObject(ValueType.DOUBLE, null);
MatrixCharacteristics mc = new MatrixCharacteristics(
dim1, dim2,
- DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize,
+ ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(),
nnz);
MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
moOut.setMetaData(meta);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
index aa0eb03..3edf7f7 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.wink.json4j.JSONObject;
-
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.DataGenOp;
@@ -62,7 +61,6 @@ import org.apache.sysml.lops.LopsException;
import org.apache.sysml.lops.ReBlock;
import org.apache.sysml.lops.compile.Dag;
import org.apache.sysml.parser.DMLProgram;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.DataExpression;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
@@ -125,15 +123,18 @@ public class Recompiler
private static final long CP_CSV_REBLOCK_UNKNOWN_THRESHOLD_SIZE = (long)256*1024*1024;
private static final long CP_TRANSFORM_UNKNOWN_THRESHOLD_SIZE = (long)1024*1024*1024;
- //reused rewriter for dynamic rewrites during recompile
- private static ProgramRewriter rewriter = new ProgramRewriter(false, true);
+ /** Local reused rewriter for dynamic rewrites during recompile */
+
+ /** Local DML configuration for thread-local config updates */
+ private static ThreadLocal<ProgramRewriter> _rewriter = new ThreadLocal<ProgramRewriter>() {
+ @Override protected ProgramRewriter initialValue() { return new ProgramRewriter(false, true); }
+ };
/**
* Re-initializes the recompiler according to the current optimizer flags.
*/
- public static void reinitRecompiler()
- {
- rewriter = new ProgramRewriter(false, true);
+ public static void reinitRecompiler() {
+ _rewriter.set(new ProgramRewriter(false, true));
}
/**
@@ -190,7 +191,7 @@ public class Recompiler
// dynamic hop rewrites
if( !inplace )
- rewriter.rewriteHopDAGs( hops, null );
+ _rewriter.get().rewriteHopDAGs( hops, null );
// refresh memory estimates (based on updated stats,
// before: init memo table with propagated worst-case estimates,
@@ -211,7 +212,7 @@ public class Recompiler
}
// generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(sb, ConfigurationManager.getConfig());
+ newInst = dag.getJobs(sb, ConfigurationManager.getDMLConfig());
}
// replace thread ids in new instructions
@@ -286,7 +287,7 @@ public class Recompiler
// dynamic hop rewrites
if( !inplace )
- rewriter.rewriteHopDAG( hops, null );
+ _rewriter.get().rewriteHopDAG( hops, null );
// refresh memory estimates (based on updated stats)
MemoTable memo = new MemoTable();
@@ -301,7 +302,7 @@ public class Recompiler
lops.addToDag(dag);
// generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(null, ConfigurationManager.getConfig());
+ newInst = dag.getJobs(null, ConfigurationManager.getDMLConfig());
}
// replace thread ids in new instructions
@@ -363,7 +364,7 @@ public class Recompiler
}
// generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(sb, ConfigurationManager.getConfig());
+ newInst = dag.getJobs(sb, ConfigurationManager.getDMLConfig());
}
// replace thread ids in new instructions
@@ -415,7 +416,7 @@ public class Recompiler
lops.addToDag(dag);
// generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(null, ConfigurationManager.getConfig());
+ newInst = dag.getJobs(null, ConfigurationManager.getDMLConfig());
}
// replace thread ids in new instructions
@@ -461,7 +462,7 @@ public class Recompiler
}
// generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(sb, ConfigurationManager.getConfig());
+ newInst = dag.getJobs(sb, ConfigurationManager.getDMLConfig());
}
// explain recompiled hops / instructions
@@ -508,7 +509,7 @@ public class Recompiler
lops.addToDag(dag);
// generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(null, ConfigurationManager.getConfig());
+ newInst = dag.getJobs(null, ConfigurationManager.getDMLConfig());
}
// explain recompiled instructions
@@ -1164,7 +1165,7 @@ public class Recompiler
MatrixObject moOut = new MatrixObject(ValueType.DOUBLE, null);
MatrixCharacteristics mc = new MatrixCharacteristics(
dim1, dim2,
- DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize,
+ ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(),
nnz);
MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
moOut.setMetaData(meta);
@@ -1482,7 +1483,7 @@ public class Recompiler
MatrixObject mo = new MatrixObject(ValueType.DOUBLE, null);
MatrixCharacteristics mc = new MatrixCharacteristics(
hop.getDim1(), hop.getDim2(),
- DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize,
+ ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(),
hop.getNnz());
MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
mo.setMetaData(meta);
@@ -1925,7 +1926,7 @@ public class Recompiler
long nnz = mc.getNonZeros();
//check valid cp reblock recompilation hook
- if( !OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ if( !ConfigurationManager.isDynamicRecompilation()
|| !OptimizerUtils.isHybridExecutionMode() )
{
return false;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
index 5c36074..cfc5855 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
@@ -25,7 +25,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.HopsException;
import org.apache.sysml.hops.OptimizerUtils;
@@ -109,7 +110,7 @@ public class ProgramRewriter
_sbRuleSet.add( new RewriteRemoveUnnecessaryBranches() ); //dependency: constant folding
if( OptimizerUtils.ALLOW_SPLIT_HOP_DAGS )
_sbRuleSet.add( new RewriteSplitDagUnknownCSVRead() ); //dependency: reblock
- if( OptimizerUtils.ALLOW_INDIVIDUAL_SB_SPECIFIC_OPS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.ALLOW_INDIVIDUAL_SB_SPECIFIC_OPS) )
_sbRuleSet.add( new RewriteSplitDagDataDependentOperators() );
if( OptimizerUtils.ALLOW_AUTO_VECTORIZATION )
_sbRuleSet.add( new RewriteForLoopVectorization() ); //dependency: reblock (reblockop)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
index f23f87d..1d7e1db 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.AggBinaryOp;
import org.apache.sysml.hops.AggUnaryOp;
import org.apache.sysml.hops.BinaryOp;
@@ -44,7 +45,6 @@ import org.apache.sysml.hops.Hop.OpOp2;
import org.apache.sysml.hops.ReorgOp;
import org.apache.sysml.hops.UnaryOp;
import org.apache.sysml.lops.MapMultChain.ChainType;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.DataExpression;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
@@ -612,7 +612,7 @@ public class RewriteAlgebraicSimplificationDynamic extends HopRewriteRule
//create cast to keep same output datatype
UnaryOp cast = new UnaryOp(uhi.getName(), DataType.MATRIX, ValueType.DOUBLE,
OpOp1.CAST_AS_MATRIX, uhi);
- HopRewriteUtils.setOutputParameters(cast, 1, 1, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, -1);
+ HopRewriteUtils.setOutputParameters(cast, 1, 1, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), -1);
//rehang cast under all parents
for( Hop p : parents ) {
@@ -700,7 +700,7 @@ public class RewriteAlgebraicSimplificationDynamic extends HopRewriteRule
//create cast to keep same output datatype
UnaryOp cast = new UnaryOp(uhi.getName(), DataType.MATRIX, ValueType.DOUBLE,
OpOp1.CAST_AS_MATRIX, uhi);
- HopRewriteUtils.setOutputParameters(cast, 1, 1, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, -1);
+ HopRewriteUtils.setOutputParameters(cast, 1, 1, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), -1);
//rehang cast under all parents
for( Hop p : parents ) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java
index 96b20e0..ce09a6f 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.DataOp;
import org.apache.sysml.hops.FunctionOp;
import org.apache.sysml.hops.Hop;
@@ -31,7 +32,6 @@ import org.apache.sysml.hops.Hop.FileFormatTypes;
import org.apache.sysml.hops.Hop.ParamBuiltinOp;
import org.apache.sysml.hops.HopsException;
import org.apache.sysml.hops.ParameterizedBuiltinOp;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.Expression.DataType;
/**
@@ -52,11 +52,11 @@ public class RewriteBlockSizeAndReblock extends HopRewriteRule
//maintain rewrite status
if( isReblockValid() )
- state.setBlocksize(DMLTranslator.DMLBlockSize);
+ state.setBlocksize(ConfigurationManager.getBlocksize());
//perform reblock and blocksize rewrite
for( Hop h : roots )
- rule_BlockSizeAndReblock(h, DMLTranslator.DMLBlockSize);
+ rule_BlockSizeAndReblock(h, ConfigurationManager.getBlocksize());
return roots;
}
@@ -70,10 +70,10 @@ public class RewriteBlockSizeAndReblock extends HopRewriteRule
//maintain rewrite status
if( isReblockValid() )
- state.setBlocksize(DMLTranslator.DMLBlockSize);
+ state.setBlocksize(ConfigurationManager.getBlocksize());
//perform reblock and blocksize rewrite
- rule_BlockSizeAndReblock(root, DMLTranslator.DMLBlockSize);
+ rule_BlockSizeAndReblock(root, ConfigurationManager.getBlocksize());
return root;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java
index a0d8e7b..945b21f 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java
@@ -208,7 +208,7 @@ public class RewriteConstantFolding extends HopRewriteRule
Recompiler.rClearLops(tmpWrite); //prevent lops reuse
Lop lops = tmpWrite.constructLops(); //reconstruct lops
lops.addToDag( dag );
- ArrayList<Instruction> inst = dag.getJobs(null, ConfigurationManager.getConfig());
+ ArrayList<Instruction> inst = dag.getJobs(null, ConfigurationManager.getDMLConfig());
//execute instructions
ExecutionContext ec = getExecutionContext();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
index 1785a51..6f9ab63 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
@@ -22,6 +22,7 @@ package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
import java.util.HashSet;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.AggBinaryOp;
import org.apache.sysml.hops.DataOp;
import org.apache.sysml.hops.Hop;
@@ -33,7 +34,6 @@ import org.apache.sysml.hops.Hop.ReOrgOp;
import org.apache.sysml.hops.Hop.VisitStatus;
import org.apache.sysml.hops.HopsException;
import org.apache.sysml.hops.LiteralOp;
-import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.ParameterizedBuiltinOp;
import org.apache.sysml.hops.ReorgOp;
import org.apache.sysml.hops.TernaryOp;
@@ -276,7 +276,7 @@ public class RewriteSplitDagDataDependentOperators extends StatementBlockRewrite
//configure rmEmpty to directly output selection vector
//(only applied if dynamic recompilation enabled)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION )
+ if( ConfigurationManager.isDynamicRecompilation() )
pbhop.setOutputPermutationMatrix(true);
for( Hop p : hop.getParent() )
((AggBinaryOp)p).setHasLeftPMInput(true);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/lops/compile/Dag.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/compile/Dag.java b/src/main/java/org/apache/sysml/lops/compile/Dag.java
index 2944444..472149e 100644
--- a/src/main/java/org/apache/sysml/lops/compile/Dag.java
+++ b/src/main/java/org/apache/sysml/lops/compile/Dag.java
@@ -202,8 +202,7 @@ public class Dag<N extends Lop>
IDMap = new HashMap<Long, Integer>();
// get number of reducers from dml config
- DMLConfig conf = ConfigurationManager.getConfig();
- total_reducers = conf.getIntValue(DMLConfig.NUM_REDUCERS);
+ total_reducers = ConfigurationManager.getNumReducers();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java b/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
index bf40415..a49c61b 100644
--- a/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
+++ b/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
@@ -178,7 +178,7 @@ public class RunMRJobs
String shuffleInst = inst.getIv_shuffleInstructions();
String aggInst = inst.getIv_aggInstructions();
String otherInst = inst.getIv_otherInstructions();
- boolean jvmReuse = ConfigurationManager.getConfig().getBooleanValue(DMLConfig.JVM_REUSE);
+ boolean jvmReuse = ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.JVM_REUSE);
switch(inst.getJobType()) {
@@ -193,7 +193,7 @@ public class RunMRJobs
break;
case DATAGEN:
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ if( ConfigurationManager.isDynamicRecompilation()
&& OptimizerUtils.ALLOW_RAND_JOB_RECOMPILE
&& DMLScript.rtplatform != RUNTIME_PLATFORM.HADOOP
&& Recompiler.checkCPDataGen( inst, rdInst ) )
@@ -229,7 +229,7 @@ public class RunMRJobs
case REBLOCK:
case CSV_REBLOCK:
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ if( ConfigurationManager.isDynamicRecompilation()
&& DMLScript.rtplatform != RUNTIME_PLATFORM.HADOOP
&& Recompiler.checkCPReblock( inst, inputMatrices ) )
{
@@ -310,7 +310,7 @@ public class RunMRJobs
case TRANSFORM:
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ if( ConfigurationManager.isDynamicRecompilation()
&& OptimizerUtils.ALLOW_TRANSFORM_RECOMPILE
&& DMLScript.rtplatform != RUNTIME_PLATFORM.HADOOP
&& Recompiler.checkCPTransform( inst, inputMatrices ) )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/AParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/AParserWrapper.java b/src/main/java/org/apache/sysml/parser/AParserWrapper.java
index 7b7a797..b22e60b 100644
--- a/src/main/java/org/apache/sysml/parser/AParserWrapper.java
+++ b/src/main/java/org/apache/sysml/parser/AParserWrapper.java
@@ -39,12 +39,16 @@ import org.apache.sysml.runtime.util.LocalFileUtils;
*/
public abstract class AParserWrapper
{
- //global parser configuration dml/pydml:
- //1) skip errors on unspecified args (modified by mlcontext / jmlc)
- public static boolean IGNORE_UNSPECIFIED_ARGS = false;
-
-
- public abstract DMLProgram parse(String fileName, String dmlScript, HashMap<String, String> argVals) throws ParseException;
+ /**
+ *
+ * @param fileName
+ * @param dmlScript
+ * @param argVals
+ * @return
+ * @throws ParseException
+ */
+ public abstract DMLProgram parse(String fileName, String dmlScript, HashMap<String, String> argVals)
+ throws ParseException;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/DMLTranslator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
index 133b816..4e69819 100644
--- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java
+++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
@@ -26,9 +26,7 @@ import java.util.Iterator;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.AggBinaryOp;
import org.apache.sysml.hops.AggUnaryOp;
import org.apache.sysml.hops.BinaryOp;
@@ -70,31 +68,16 @@ import org.apache.sysml.runtime.DMLRuntimeException;
public class DMLTranslator
{
-
- public static int DMLBlockSize = 1000;
-
private static final Log LOG = LogFactory.getLog(DMLTranslator.class.getName());
private DMLProgram _dmlProg = null;
-
public DMLTranslator(DMLProgram dmlp)
throws DMLRuntimeException
{
_dmlProg = dmlp;
- //each script sets its own block size, opt level etc
- DMLConfig conf = ConfigurationManager.getConfig();
- DMLTranslator.setDMLBlockSize( conf.getIntValue( DMLConfig.DEFAULT_BLOCK_SIZE ) );
- OptimizerUtils.setOptimizationLevel( conf.getIntValue(DMLConfig.OPTIMIZATION_LEVEL) );
- Recompiler.reinitRecompiler(); //reinit rewriter according to opt level flags
- }
-
- /**
- *
- * @param blocksize
- */
- public static void setDMLBlockSize(int blocksize) {
- DMLBlockSize = blocksize;
+ //reinit rewriter according to opt level flags
+ Recompiler.reinitRecompiler();
}
/**
@@ -1022,7 +1005,7 @@ public class DMLTranslator
case BINARY:
// write output in binary block format
- ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateInPlace(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateInPlace(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
break;
default:
@@ -1097,11 +1080,10 @@ public class DMLTranslator
// preserve data type matrix of any index identifier
// (required for scalar input to left indexing)
- // TODO Doug, please verify this (required for scalar assignments, otherwise TWrite of LIX target becomes scalar as well)
if( target.getDataType() != DataType.MATRIX ) {
target.setDataType(DataType.MATRIX);
target.setValueType(ValueType.DOUBLE);
- target.setBlockDimensions(DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ target.setBlockDimensions(ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
}
Integer statementId = liveOutToTemp.get(target.getName());
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/DataExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DataExpression.java b/src/main/java/org/apache/sysml/parser/DataExpression.java
index 29ffb18..84e7c43 100644
--- a/src/main/java/org/apache/sysml/parser/DataExpression.java
+++ b/src/main/java/org/apache/sysml/parser/DataExpression.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.wink.json4j.JSONArray;
import org.apache.wink.json4j.JSONObject;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.DataGenOp;
import org.apache.sysml.parser.LanguageException.LanguageErrorCodes;
@@ -43,11 +44,6 @@ import org.apache.sysml.utils.JSONHelper;
public class DataExpression extends DataIdentifier
{
- //internal configuration (modified by mlcontext, jmlc apis)
- //no read of meta data on mlcontext (local) /jmlc (global); ignore unknowns on jmlc
- public static boolean IGNORE_READ_WRITE_METADATA = false; // global skip meta data reads
- public static boolean REJECT_READ_WRITE_UNKNOWNS = true; // ignore missing meta data
-
public static final String RAND_ROWS = "rows";
public static final String RAND_COLS = "cols";
public static final String RAND_MIN = "min";
@@ -624,7 +620,8 @@ public class DataExpression extends DataIdentifier
String mtdFileName = getMTDFileName(inputFileName);
// track whether should attempt to read MTD file or not
- boolean shouldReadMTD = _checkMetadata && !IGNORE_READ_WRITE_METADATA;
+ boolean shouldReadMTD = _checkMetadata && !ConfigurationManager
+ .getCompilerConfigFlag(ConfigType.IGNORE_READ_WRITE_METADATA);
// Check for file existence (before metadata parsing for meaningful error messages)
if( shouldReadMTD //skip check for jmlc/mlcontext
@@ -894,7 +891,8 @@ public class DataExpression extends DataIdentifier
// initialize size of target data identifier to UNKNOWN
getOutput().setDimensions(-1, -1);
- if ( !isCSV && REJECT_READ_WRITE_UNKNOWNS //skip check for csv format / jmlc api
+ if ( !isCSV && ConfigurationManager.getCompilerConfig()
+ .getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) //skip check for csv format / jmlc api
&& (getVarParam(READROWPARAM) == null || getVarParam(READCOLPARAM) == null) ) {
raiseValidateError("Missing or incomplete dimension information in read statement: "
+ mtdFileName, conditional, LanguageErrorCodes.INVALID_PARAMETERS);
@@ -906,7 +904,8 @@ public class DataExpression extends DataIdentifier
// these are strings that are long values
Long dim1 = (getVarParam(READROWPARAM) == null) ? null : Long.valueOf( getVarParam(READROWPARAM).toString());
Long dim2 = (getVarParam(READCOLPARAM) == null) ? null : Long.valueOf( getVarParam(READCOLPARAM).toString());
- if ( !isCSV && (dim1 <= 0 || dim2 <= 0) && REJECT_READ_WRITE_UNKNOWNS ) {
+ if ( !isCSV && (dim1 <= 0 || dim2 <= 0) && ConfigurationManager
+ .getCompilerConfig().getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) ) {
raiseValidateError("Invalid dimension information in read statement", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
}
@@ -1065,7 +1064,7 @@ public class DataExpression extends DataIdentifier
if (getVarParam(FORMAT_TYPE) == null || getVarParam(FORMAT_TYPE).toString().equalsIgnoreCase("text"))
getOutput().setBlockDimensions(-1, -1);
else if (getVarParam(FORMAT_TYPE).toString().equalsIgnoreCase("binary"))
- getOutput().setBlockDimensions(DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ getOutput().setBlockDimensions(ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
else if (getVarParam(FORMAT_TYPE).toString().equalsIgnoreCase(FORMAT_TYPE_VALUE_MATRIXMARKET) || (getVarParam(FORMAT_TYPE).toString().equalsIgnoreCase(FORMAT_TYPE_VALUE_CSV)))
getOutput().setBlockDimensions(-1, -1);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/ForStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ForStatementBlock.java b/src/main/java/org/apache/sysml/parser/ForStatementBlock.java
index 2469131..a13db39 100644
--- a/src/main/java/org/apache/sysml/parser/ForStatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/ForStatementBlock.java
@@ -23,9 +23,9 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.HopsException;
-import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.runtime.instructions.cp.BooleanObject;
@@ -420,8 +420,7 @@ public class ForStatementBlock extends StatementBlock
public void updatePredicateRecompilationFlags()
throws HopsException
{
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION )
- {
+ if( ConfigurationManager.isDynamicRecompilation() ) {
_requiresFromRecompile = Recompiler.requiresRecompilation(getFromHops());
_requiresToRecompile = Recompiler.requiresRecompilation(getToHops());
_requiresIncrementRecompile = Recompiler.requiresRecompilation(getIncrementHops());
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/IfStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/IfStatementBlock.java b/src/main/java/org/apache/sysml/parser/IfStatementBlock.java
index 99f5151..c172ffa 100644
--- a/src/main/java/org/apache/sysml/parser/IfStatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/IfStatementBlock.java
@@ -24,9 +24,9 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map.Entry;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.HopsException;
-import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.parser.Expression.DataType;
@@ -528,7 +528,7 @@ public class IfStatementBlock extends StatementBlock
public void updatePredicateRecompilationFlag()
throws HopsException
{
- _requiresPredicateRecompile = OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ _requiresPredicateRecompile = ConfigurationManager.isDynamicRecompilation()
&& Recompiler.requiresRecompilation(getPredicateHops());
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/StatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/StatementBlock.java b/src/main/java/org/apache/sysml/parser/StatementBlock.java
index 8f8de82..ef38989 100644
--- a/src/main/java/org/apache/sysml/parser/StatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/StatementBlock.java
@@ -25,12 +25,11 @@ import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.MLContextProxy;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.HopsException;
-import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.parser.Expression.DataType;
@@ -1052,7 +1051,7 @@ public class StatementBlock extends LiveVariableAnalysis
public void updateRecompilationFlag()
throws HopsException
{
- _requiresRecompile = OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ _requiresRecompile = ConfigurationManager.isDynamicRecompilation()
&& Recompiler.requiresRecompilation(get_hops());
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java b/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java
index f5852ae..866bf1c 100644
--- a/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java
@@ -23,9 +23,9 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.HopsException;
-import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.lops.Lop;
@@ -325,7 +325,7 @@ public class WhileStatementBlock extends StatementBlock
public void updatePredicateRecompilationFlag()
throws HopsException
{
- _requiresPredicateRecompile = OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ _requiresPredicateRecompile = ConfigurationManager.isDynamicRecompilation()
&& Recompiler.requiresRecompilation(getPredicateHops());
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java
index 02c8b75..7381c73 100644
--- a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java
+++ b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java
@@ -30,7 +30,8 @@ import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.TerminalNode;
-import org.apache.sysml.parser.AParserWrapper;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.parser.AssignmentStatement;
import org.apache.sysml.parser.ConditionalPredicate;
import org.apache.sysml.parser.DMLProgram;
@@ -301,7 +302,7 @@ public class DmlSyntacticValidator extends CommonSyntacticValidator implements D
if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) {
String msg = "The parameter " + varName + " either needs to be passed "
+ "through commandline or initialized to default value.";
- if( AParserWrapper.IGNORE_UNSPECIFIED_ARGS ) {
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.IGNORE_UNSPECIFIED_ARGS) ) {
ctx.dataInfo.expr = getConstIdFromString(" ", ctx.start);
raiseWarning(msg, ctx.start);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java
index fd12d53..2da5196 100644
--- a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java
+++ b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java
@@ -31,7 +31,8 @@ import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.apache.commons.lang.StringUtils;
-import org.apache.sysml.parser.AParserWrapper;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.parser.AssignmentStatement;
import org.apache.sysml.parser.BinaryExpression;
import org.apache.sysml.parser.ConditionalPredicate;
@@ -349,7 +350,7 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements
if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) {
String msg = "The parameter " + varName + " either needs to be passed "
+ "through commandline or initialized to default value.";
- if( AParserWrapper.IGNORE_UNSPECIFIED_ARGS ) {
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.IGNORE_UNSPECIFIED_ARGS) ) {
ctx.dataInfo.expr = getConstIdFromString(" ", ctx.start);
raiseWarning(msg, ctx.start);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
index 0ea80d8..b36bd59 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
@@ -28,11 +28,9 @@ import java.util.TreeMap;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.ReBlock;
import org.apache.sysml.lops.compile.JobType;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.DataIdentifier;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
@@ -361,7 +359,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
String reblock = "";
String reblockStr = ""; //Keep a copy of a single MR reblock instruction
- String scratchSpaceLoc = ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
try {
// create a RBLK job that transforms each output matrix from cell to block
@@ -380,14 +378,14 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
reblock += "MR" + ReBlock.OPERAND_DELIMITOR + "rblk" + ReBlock.OPERAND_DELIMITOR +
i + ReBlock.DATATYPE_PREFIX + matrices.get(i).getDataType() + ReBlock.VALUETYPE_PREFIX + matrices.get(i).getValueType() + ReBlock.OPERAND_DELIMITOR +
i + ReBlock.DATATYPE_PREFIX + matrices.get(i).getDataType() + ReBlock.VALUETYPE_PREFIX + matrices.get(i).getValueType() + ReBlock.OPERAND_DELIMITOR +
- DMLTranslator.DMLBlockSize + ReBlock.OPERAND_DELIMITOR + DMLTranslator.DMLBlockSize + ReBlock.OPERAND_DELIMITOR + "true";
+ ConfigurationManager.getBlocksize() + ReBlock.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize() + ReBlock.OPERAND_DELIMITOR + "true";
if(DMLScript.ENABLE_DEBUG_MODE) {
//Create a copy of reblock instruction but as a single instruction (FOR DEBUGGER)
reblockStr = "MR" + ReBlock.OPERAND_DELIMITOR + "rblk" + ReBlock.OPERAND_DELIMITOR +
i + ReBlock.DATATYPE_PREFIX + matrices.get(i).getDataType() + ReBlock.VALUETYPE_PREFIX + matrices.get(i).getValueType() + ReBlock.OPERAND_DELIMITOR +
i + ReBlock.DATATYPE_PREFIX + matrices.get(i).getDataType() + ReBlock.VALUETYPE_PREFIX + matrices.get(i).getValueType() + ReBlock.OPERAND_DELIMITOR +
- DMLTranslator.DMLBlockSize + ReBlock.OPERAND_DELIMITOR + DMLTranslator.DMLBlockSize + ReBlock.OPERAND_DELIMITOR + "true";
+ ConfigurationManager.getBlocksize() + ReBlock.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize() + ReBlock.OPERAND_DELIMITOR + "true";
//Set MR reblock instruction line number (FOR DEBUGGER)
if (!MRJobLineNumbers.containsKey(matrices.get(i).getBeginLine())) {
MRJobLineNumbers.put(matrices.get(i).getBeginLine(), new ArrayList<String>());
@@ -487,7 +485,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
String[] outputs = new String[matrices.size()];
byte[] resultIndex = new byte[matrices.size()];
- String scratchSpaceLoc = ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
try {
@@ -514,7 +512,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
gmrStr = "MR" + Lop.OPERAND_DELIMITOR + "gmr" + Lop.OPERAND_DELIMITOR +
i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR +
i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR +
- DMLTranslator.DMLBlockSize + Lop.OPERAND_DELIMITOR + DMLTranslator.DMLBlockSize;
+ ConfigurationManager.getBlocksize() + Lop.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize();
//Set MR gmr instruction line number (FOR DEBUGGER)
if (!MRJobLineNumbers.containsKey(matrices.get(i).getBeginLine())) {
@@ -579,7 +577,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
for( int i=0; i<matricesNoReblock.size(); i++ )
{
- String scratchSpaceLoc = ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
try{
String filename = scratchSpaceLoc +
@@ -765,7 +763,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
protected MatrixObject createOutputMatrixObject( Matrix m )
throws CacheException
{
- MatrixCharacteristics mc = new MatrixCharacteristics(m.getNumRows(),m.getNumCols(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ MatrixCharacteristics mc = new MatrixCharacteristics(m.getNumRows(),m.getNumCols(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
MatrixFormatMetaData mfmd = new MatrixFormatMetaData(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo);
return new MatrixObject(ValueType.DOUBLE, m.getFilePath(), mfmd);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
index ad0dbcd..3740e83 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
@@ -22,7 +22,7 @@ package org.apache.sysml.runtime.controlprogram;
import java.util.ArrayList;
import java.util.HashMap;
-import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.parser.DataIdentifier;
import org.apache.sysml.parser.ExternalFunctionStatement;
import org.apache.sysml.parser.Expression.ValueType;
@@ -164,7 +164,7 @@ public class ExternalFunctionProgramBlockCP extends ExternalFunctionProgramBlock
if( ret == null ) //otherwise, pass in-memory matrix from extfunct back to invoking program
{
- MatrixCharacteristics mc = new MatrixCharacteristics(m.getNumRows(),m.getNumCols(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ MatrixCharacteristics mc = new MatrixCharacteristics(m.getNumRows(),m.getNumCols(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
MatrixFormatMetaData mfmd = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
ret = new MatrixObject(ValueType.DOUBLE, m.getFilePath(), mfmd);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java
index ff9113f..c28a6cc 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java
@@ -22,7 +22,7 @@ package org.apache.sysml.runtime.controlprogram;
import java.util.ArrayList;
import org.apache.sysml.api.DMLScript;
-import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.parser.DataIdentifier;
import org.apache.sysml.runtime.DMLRuntimeException;
@@ -84,7 +84,7 @@ public class FunctionProgramBlock extends ProgramBlock
{
//dynamically recompile entire function body (according to function inputs)
try {
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ if( ConfigurationManager.isDynamicRecompilation()
&& isRecompileOnce()
&& ParForProgramBlock.RESET_RECOMPILATION_FLAGs )
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
index 5d5b4ca..eb85306 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
@@ -31,9 +31,9 @@ import java.util.List;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Level;
-
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.OptimizerUtils;
@@ -314,7 +314,7 @@ public class ParForProgramBlock extends ForProgramBlock
super(prog, iterPredVars);
//init internal flags according to DML config
- initInternalConfigurations(ConfigurationManager.getConfig());
+ initInternalConfigurations(ConfigurationManager.getDMLConfig());
//ID generation and setting
setParForProgramBlockIDs( ID );
@@ -1404,8 +1404,7 @@ public class ParForProgramBlock extends ForProgramBlock
try
{
- //create deep copies of required elements
- //child blocks
+ //create deep copies of required elements child blocks
ArrayList<ProgramBlock> cpChildBlocks = null;
HashSet<String> fnNames = new HashSet<String>();
if( USE_PB_CACHE )
@@ -1425,12 +1424,15 @@ public class ParForProgramBlock extends ForProgramBlock
cpChildBlocks = ProgramConverter.rcreateDeepCopyProgramBlocks(_childBlocks, pwID, _IDPrefix, new HashSet<String>(), fnNames, false, false);
}
- // Deep copy Execution Context
+ //deep copy execution context
ExecutionContext cpEc = ProgramConverter.createDeepCopyExecutionContext(ec);
+ //copy compiler configuration (for jmlc w/o global config)
+ CompilerConfig cconf = ConfigurationManager.getCompilerConfig();
+
//create the actual parallel worker
ParForBody body = new ParForBody( cpChildBlocks, _resultVars, cpEc );
- pw = new LocalParWorker( pwID, queue, body, MAX_RETRYS_ON_ERROR, _monitor );
+ pw = new LocalParWorker( pwID, queue, body, cconf, MAX_RETRYS_ON_ERROR, _monitor );
pw.setFunctionNames(fnNames);
}
catch(Exception ex)
@@ -1506,7 +1508,7 @@ public class ParForProgramBlock extends ForProgramBlock
DataPartitioner dp = null;
//determine max degree of parallelism
- int numReducers = ConfigurationManager.getConfig().getIntValue(DMLConfig.NUM_REDUCERS);
+ int numReducers = ConfigurationManager.getNumReducers();
int maxNumRed = InfrastructureAnalyzer.getRemoteParallelReduceTasks();
//correction max number of reducers on yarn clusters
if( InfrastructureAnalyzer.isYarnEnabled() )
@@ -1550,7 +1552,7 @@ public class ParForProgramBlock extends ForProgramBlock
ResultMerge rm = null;
//determine degree of parallelism
- int numReducers = ConfigurationManager.getConfig().getIntValue(DMLConfig.NUM_REDUCERS);
+ int numReducers = ConfigurationManager.getNumReducers();
int maxMap = InfrastructureAnalyzer.getRemoteParallelMapTasks();
int maxRed = InfrastructureAnalyzer.getRemoteParallelReduceTasks();
//correction max number of reducers on yarn clusters
@@ -1920,8 +1922,7 @@ public class ParForProgramBlock extends ForProgramBlock
*/
private String constructTaskFileName()
{
- String scratchSpaceLoc = ConfigurationManager.getConfig()
- .getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
StringBuilder sb = new StringBuilder();
sb.append(scratchSpaceLoc);
@@ -1941,8 +1942,7 @@ public class ParForProgramBlock extends ForProgramBlock
*/
private String constructResultFileName()
{
- String scratchSpaceLoc = ConfigurationManager.getConfig()
- .getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
StringBuilder sb = new StringBuilder();
sb.append(scratchSpaceLoc);
@@ -1960,8 +1960,7 @@ public class ParForProgramBlock extends ForProgramBlock
*/
private String constructResultMergeFileName()
{
- String scratchSpaceLoc = ConfigurationManager.getConfig()
- .getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
String fname = PARFOR_MR_RESULTMERGE_FNAME;
fname = fname.replaceAll("%ID%", String.valueOf(_ID)); //replace workerID
@@ -1983,8 +1982,7 @@ public class ParForProgramBlock extends ForProgramBlock
*/
private String constructDataPartitionsFileName()
{
- String scratchSpaceLoc = ConfigurationManager.getConfig()
- .getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
String fname = PARFOR_DATAPARTITIONS_FNAME;
fname = fname.replaceAll("%ID%", String.valueOf(_ID)); //replace workerID
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
index 209b68d..02e3e0c 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
@@ -23,11 +23,10 @@ import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.MLContextProxy;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop;
-import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.parser.StatementBlock;
import org.apache.sysml.parser.Expression.ValueType;
@@ -144,7 +143,7 @@ public class ProgramBlock
DMLAppMasterUtils.setupProgramBlockRemoteMaxMemory(this);
long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ if( ConfigurationManager.isDynamicRecompilation()
&& _sb != null
&& _sb.requiresRecompilation() )
{
@@ -186,7 +185,7 @@ public class ProgramBlock
//dynamically recompile instructions if enabled and required
try {
long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ if( ConfigurationManager.isDynamicRecompilation()
&& requiresRecompile )
{
tmp = Recompiler.recompileHopsDag(hops, ec.getVariables(), null, false, _tid);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
index 57c3da8..4f9d886 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
@@ -23,12 +23,11 @@ import java.io.IOException;
import java.lang.ref.SoftReference;
import org.apache.commons.lang.mutable.MutableBoolean;
-
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.lops.Lop;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
@@ -1500,9 +1499,9 @@ public class MatrixObject extends CacheableData<MatrixBlock>
// when outputFormat is binaryblock, make sure that matrixCharacteristics has correct blocking dimensions
// note: this is only required if singlenode (due to binarycell default)
if ( oinfo == OutputInfo.BinaryBlockOutputInfo && DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE &&
- (mc.getRowsPerBlock() != DMLTranslator.DMLBlockSize || mc.getColsPerBlock() != DMLTranslator.DMLBlockSize) )
+ (mc.getRowsPerBlock() != ConfigurationManager.getBlocksize() || mc.getColsPerBlock() != ConfigurationManager.getBlocksize()) )
{
- DataConverter.writeMatrixToHDFS(_data, filePathAndName, oinfo, new MatrixCharacteristics(mc.getRows(), mc.getCols(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, mc.getNonZeros()), replication, formatProperties);
+ DataConverter.writeMatrixToHDFS(_data, filePathAndName, oinfo, new MatrixCharacteristics(mc.getRows(), mc.getCols(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), mc.getNonZeros()), replication, formatProperties);
}
else {
DataConverter.writeMatrixToHDFS(_data, filePathAndName, oinfo, mc, replication, formatProperties);
@@ -1545,9 +1544,9 @@ public class MatrixObject extends CacheableData<MatrixBlock>
// when outputFormat is binaryblock, make sure that matrixCharacteristics has correct blocking dimensions
// note: this is only required if singlenode (due to binarycell default)
if ( oinfo == OutputInfo.BinaryBlockOutputInfo && DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE &&
- (mc.getRowsPerBlock() != DMLTranslator.DMLBlockSize || mc.getColsPerBlock() != DMLTranslator.DMLBlockSize) )
+ (mc.getRowsPerBlock() != ConfigurationManager.getBlocksize() || mc.getColsPerBlock() != ConfigurationManager.getBlocksize()) )
{
- mc = new MatrixCharacteristics(mc.getRows(), mc.getCols(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, mc.getNonZeros());
+ mc = new MatrixCharacteristics(mc.getRows(), mc.getCols(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), mc.getNonZeros());
}
MapReduceTool.writeMetaDataFile (filePathAndName + ".mtd", valueType, mc, oinfo, formatProperties);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java
index 9f609bb..f666feb 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java
@@ -176,7 +176,7 @@ public class DataPartitionerRemoteMR extends DataPartitioner
job.setInt(MRConfigurationNames.DFS_REPLICATION, _replication);
//set up map/reduce memory configurations (if in AM context)
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
DMLAppMasterUtils.setupMRJobRemoteMaxMemory(job, config);
//set up custom map/reduce configurations
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java
index 1c66c11..e907c46 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java
@@ -21,6 +21,8 @@ package org.apache.sysml.runtime.controlprogram.parfor;
import java.util.Collection;
+import org.apache.sysml.conf.CompilerConfig;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysml.runtime.controlprogram.parfor.stat.Stat;
@@ -41,16 +43,18 @@ public class LocalParWorker extends ParWorker implements Runnable
protected Collection<String> _fnNames = null;
+ protected CompilerConfig _cconf = null;
protected boolean _stopped = false;
protected int _max_retry = -1;
- public LocalParWorker( long ID, LocalTaskQueue<Task> q, ParForBody body, int max_retry, boolean monitor )
+ public LocalParWorker( long ID, LocalTaskQueue<Task> q, ParForBody body, CompilerConfig cconf, int max_retry, boolean monitor )
{
super(ID, body, monitor);
_taskQueue = q;
- _stopped = false;
+ _cconf = cconf;
+ _stopped = false;
_max_retry = max_retry;
}
@@ -82,6 +86,9 @@ public class LocalParWorker extends ParWorker implements Runnable
sec.setThreadLocalSchedulerPool("parforPool"+_workerID);
}
+ //setup compiler config for worker thread
+ ConfigurationManager.setLocalConfig(_cconf);
+
// continuous execution (execute tasks until (1) stopped or (2) no more tasks)
Task lTask = null;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
index 004e980..a684c08 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
@@ -29,12 +29,11 @@ import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapred.JobConf;
-
import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.Hop;
-import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.parser.DMLProgram;
import org.apache.sysml.parser.DataIdentifier;
@@ -625,7 +624,7 @@ public class ProgramConverter
try
{
- if( OptimizerUtils.ALLOW_PARALLEL_DYN_RECOMPILATION
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.ALLOW_PARALLEL_DYN_RECOMPILATION)
&& sb != null //forced deep copy for function recompilation
&& (Recompiler.requiresRecompilation( sb.get_hops() ) || forceDeepCopy) )
{
@@ -673,7 +672,7 @@ public class ProgramConverter
try
{
- if( OptimizerUtils.ALLOW_PARALLEL_DYN_RECOMPILATION
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.ALLOW_PARALLEL_DYN_RECOMPILATION)
&& sb != null //forced deep copy for function recompile
&& (Recompiler.requiresRecompilation( sb.getPredicateHops() ) || forceDeepCopy) )
{
@@ -722,7 +721,7 @@ public class ProgramConverter
try
{
- if( OptimizerUtils.ALLOW_PARALLEL_DYN_RECOMPILATION
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.ALLOW_PARALLEL_DYN_RECOMPILATION)
&& sb != null //forced deep copy for function recompile
&& (Recompiler.requiresRecompilation( sb.getPredicateHops() ) || forceDeepCopy) )
{
@@ -771,7 +770,7 @@ public class ProgramConverter
try
{
- if( OptimizerUtils.ALLOW_PARALLEL_DYN_RECOMPILATION
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.ALLOW_PARALLEL_DYN_RECOMPILATION)
&& sb != null
&& ( Recompiler.requiresRecompilation(sb.getFromHops()) ||
Recompiler.requiresRecompilation(sb.getToHops()) ||
@@ -852,7 +851,7 @@ public class ProgramConverter
sb.append( NEWLINE );
//handle DML config
- sb.append( ConfigurationManager.getConfig().serializeDMLConfig() );
+ sb.append( ConfigurationManager.getDMLConfig().serializeDMLConfig() );
sb.append( COMPONENTS_DELIM );
sb.append( NEWLINE );
@@ -1558,7 +1557,7 @@ public class ProgramConverter
JobConf job = ConfigurationManager.getCachedJobConf();
if( !InfrastructureAnalyzer.isLocalMode(job) ) {
DMLConfig config = DMLConfig.parseDMLConfig(confStr);
- ConfigurationManager.setConfig(config);
+ ConfigurationManager.setLocalConfig(config);
ParForProgramBlock.initInternalConfigurations(config);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java
index 2cbfd36..15b1fb3 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java
@@ -160,7 +160,7 @@ public class RemoteDPParForMR
MRJobConfiguration.addBinaryBlockSerializationFramework( job );
//set up map/reduce memory configurations (if in AM context)
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
DMLAppMasterUtils.setupMRJobRemoteMaxMemory(job, config);
//set up custom map/reduce configurations
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java
index 94be120..7710fe8 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java
@@ -164,7 +164,7 @@ public class RemoteParForMR
job.setMapSpeculativeExecution(false);
//set up map/reduce memory configurations (if in AM context)
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
DMLAppMasterUtils.setupMRJobRemoteMaxMemory(job, config);
//set up custom map/reduce configurations
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
index 356ae3c..185acbb 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
@@ -294,7 +294,7 @@ public class ResultMergeRemoteMR extends ResultMerge
MRJobConfiguration.addBinaryBlockSerializationFramework( job );
//set up custom map/reduce configurations
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
MRJobConfiguration.setupCustomMRConfigurations(job, config);
//enables the reuse of JVMs (multiple tasks per MR task)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
index 360bbd2..edae6a1 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
@@ -29,8 +29,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-
import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.ipa.InterProceduralAnalysis;
import org.apache.sysml.hops.rewrite.HopRewriteRule;
@@ -228,7 +228,7 @@ public class OptimizationWrapper
OptTree tree = null;
//recompile parfor body
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION )
+ if( ConfigurationManager.isDynamicRecompilation() )
{
ForStatement fs = (ForStatement) sb.getStatement(0);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
index a6b7763..a2efb93 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
@@ -22,6 +22,7 @@ package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.util.HashMap;
import java.util.HashSet;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.lops.LopProperties;
import org.apache.sysml.parser.ParForStatementBlock;
@@ -265,7 +266,7 @@ public class OptimizerConstrained extends OptimizerRuleBased
boolean ret = false;
// constraint awareness
- if( n.getExecType() != null && OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR )
+ if( n.getExecType() != null && ConfigurationManager.isParallelParFor() )
{
ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter
.getAbstractPlanMapping().getMappedProg(n.getID())[1];
@@ -298,7 +299,7 @@ public class OptimizerConstrained extends OptimizerRuleBased
throws DMLRuntimeException
{
// constraint awareness
- if( n.getK() > 0 && OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR )
+ if( n.getK() > 0 && ConfigurationManager.isParallelParFor() )
{
ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter
.getAbstractPlanMapping().getMappedProg(n.getID())[1];
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index 4e871a1..98fb6bb 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -963,7 +963,7 @@ public class OptimizerRuleBased extends Optimizer
PDataPartitioner REMOTE_DP = OptimizerUtils.isSparkExecutionMode() ? PDataPartitioner.REMOTE_SPARK : PDataPartitioner.REMOTE_MR;
//deciding on the execution strategy
- if( OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR //allowed remote parfor execution
+ if( ConfigurationManager.isParallelParFor() //allowed remote parfor execution
&& ( (isCPOnly && M <= _rm ) //Required: all instruction can be be executed in CP
||(isCPOnlyPossible && M2 <= _rm)) ) //Required: cp inst fit into remote JVM mem
{
@@ -1436,7 +1436,7 @@ public class OptimizerRuleBased extends Optimizer
if( type == ExecType.CP )
{
//determine local max parallelism constraint
- int kMax = OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR ?
+ int kMax = ConfigurationManager.isParallelParFor() ?
(n.isCPOnly() ? _lkmaxCP : _lkmaxMR) : 1;
//ensure local memory constraint (for spark more conservative in order to
@@ -1535,7 +1535,7 @@ public class OptimizerRuleBased extends Optimizer
{
//set degree of parallelism for multi-threaded leaf nodes
Hop h = OptTreeConverter.getAbstractPlanMapping().getMappedHop(c.getID());
- if( OptimizerUtils.PARALLEL_CP_MATRIX_OPERATIONS
+ if( ConfigurationManager.isParallelMatrixOperations()
&& h instanceof MultiThreadedHop //abop, datagenop, qop, paramop
&& !( h instanceof ParameterizedBuiltinOp //only paramop-grpagg
&& ((ParameterizedBuiltinOp)h).getOp()!=ParamBuiltinOp.GROUPEDAGG) )
@@ -1583,7 +1583,7 @@ public class OptimizerRuleBased extends Optimizer
if( flagNested && flagLIX )
LOG.warn(getOptMode()+" OPT: Task partitioner decision has conflicting input from rewrites 'nested parallelism' and 'result partitioning'.");
- boolean jvmreuse = ConfigurationManager.getConfig().getBooleanValue(DMLConfig.JVM_REUSE);
+ boolean jvmreuse = ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.JVM_REUSE);
//set task partitioner
if( flagNested )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java
index c88f657..5289ce2 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java
@@ -49,10 +49,10 @@ import au.com.bytecode.opencsv.CSVWriter;
import org.apache.sysml.api.DMLException;
import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.MMTSJ.MMTSJType;
import org.apache.sysml.parser.DMLProgram;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.DataIdentifier;
import org.apache.sysml.parser.ExternalFunctionStatement;
import org.apache.sysml.parser.ParseException;
@@ -1118,7 +1118,7 @@ public class PerfTestTool
MapReduceTool.deleteFileIfExistOnHDFS(fname);
- MatrixCharacteristics mc = new MatrixCharacteristics(dim, dim, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ MatrixCharacteristics mc = new MatrixCharacteristics(dim, dim, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
MatrixFormatMetaData md = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
MatrixObject mo = new MatrixObject(ValueType.DOUBLE,fname,md);
mo.acquireModify(mb);
@@ -1170,7 +1170,7 @@ public class PerfTestTool
MapReduceTool.deleteFileIfExistOnHDFS(fname);
- MatrixCharacteristics mc = new MatrixCharacteristics(d1, d2, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ MatrixCharacteristics mc = new MatrixCharacteristics(d1, d2, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
MatrixFormatMetaData md = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
MatrixObject mo = new MatrixObject(ValueType.DOUBLE,fname,md);
mo.acquireModify(mb);
@@ -1205,8 +1205,8 @@ public class PerfTestTool
mb = new MatrixBlock(dim,dim,true);
break;
}*/
-
- MatrixCharacteristics mc = new MatrixCharacteristics(dim, dim, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+
+ MatrixCharacteristics mc = new MatrixCharacteristics(dim, dim, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
MatrixFormatMetaData md = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
MatrixObject mo = new MatrixObject(ValueType.DOUBLE,fname,md);
@@ -1241,7 +1241,7 @@ public class PerfTestTool
break;
}*/
- MatrixCharacteristics mc = new MatrixCharacteristics(d1, d2, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ MatrixCharacteristics mc = new MatrixCharacteristics(d1, d2, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
MatrixFormatMetaData md = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
MatrixObject mo = new MatrixObject(ValueType.DOUBLE,fname,md);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
index 72b8c71..a914c0b 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
@@ -82,7 +82,7 @@ public class ProgramRecompiler
throws LopsException, DMLRuntimeException, DMLUnsupportedOperationException, IOException, HopsException
{
ArrayList<ProgramBlock> ret = new ArrayList<ProgramBlock>();
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
//construct lops from hops if not existing
DMLTranslator dmlt = new DMLTranslator(sbs.get(0).getDMLProg());
@@ -599,7 +599,7 @@ public class ProgramRecompiler
}
//construct new instructions
- ArrayList<Instruction> newInst = dag.getJobs(sbOld, ConfigurationManager.getConfig());
+ ArrayList<Instruction> newInst = dag.getJobs(sbOld, ConfigurationManager.getDMLConfig());
//exchange instructions
@@ -654,7 +654,7 @@ public class ProgramRecompiler
}
//construct new instructions
- ArrayList<Instruction> newInst = dag.getJobs(sbOld, ConfigurationManager.getConfig());
+ ArrayList<Instruction> newInst = dag.getJobs(sbOld, ConfigurationManager.getDMLConfig());
//exchange instructions
pbNew = new ProgramBlock(pbOld.getProgram());
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
index e598e34..ba511c7 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
@@ -326,14 +325,14 @@ public class ParameterizedBuiltinCPFileInstruction extends ParameterizedBuiltinC
if( buffer.size() > StagingFileUtils.CELL_BUFFER_SIZE )
{
- appendCellBufferToStagingArea(stagingDir, buffer, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ appendCellBufferToStagingArea(stagingDir, buffer, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
buffer.clear();
}
}
if( !buffer.isEmpty() )
{
- appendCellBufferToStagingArea(stagingDir, buffer, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ appendCellBufferToStagingArea(stagingDir, buffer, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
buffer.clear();
}
}
@@ -383,14 +382,14 @@ public class ParameterizedBuiltinCPFileInstruction extends ParameterizedBuiltinC
if( buffer.size() > StagingFileUtils.CELL_BUFFER_SIZE )
{
- appendCellBufferToStagingArea(stagingDir, buffer, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ appendCellBufferToStagingArea(stagingDir, buffer, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
buffer.clear();
}
}
if( !buffer.isEmpty() )
{
- appendCellBufferToStagingArea(stagingDir, buffer, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ appendCellBufferToStagingArea(stagingDir, buffer, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
buffer.clear();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/io/MatrixReaderFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/MatrixReaderFactory.java b/src/main/java/org/apache/sysml/runtime/io/MatrixReaderFactory.java
index d1065f8..8da8fd7 100644
--- a/src/main/java/org/apache/sysml/runtime/io/MatrixReaderFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/io/MatrixReaderFactory.java
@@ -19,7 +19,8 @@
package org.apache.sysml.runtime.io;
-import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.InputInfo;
@@ -43,14 +44,14 @@ public class MatrixReaderFactory
if( iinfo == InputInfo.TextCellInputInfo || iinfo == InputInfo.MatrixMarketInputInfo )
{
- if( OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_TEXTFORMATS) )
reader = new ReaderTextCellParallel( iinfo );
else
reader = new ReaderTextCell( iinfo );
}
else if( iinfo == InputInfo.CSVInputInfo )
{
- if( OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_TEXTFORMATS) )
reader = new ReaderTextCSVParallel(new CSVFileFormatProperties());
else
reader = new ReaderTextCSV(new CSVFileFormatProperties());
@@ -58,7 +59,7 @@ public class MatrixReaderFactory
else if( iinfo == InputInfo.BinaryCellInputInfo )
reader = new ReaderBinaryCell();
else if( iinfo == InputInfo.BinaryBlockInputInfo ) {
- if( OptimizerUtils.PARALLEL_CP_READ_BINARYFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_BINARYFORMATS) )
reader = new ReaderBinaryBlockParallel( false );
else
reader = new ReaderBinaryBlock( false );
@@ -88,13 +89,13 @@ public class MatrixReaderFactory
InputInfo iinfo = props.inputInfo;
if( iinfo == InputInfo.TextCellInputInfo || iinfo == InputInfo.MatrixMarketInputInfo ) {
- if( OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_TEXTFORMATS) )
reader = new ReaderTextCellParallel( iinfo );
else
reader = new ReaderTextCell( iinfo );
}
else if( iinfo == InputInfo.CSVInputInfo ) {
- if( OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_TEXTFORMATS) )
reader = new ReaderTextCSVParallel( props.formatProperties!=null ? (CSVFileFormatProperties)props.formatProperties : new CSVFileFormatProperties());
else
reader = new ReaderTextCSV( props.formatProperties!=null ? (CSVFileFormatProperties)props.formatProperties : new CSVFileFormatProperties());
@@ -102,7 +103,7 @@ public class MatrixReaderFactory
else if( iinfo == InputInfo.BinaryCellInputInfo )
reader = new ReaderBinaryCell();
else if( iinfo == InputInfo.BinaryBlockInputInfo ) {
- if( OptimizerUtils.PARALLEL_CP_READ_BINARYFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_BINARYFORMATS) )
reader = new ReaderBinaryBlockParallel( props.localFS );
else
reader = new ReaderBinaryBlock( props.localFS );
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/io/MatrixWriterFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/MatrixWriterFactory.java b/src/main/java/org/apache/sysml/runtime/io/MatrixWriterFactory.java
index 85ff514..fa56fef 100644
--- a/src/main/java/org/apache/sysml/runtime/io/MatrixWriterFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/io/MatrixWriterFactory.java
@@ -19,7 +19,8 @@
package org.apache.sysml.runtime.io;
-import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
@@ -58,7 +59,7 @@ public class MatrixWriterFactory
MatrixWriter writer = null;
if( oinfo == OutputInfo.TextCellOutputInfo ) {
- if( OptimizerUtils.PARALLEL_CP_WRITE_TEXTFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS) )
writer = new WriterTextCellParallel();
else
writer = new WriterTextCell();
@@ -75,7 +76,7 @@ public class MatrixWriterFactory
else if( oinfo == OutputInfo.CSVOutputInfo ) {
if( props!=null && !(props instanceof CSVFileFormatProperties) )
throw new DMLRuntimeException("Wrong type of file format properties for CSV writer.");
- if( OptimizerUtils.PARALLEL_CP_WRITE_TEXTFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS) )
writer = new WriterTextCSVParallel((CSVFileFormatProperties)props);
else
writer = new WriterTextCSV((CSVFileFormatProperties)props);
@@ -84,7 +85,7 @@ public class MatrixWriterFactory
writer = new WriterBinaryCell();
}
else if( oinfo == OutputInfo.BinaryBlockOutputInfo ) {
- if( OptimizerUtils.PARALLEL_CP_WRITE_BINARYFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_WRITE_BINARYFORMATS) )
writer = new WriterBinaryBlockParallel(replication);
else
writer = new WriterBinaryBlock(replication);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/matrix/CMCOVMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/CMCOVMR.java b/src/main/java/org/apache/sysml/runtime/matrix/CMCOVMR.java
index 1747214..06f4054 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/CMCOVMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/CMCOVMR.java
@@ -90,7 +90,7 @@ public class CMCOVMR
job.setInt(MRConfigurationNames.DFS_REPLICATION, replication);
//set up custom map/reduce configurations
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
MRJobConfiguration.setupCustomMRConfigurations(job, config);
//set up what matrices are needed to pass from the mapper to reducer
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java b/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java
index 68611bb..0cae638 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java
@@ -316,7 +316,7 @@ public class CSVReblockMR
job.setInt(MRConfigurationNames.DFS_REPLICATION, replication);
//set up custom map/reduce configurations
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
MRJobConfiguration.setupCustomMRConfigurations(job, config);
//set up the number of reducers
@@ -412,7 +412,7 @@ public class CSVReblockMR
MRJobConfiguration.addBinaryBlockSerializationFramework( job );
//set up custom map/reduce configurations
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
MRJobConfiguration.setupCustomMRConfigurations(job, config);
//set up what matrices are needed to pass from the mapper to reducer
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java b/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
index 99d561f..e48c868 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
@@ -332,7 +332,7 @@ public class CombineMR
job.setInt(MRConfigurationNames.DFS_REPLICATION, replication);
//set up custom map/reduce configurations
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
MRJobConfiguration.setupCustomMRConfigurations(job, config);
//set up what matrices are needed to pass from the mapper to reducer
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java b/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java
index 4cf2472..974c7df 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java
@@ -297,7 +297,7 @@ public class DataGenMR
job.setInt(MRConfigurationNames.DFS_REPLICATION, replication);
//set up map/reduce memory configurations (if in AM context)
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
DMLAppMasterUtils.setupMRJobRemoteMaxMemory(job, config);
//set up custom map/reduce configurations
[2/4] incubator-systemml git commit: [SYSTEMML-584] New thread-local
configuration handling (dml/compiler)
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/matrix/GMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/GMR.java b/src/main/java/org/apache/sysml/runtime/matrix/GMR.java
index d1fdb87..0d399fa 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/GMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/GMR.java
@@ -212,7 +212,7 @@ public class GMR
MRJobConfiguration.addBinaryBlockSerializationFramework( job );
//set up map/reduce memory configurations (if in AM context)
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
DMLAppMasterUtils.setupMRJobRemoteMaxMemory(job, config);
//set up custom map/reduce configurations
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/matrix/GroupedAggMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/GroupedAggMR.java b/src/main/java/org/apache/sysml/runtime/matrix/GroupedAggMR.java
index 5a65ab0..65aff10 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/GroupedAggMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/GroupedAggMR.java
@@ -96,7 +96,7 @@ public class GroupedAggMR
job.setInt(MRConfigurationNames.DFS_REPLICATION, replication);
//set up custom map/reduce configurations
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
MRJobConfiguration.setupCustomMRConfigurations(job, config);
//set up what matrices are needed to pass from the mapper to reducer
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/matrix/MMCJMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MMCJMR.java b/src/main/java/org/apache/sysml/runtime/matrix/MMCJMR.java
index 44513a6..eb4ae4c 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/MMCJMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/MMCJMR.java
@@ -210,7 +210,7 @@ public class MMCJMR
MRJobConfiguration.addBinaryBlockSerializationFramework( job );
//set up map/reduce memory configurations (if in AM context)
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
DMLAppMasterUtils.setupMRJobRemoteMaxMemory(job, config);
//set up custom map/reduce configurations
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/matrix/MMRJMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MMRJMR.java b/src/main/java/org/apache/sysml/runtime/matrix/MMRJMR.java
index 2d120bc..54df00a 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/MMRJMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/MMRJMR.java
@@ -119,7 +119,7 @@ public class MMRJMR
job.setInt(MRConfigurationNames.DFS_REPLICATION, replication);
//set up map/reduce memory configurations (if in AM context)
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
DMLAppMasterUtils.setupMRJobRemoteMaxMemory(job, config);
//set up custom map/reduce configurations
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/matrix/ReblockMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/ReblockMR.java b/src/main/java/org/apache/sysml/runtime/matrix/ReblockMR.java
index 8dd930d..b60d3d6 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/ReblockMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/ReblockMR.java
@@ -114,7 +114,7 @@ public class ReblockMR
MRJobConfiguration.addBinaryBlockSerializationFramework( job );
//set up custom map/reduce configurations
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
MRJobConfiguration.setupCustomMRConfigurations(job, config);
//enable jvm reuse (based on SystemML configuration)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java b/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java
index 8f56336..c5d7a5b 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java
@@ -256,7 +256,7 @@ public class SortMR
job.setInt(MRConfigurationNames.DFS_REPLICATION, replication);
//set up custom map/reduce configurations
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
MRJobConfiguration.setupCustomMRConfigurations(job, config);
MatrixCharacteristics[] s = new MatrixCharacteristics[1];
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/matrix/WriteCSVMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/WriteCSVMR.java b/src/main/java/org/apache/sysml/runtime/matrix/WriteCSVMR.java
index 33939f8..87c25b9 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/WriteCSVMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/WriteCSVMR.java
@@ -84,7 +84,7 @@ public class WriteCSVMR
MRJobConfiguration.addBinaryBlockSerializationFramework( job );
//set up custom map/reduce configurations
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
MRJobConfiguration.setupCustomMRConfigurations(job, config);
long maxRlen=0;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
index 49fb9b0..993adfd 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java
@@ -34,13 +34,11 @@ import java.util.Iterator;
import org.apache.commons.math3.random.Well1024a;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.Hop.OpOp2;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.lops.MMTSJ.MMTSJType;
import org.apache.sysml.lops.MapMultChain.ChainType;
import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.DMLUnsupportedOperationException;
import org.apache.sysml.runtime.functionobjects.Builtin;
@@ -5856,11 +5854,8 @@ public class MatrixBlock extends MatrixValue implements Externalizable
public static MatrixBlock randOperations(int rows, int cols, double sparsity, double min, double max, String pdf, long seed, int k)
throws DMLRuntimeException
{
- DMLConfig conf = ConfigurationManager.getConfig();
- int blocksize = (conf!=null) ? ConfigurationManager.getConfig().getIntValue(DMLConfig.DEFAULT_BLOCK_SIZE)
- : DMLTranslator.DMLBlockSize;
-
- RandomMatrixGenerator rgen = new RandomMatrixGenerator(pdf, rows, cols, blocksize, blocksize, sparsity, min, max);
+ RandomMatrixGenerator rgen = new RandomMatrixGenerator(pdf, rows, cols,
+ ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), sparsity, min, max);
if (k > 1)
return randOperations(rgen, seed, k);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java
index 06ae844..503e3e4 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MRJobConfiguration.java
@@ -1360,7 +1360,7 @@ public class MRJobConfiguration
//correction max number of reducers on yarn clusters
if( InfrastructureAnalyzer.isYarnEnabled() )
n = (int)Math.max( n, YarnClusterAnalyzer.getNumCores()/2 );
- n=Math.min(n, ConfigurationManager.getConfig().getIntValue(DMLConfig.NUM_REDUCERS));
+ n=Math.min(n, ConfigurationManager.getNumReducers());
n=Math.min(n, numFromCompiler);
if(numReducerGroups>0)
n=(int) Math.min(n, numReducerGroups);
@@ -1876,7 +1876,7 @@ public class MRJobConfiguration
public static String constructTempOutputFilename()
{
StringBuilder sb = new StringBuilder();
- sb.append(ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE));
+ sb.append(ConfigurationManager.getScratchSpace());
sb.append(Lop.FILE_SEPARATOR);
sb.append(Lop.PROCESS_PREFIX);
sb.append(DMLScript.getUUID());
@@ -1893,7 +1893,7 @@ public class MRJobConfiguration
private static String constructPartitionFilename()
{
StringBuilder sb = new StringBuilder();
- sb.append(ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE));
+ sb.append(ConfigurationManager.getScratchSpace());
sb.append(Lop.FILE_SEPARATOR);
sb.append(Lop.PROCESS_PREFIX);
sb.append(DMLScript.getUUID());
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/transform/ApplyTfBBMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/ApplyTfBBMR.java b/src/main/java/org/apache/sysml/runtime/transform/ApplyTfBBMR.java
index 043c5c3..30bf1ff 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/ApplyTfBBMR.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/ApplyTfBBMR.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.runtime.instructions.InstructionParser;
import org.apache.sysml.runtime.instructions.mr.CSVReblockInstruction;
import org.apache.sysml.runtime.matrix.CSVReblockMR;
@@ -100,7 +99,7 @@ public class ApplyTfBBMR {
null, rblkInst, null, null, null, resultIndexes, mapoutputIndexes, false);
//set up the number of reducers
- int numRed = WriteCSVMR.determineNumReducers(rlens, clens, ConfigurationManager.getConfig().getIntValue(DMLConfig.NUM_REDUCERS), ret.numReducerGroups);
+ int numRed = WriteCSVMR.determineNumReducers(rlens, clens, ConfigurationManager.getNumReducers(), ret.numReducerGroups);
job.setNumReduceTasks( numRed );
//set up the multiple output files, and their format information
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java b/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java
index 44e9141..af74b8c 100644
--- a/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java
+++ b/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java
@@ -796,7 +796,7 @@ public class DataTransform
if(isBB)
{
- DMLConfig conf = ConfigurationManager.getConfig();
+ DMLConfig conf = ConfigurationManager.getDMLConfig();
int blockSize = conf.getIntValue(DMLConfig.DEFAULT_BLOCK_SIZE);
CSVReblockInstruction rblk = prepDummyReblockInstruction(oprnds.inputCSVProperties, blockSize);
@@ -835,7 +835,7 @@ public class DataTransform
if (isCSV)
{
- DMLConfig conf = ConfigurationManager.getConfig();
+ DMLConfig conf = ConfigurationManager.getDMLConfig();
int blockSize = conf.getIntValue(DMLConfig.DEFAULT_BLOCK_SIZE);
CSVReblockInstruction rblk = prepDummyReblockInstruction(oprnds.inputCSVProperties, blockSize);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java b/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java
index b70ab35..1511d81 100644
--- a/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java
+++ b/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java
@@ -367,7 +367,7 @@ public class LocalFileUtils
{
//create local tmp dir if not existing
String dirRoot = null;
- DMLConfig conf = ConfigurationManager.getConfig();
+ DMLConfig conf = ConfigurationManager.getDMLConfig();
if( conf != null )
dirRoot = conf.getTextValue(DMLConfig.LOCAL_TMP_DIR);
else
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/udf/Matrix.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/Matrix.java b/src/main/java/org/apache/sysml/udf/Matrix.java
index 6d710e4..6aa733c 100644
--- a/src/main/java/org/apache/sysml/udf/Matrix.java
+++ b/src/main/java/org/apache/sysml/udf/Matrix.java
@@ -21,7 +21,7 @@ package org.apache.sysml.udf;
import java.io.IOException;
-import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.parser.Expression;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlockCP;
@@ -216,8 +216,8 @@ public class Matrix extends FunctionParameter
_rows = mb.getNumRows();
_cols = mb.getNumColumns();
long nnz = mb.getNonZeros();
- int rblen = DMLTranslator.DMLBlockSize;
- int cblen = DMLTranslator.DMLBlockSize;
+ int rblen = ConfigurationManager.getBlocksize();
+ int cblen = ConfigurationManager.getBlocksize();
MatrixCharacteristics mc = new MatrixCharacteristics(_rows, _cols, rblen, cblen, nnz);
MatrixFormatMetaData mfmd = new MatrixFormatMetaData(mc, oinfo, iinfo);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixCP.java b/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixCP.java
index 80dc3ee..4147665 100644
--- a/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixCP.java
+++ b/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixCP.java
@@ -19,7 +19,7 @@
package org.apache.sysml.udf.lib;
-import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
@@ -66,7 +66,7 @@ public class DynamicReadMatrixCP extends PackageFunction
InputInfo ii = InputInfo.stringToInputInfo(format);
OutputInfo oi = OutputInfo.BinaryBlockOutputInfo;
- MatrixBlock mbTmp = DataConverter.readMatrixFromHDFS(fname, ii, m, n, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ MatrixBlock mbTmp = DataConverter.readMatrixFromHDFS(fname, ii, m, n, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
String fnameTmp = createOutputFilePathAndName("TMP");
_ret = new Matrix(fnameTmp, m, n, ValueType.Double);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixRcCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixRcCP.java b/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixRcCP.java
index f4faf59..e96ee52 100644
--- a/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixRcCP.java
+++ b/src/main/java/org/apache/sysml/udf/lib/DynamicReadMatrixRcCP.java
@@ -19,7 +19,7 @@
package org.apache.sysml.udf.lib;
-import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
@@ -78,7 +78,7 @@ public class DynamicReadMatrixRcCP extends PackageFunction
String fnameTmp = createOutputFilePathAndName("TMP");
_ret = new Matrix(fnameTmp, m, n, ValueType.Double);
- MatrixBlock mbTmp = DataConverter.readMatrixFromHDFS(fname, ii, m, n, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ MatrixBlock mbTmp = DataConverter.readMatrixFromHDFS(fname, ii, m, n, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
_ret.setMatrixDoubleArray(mbTmp, oi, ii);
_rc = new Scalar(ScalarValueType.Integer, "0");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/yarn/DMLAppMaster.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/DMLAppMaster.java b/src/main/java/org/apache/sysml/yarn/DMLAppMaster.java
index 0c5ba81..0f10e9e 100644
--- a/src/main/java/org/apache/sysml/yarn/DMLAppMaster.java
+++ b/src/main/java/org/apache/sysml/yarn/DMLAppMaster.java
@@ -140,7 +140,7 @@ public class DMLAppMaster
private void writeMessageToHDFSWorkingDir(String msg)
{
//construct working directory (consistent with client)
- DMLConfig conf = ConfigurationManager.getConfig();
+ DMLConfig conf = ConfigurationManager.getDMLConfig();
String hdfsWD = DMLAppMasterUtils.constructHDFSWorkingDir(conf, _appId);
Path msgPath = new Path(hdfsWD, DMLYarnClient.DML_STOPMSG_NAME);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/yarn/DMLAppMasterUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/DMLAppMasterUtils.java b/src/main/java/org/apache/sysml/yarn/DMLAppMasterUtils.java
index 084de46..c4de6b7 100644
--- a/src/main/java/org/apache/sysml/yarn/DMLAppMasterUtils.java
+++ b/src/main/java/org/apache/sysml/yarn/DMLAppMasterUtils.java
@@ -26,6 +26,8 @@ import java.util.HashMap;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.CompilerConfig;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.HopsException;
import org.apache.sysml.hops.OptimizerUtils;
@@ -77,8 +79,9 @@ public class DMLAppMasterUtils
if( DMLScript.isActiveAM() ){
//set optimization level (for awareness of resource optimization)
- OptimizerUtils.setOptimizationLevel( conf.getIntValue(DMLConfig.OPTIMIZATION_LEVEL) );
-
+ CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(conf);
+ ConfigurationManager.setGlobalConfig(cconf);
+
if( isResourceOptimizerEnabled() )
{
//handle optimized memory (mr memory budget per program block)
@@ -144,7 +147,7 @@ public class DMLAppMasterUtils
long mem = _rcMap.get(pb);
InfrastructureAnalyzer.setRemoteMaxMemoryMap(mem);
InfrastructureAnalyzer.setRemoteMaxMemoryReduce(mem);
- OptimizerUtils.setDefaultSize();
+ OptimizerUtils.resetDefaultSize();
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java
index 0aaec58..4b31af2 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java
@@ -220,7 +220,7 @@ public class ResourceOptimizer
InfrastructureAnalyzer.setLocalMaxMemory( (long)cp );
InfrastructureAnalyzer.setRemoteMaxMemoryMap( (long)mr );
InfrastructureAnalyzer.setRemoteMaxMemoryReduce( (long)mr );
- OptimizerUtils.setDefaultSize(); //dependent on cp, mr
+ OptimizerUtils.resetDefaultSize(); //dependent on cp, mr
}
for( ProgramBlock pb : prog )
@@ -349,7 +349,7 @@ public class ResourceOptimizer
InfrastructureAnalyzer.setLocalMaxMemory( cp );
InfrastructureAnalyzer.setRemoteMaxMemoryMap( mr );
InfrastructureAnalyzer.setRemoteMaxMemoryReduce( mr );
- OptimizerUtils.setDefaultSize(); //dependent on cp, mr
+ OptimizerUtils.resetDefaultSize(); //dependent on cp, mr
//recompile instructions (incl predicates)
if (pb instanceof WhileProgramBlock)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/yarn/ropt/YarnOptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/yarn/ropt/YarnOptimizerUtils.java b/src/main/java/org/apache/sysml/yarn/ropt/YarnOptimizerUtils.java
index 610a60e..b9b45f7 100644
--- a/src/main/java/org/apache/sysml/yarn/ropt/YarnOptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/yarn/ropt/YarnOptimizerUtils.java
@@ -20,7 +20,6 @@
package org.apache.sysml.yarn.ropt;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.OptimizerUtils;
public class YarnOptimizerUtils
@@ -73,7 +72,7 @@ public class YarnOptimizerUtils
*/
public static int getNumReducers(boolean configOnly, long jobLookupId)
{
- int ret = ConfigurationManager.getConfig().getIntValue(DMLConfig.NUM_REDUCERS);
+ int ret = ConfigurationManager.getNumReducers();
if( !configOnly )
ret = Math.min(ret,YarnClusterAnalyzer.getRemoteParallelReduceTasks(jobLookupId));
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
index 076e5d0..f044774 100644
--- a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
+++ b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
@@ -44,7 +44,6 @@ import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
import org.apache.sysml.api.MLContext;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.OptimizerUtils;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.DataExpression;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
@@ -501,13 +500,13 @@ public abstract class AutomatedTestBase
protected double[][] writeInputMatrixWithMTD(String name, double[][] matrix, boolean bIncludeR)
{
- MatrixCharacteristics mc = new MatrixCharacteristics(matrix.length, matrix[0].length, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, -1);
+ MatrixCharacteristics mc = new MatrixCharacteristics(matrix.length, matrix[0].length, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE, -1);
return writeInputMatrixWithMTD(name, matrix, bIncludeR, mc);
}
protected double[][] writeInputMatrixWithMTD(String name, double[][] matrix, int nnz, boolean bIncludeR)
{
- MatrixCharacteristics mc = new MatrixCharacteristics(matrix.length, matrix[0].length, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, nnz);
+ MatrixCharacteristics mc = new MatrixCharacteristics(matrix.length, matrix[0].length, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE, nnz);
return writeInputMatrixWithMTD(name, matrix, bIncludeR, mc);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/applications/GLMTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/applications/GLMTest.java b/src/test/java/org/apache/sysml/test/integration/applications/GLMTest.java
index aaa776d..854bf63 100644
--- a/src/test/java/org/apache/sysml/test/integration/applications/GLMTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/applications/GLMTest.java
@@ -27,7 +27,7 @@ import java.util.List;
import java.util.Random;
import org.junit.runners.Parameterized.Parameters;
-
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
import org.apache.sysml.test.integration.AutomatedTestBase;
@@ -280,7 +280,7 @@ public abstract class GLMTest extends AutomatedTestBase
}
- int defaultBlockSize = org.apache.sysml.parser.DMLTranslator.DMLBlockSize;
+ int defaultBlockSize = OptimizerUtils.DEFAULT_BLOCKSIZE;
MatrixCharacteristics mc_X = new MatrixCharacteristics (rows, cols, defaultBlockSize, defaultBlockSize, nnz_in_X);
writeInputMatrixWithMTD ("X", X, true, mc_X);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/append/AppendMatrixTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/append/AppendMatrixTest.java b/src/test/java/org/apache/sysml/test/integration/functions/append/AppendMatrixTest.java
index 551c7ab..07a358f 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/append/AppendMatrixTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/append/AppendMatrixTest.java
@@ -24,12 +24,11 @@ import java.util.Random;
import org.junit.Assert;
import org.junit.Test;
-
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
import org.apache.sysml.hops.BinaryOp;
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.BinaryOp.AppendMethod;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
import org.apache.sysml.test.integration.AutomatedTestBase;
import org.apache.sysml.test.integration.TestConfiguration;
@@ -55,7 +54,7 @@ public class AppendMatrixTest extends AutomatedTestBase
private final static int cols1b = 1059;
private final static int cols2b = 1010;
//usecase c: outblock blocksize
- private final static int cols1c = 2*DMLTranslator.DMLBlockSize;
+ private final static int cols1c = 2*OptimizerUtils.DEFAULT_BLOCKSIZE;
private final static int cols2c = 1010;
//usecase d: outblock blocksize
private final static int cols1d = 1460;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/append/AppendVectorTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/append/AppendVectorTest.java b/src/test/java/org/apache/sysml/test/integration/functions/append/AppendVectorTest.java
index 5272904..f5bf5ee 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/append/AppendVectorTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/append/AppendVectorTest.java
@@ -24,10 +24,9 @@ import java.util.Random;
import org.junit.Assert;
import org.junit.Test;
-
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
-import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
import org.apache.sysml.test.integration.AutomatedTestBase;
import org.apache.sysml.test.integration.TestConfiguration;
@@ -36,8 +35,6 @@ import org.apache.sysml.utils.Statistics;
public class AppendVectorTest extends AutomatedTestBase
{
-
-
private final static String TEST_NAME = "AppendVectorTest";
private final static String TEST_DIR = "functions/append/";
private final static String TEST_CLASS_DIR = TEST_DIR + AppendVectorTest.class.getSimpleName() + "/";
@@ -46,7 +43,7 @@ public class AppendVectorTest extends AutomatedTestBase
private final static int rows1 = 1279;
private final static int cols1 = 1059;
private final static int rows2 = 2021;
- private final static int cols2 = DMLTranslator.DMLBlockSize;
+ private final static int cols2 = OptimizerUtils.DEFAULT_BLOCKSIZE;
private final static int min=0;
private final static int max=100;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullIntegerDivisionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullIntegerDivisionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullIntegerDivisionTest.java
index bb7567a..4b3f5b0 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullIntegerDivisionTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullIntegerDivisionTest.java
@@ -25,10 +25,9 @@ import java.util.HashMap;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.lops.LopProperties.ExecType;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
@@ -39,8 +38,7 @@ import org.apache.sysml.test.integration.TestConfiguration;
import org.apache.sysml.test.utils.TestUtils;
public class FullIntegerDivisionTest extends AutomatedTestBase
-{
-
+{
private final static String TEST_NAME1 = "IntegerDivision_mod";
private final static String TEST_NAME2 = "IntegerDivision_div";
@@ -349,7 +347,7 @@ public class FullIntegerDivisionTest extends AutomatedTestBase
//generate dataset A
if( dt1 == DataType.MATRIX ){
double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, 7);
- MatrixCharacteristics mcA = new MatrixCharacteristics(rows, cols, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, (long) (rows*cols*sparsity));
+ MatrixCharacteristics mcA = new MatrixCharacteristics(rows, cols, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE, (long) (rows*cols*sparsity));
writeInputMatrixWithMTD("A", A, true, mcA);
}
else{
@@ -359,7 +357,7 @@ public class FullIntegerDivisionTest extends AutomatedTestBase
//generate dataset B
if( dt2 == DataType.MATRIX ){
- MatrixCharacteristics mcB = new MatrixCharacteristics(rows, cols, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, (long) (rows*cols*sparsity));
+ MatrixCharacteristics mcB = new MatrixCharacteristics(rows, cols, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE, (long) (rows*cols*sparsity));
double[][] B = getRandomMatrix(rows, cols, min, max, sparsity, 3);
writeInputMatrixWithMTD("B", B, true, mcB);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPowerTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPowerTest.java b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPowerTest.java
index 3e3df1f..b59fd26 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPowerTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix_full_other/FullPowerTest.java
@@ -25,10 +25,9 @@ import java.util.HashMap;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.lops.LopProperties.ExecType;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
@@ -233,7 +232,7 @@ public class FullPowerTest extends AutomatedTestBase
//generate dataset A
if( dt1 == DataType.MATRIX ){
double[][] A = getRandomMatrix(rows, cols, min, max, sparsity, 7);
- MatrixCharacteristics mcA = new MatrixCharacteristics(rows, cols, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, (long) (rows*cols*sparsity));
+ MatrixCharacteristics mcA = new MatrixCharacteristics(rows, cols, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE, (long) (rows*cols*sparsity));
writeInputMatrixWithMTD("A", A, true, mcA);
}
else{
@@ -243,7 +242,7 @@ public class FullPowerTest extends AutomatedTestBase
//generate dataset B
if( dt2 == DataType.MATRIX ){
- MatrixCharacteristics mcB = new MatrixCharacteristics(rows, cols, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, (long) (rows*cols*sparsity));
+ MatrixCharacteristics mcB = new MatrixCharacteristics(rows, cols, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE, (long) (rows*cols*sparsity));
double[][] B = getRandomMatrix(rows, cols, min, max, sparsity, 3);
writeInputMatrixWithMTD("B", B, true, mcB);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/caching/CachingPWriteExportTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/caching/CachingPWriteExportTest.java b/src/test/java/org/apache/sysml/test/integration/functions/caching/CachingPWriteExportTest.java
index cb21346..201d32a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/caching/CachingPWriteExportTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/caching/CachingPWriteExportTest.java
@@ -21,9 +21,8 @@ package org.apache.sysml.test.integration.functions.caching;
import org.junit.Assert;
import org.junit.Test;
-
import org.apache.sysml.hops.Hop;
-import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.util.DataConverter;
@@ -97,7 +96,7 @@ public class CachingPWriteExportTest extends AutomatedTestBase
else
ii = InputInfo.TextCellInputInfo;
- MatrixBlock mb = DataConverter.readMatrixFromHDFS(output("V"), ii, rows, cols, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, sparsity);
+ MatrixBlock mb = DataConverter.readMatrixFromHDFS(output("V"), ii, rows, cols, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE, sparsity);
Vp = DataConverter.convertToDoubleMatrix(mb);
}
catch(Exception ex)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/data/FullStringInitializeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/FullStringInitializeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/FullStringInitializeTest.java
index f533857..78e4021 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/FullStringInitializeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/FullStringInitializeTest.java
@@ -21,10 +21,9 @@ package org.apache.sysml.test.integration.functions.data;
import org.junit.Assert;
import org.junit.Test;
-
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.lops.LopProperties.ExecType;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -363,7 +362,7 @@ public class FullStringInitializeTest extends AutomatedTestBase
if( !expectExcept ) {
//compare matrices
MatrixBlock ret = DataConverter.readMatrixFromHDFS(output("A"), InputInfo.TextCellInputInfo,
- rows, cols, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, sparsity, null);
+ rows, cols, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE, sparsity, null);
double[][] dret = DataConverter.convertToDoubleMatrix(ret);
TestUtils.compareMatrices(A, dret, rows, cols, eps);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/data/ReadMMTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/ReadMMTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/ReadMMTest.java
index 9535c82..17f552c 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/data/ReadMMTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/data/ReadMMTest.java
@@ -24,9 +24,8 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import org.junit.Test;
-
import org.apache.sysml.api.DMLException;
-import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.OutputInfo;
@@ -226,8 +225,8 @@ public class ReadMMTest extends AutomatedTestBase
public void testBinaryWrongRowDimension() throws IOException {
int rows = 5;
int cols = 10;
- int rowsInBlock = DMLTranslator.DMLBlockSize;
- int colsInBlock = DMLTranslator.DMLBlockSize;
+ int rowsInBlock = OptimizerUtils.DEFAULT_BLOCKSIZE;
+ int colsInBlock = OptimizerUtils.DEFAULT_BLOCKSIZE;
TestConfiguration config = availableTestConfigurations.get("BinaryWrongRowDimensionTest");
config.addVariable("rows", rows);
@@ -246,8 +245,8 @@ public class ReadMMTest extends AutomatedTestBase
public void testBinaryWrongColDimension() throws IOException {
int rows = 10;
int cols = 5;
- int rowsInBlock = DMLTranslator.DMLBlockSize;
- int colsInBlock = DMLTranslator.DMLBlockSize;
+ int rowsInBlock = OptimizerUtils.DEFAULT_BLOCKSIZE;
+ int colsInBlock = OptimizerUtils.DEFAULT_BLOCKSIZE;
TestConfiguration config = availableTestConfigurations.get("BinaryWrongColDimensionTest");
config.addVariable("rows", rows);
@@ -274,8 +273,8 @@ public class ReadMMTest extends AutomatedTestBase
public void testBinaryWrongDimensions() throws IOException {
int rows = 3;
int cols = 2;
- int rowsInBlock = DMLTranslator.DMLBlockSize;
- int colsInBlock = DMLTranslator.DMLBlockSize;
+ int rowsInBlock = OptimizerUtils.DEFAULT_BLOCKSIZE;
+ int colsInBlock = OptimizerUtils.DEFAULT_BLOCKSIZE;
TestConfiguration config = availableTestConfigurations.get("TextWrongDimensionsTest");
config.addVariable("rows", rows);
@@ -359,7 +358,7 @@ public class ReadMMTest extends AutomatedTestBase
MapReduceTool.deleteFileIfExistOnHDFS(fname);
MapReduceTool.deleteFileIfExistOnHDFS(fname + ".mtd");
TestUtils.createFile(fname + "/in");
- MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE);
MapReduceTool.writeMetaDataFile(fname + ".mtd", ValueType.DOUBLE, mc, OutputInfo.stringToOutputInfo("binaryblock"));
runTest(true, DMLException.class);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/io/SeqParReadTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/SeqParReadTest.java b/src/test/java/org/apache/sysml/test/integration/functions/io/SeqParReadTest.java
index d913193..81fda82 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/SeqParReadTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/SeqParReadTest.java
@@ -22,10 +22,8 @@ package org.apache.sysml.test.integration.functions.io;
import java.io.IOException;
import org.junit.Assert;
-
import org.junit.Test;
-
-import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
@@ -212,11 +210,11 @@ public class SeqParReadTest extends AutomatedTestBase {
private void runReadTypeFormatSparsitySizeTest(boolean parallel, OutputInfo fmt, boolean dense, boolean big ) {
- boolean oldpar = OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS;
+ boolean oldpar = CompilerConfig.FLAG_PARREAD_TEXT;
try
{
- OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS = parallel;
+ CompilerConfig.FLAG_PARREAD_TEXT = parallel;
TestConfiguration config = getTestConfiguration(TEST_NAME);
loadTestConfiguration(config);
@@ -252,7 +250,7 @@ public class SeqParReadTest extends AutomatedTestBase {
}
finally
{
- OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS = oldpar;
+ CompilerConfig.FLAG_PARREAD_TEXT = oldpar;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/io/csv/ReadCSVTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/csv/ReadCSVTest.java b/src/test/java/org/apache/sysml/test/integration/functions/io/csv/ReadCSVTest.java
index 2739f89..ab1f23e 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/csv/ReadCSVTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/csv/ReadCSVTest.java
@@ -20,9 +20,8 @@
package org.apache.sysml.test.integration.functions.io.csv;
import org.junit.Test;
-
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
-import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.test.integration.AutomatedTestBase;
import org.apache.sysml.test.integration.TestConfiguration;
import org.apache.sysml.test.utils.TestUtils;
@@ -138,12 +137,12 @@ public class ReadCSVTest extends AutomatedTestBase
{
RUNTIME_PLATFORM oldPlatform = rtplatform;
- boolean oldpar = OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS;
+ boolean oldpar = CompilerConfig.FLAG_PARREAD_TEXT;
try
{
rtplatform = platform;
- OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS = parallel;
+ CompilerConfig.FLAG_PARREAD_TEXT = parallel;
TestConfiguration config = getTestConfiguration(TEST_NAME);
@@ -171,7 +170,7 @@ public class ReadCSVTest extends AutomatedTestBase
finally
{
rtplatform = oldPlatform;
- OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS = oldpar;
+ CompilerConfig.FLAG_PARREAD_TEXT = oldpar;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/io/matrixmarket/ReadMMTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/io/matrixmarket/ReadMMTest.java b/src/test/java/org/apache/sysml/test/integration/functions/io/matrixmarket/ReadMMTest.java
index 8884daf..6be1b9c 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/io/matrixmarket/ReadMMTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/io/matrixmarket/ReadMMTest.java
@@ -20,9 +20,8 @@
package org.apache.sysml.test.integration.functions.io.matrixmarket;
import org.junit.Test;
-
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
-import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.test.integration.AutomatedTestBase;
import org.apache.sysml.test.integration.TestConfiguration;
import org.apache.sysml.test.utils.TestUtils;
@@ -91,12 +90,12 @@ public class ReadMMTest extends AutomatedTestBase
private void runMMTest(int testNumber, RUNTIME_PLATFORM platform, boolean parallel) {
RUNTIME_PLATFORM oldPlatform = rtplatform;
- boolean oldpar = OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS;
+ boolean oldpar = CompilerConfig.FLAG_PARREAD_TEXT;
try
{
rtplatform = platform;
- OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS = parallel;
+ CompilerConfig.FLAG_PARREAD_TEXT = parallel;
TestConfiguration config = getTestConfiguration(TEST_NAME);
loadTestConfiguration(config);
@@ -123,7 +122,7 @@ public class ReadMMTest extends AutomatedTestBase
finally
{
rtplatform = oldPlatform;
- OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS = oldpar;
+ CompilerConfig.FLAG_PARREAD_TEXT = oldpar;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeChangeTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeChangeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeChangeTest.java
index 4a3bb4d..8d30240 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeChangeTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeChangeTest.java
@@ -184,7 +184,7 @@ public class DataTypeChangeTest extends AutomatedTestBase
loadTestConfiguration(testConfig);
DMLConfig conf = new DMLConfig(getCurConfigFile().getPath());
- ConfigurationManager.setConfig(conf);
+ ConfigurationManager.setLocalConfig(conf);
String dmlScriptString="";
HashMap<String, String> argVals = new HashMap<String,String>();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java
index 110cdf3..749f8c1 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java
@@ -178,7 +178,7 @@ public class GNMFTest extends AutomatedTestBase
MLOutput out = mlCtx.execute(fullDMLScriptName, programArgs);
if(numRegisteredOutputs >= 2) {
- String configStr = ConfigurationManager.getConfig().getConfigInfo();
+ String configStr = ConfigurationManager.getDMLConfig().getConfigInfo();
if(configStr.contains("cp.parallel.matrixmult: true"))
Assert.fail("Configuration not updated via setConfig");
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java
index a2dc04b..9f14bdc 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java
@@ -338,7 +338,7 @@ public class ParForDependencyAnalysisTest extends AutomatedTestBase
loadTestConfiguration(testConfig);
DMLConfig conf = new DMLConfig(getCurConfigFile().getPath());
- ConfigurationManager.setConfig(conf);
+ ConfigurationManager.setLocalConfig(conf);
String dmlScriptString="";
HashMap<String, String> argVals = new HashMap<String,String>();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForReplaceThreadIDRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForReplaceThreadIDRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForReplaceThreadIDRecompileTest.java
index 68dd7e0..c541fa2 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForReplaceThreadIDRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForReplaceThreadIDRecompileTest.java
@@ -22,10 +22,8 @@ package org.apache.sysml.test.integration.functions.parfor;
import java.util.HashMap;
import org.junit.Assert;
-
import org.junit.Test;
-
-import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
import org.apache.sysml.test.integration.AutomatedTestBase;
import org.apache.sysml.test.integration.TestConfiguration;
@@ -67,14 +65,14 @@ public class ParForReplaceThreadIDRecompileTest extends AutomatedTestBase
*/
private void runThreadIDReplaceTest( String TEST_NAME, boolean recompile )
{
- boolean flag = OptimizerUtils.ALLOW_DYN_RECOMPILATION;
+ boolean flag = CompilerConfig.FLAG_DYN_RECOMPILE;
try
{
TestConfiguration config = getTestConfiguration(TEST_NAME);
loadTestConfiguration(config);
- OptimizerUtils.ALLOW_DYN_RECOMPILATION = recompile;
+ CompilerConfig.FLAG_DYN_RECOMPILE = recompile;
// This is for running the junit test the new way, i.e., construct the arguments directly
String HOME = SCRIPT_DIR + TEST_DIR;
@@ -94,7 +92,7 @@ public class ParForReplaceThreadIDRecompileTest extends AutomatedTestBase
Assert.assertTrue( dmlout.size()>=1 );
}
finally{
- OptimizerUtils.ALLOW_DYN_RECOMPILATION = flag;
+ CompilerConfig.FLAG_DYN_RECOMPILE = flag;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java
index db84ea8..98a93ad 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java
@@ -22,9 +22,8 @@ package org.apache.sysml.test.integration.functions.recompile;
import java.util.HashMap;
import org.junit.Assert;
-
import org.junit.Test;
-
+import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
import org.apache.sysml.test.integration.AutomatedTestBase;
@@ -80,7 +79,7 @@ public class FunctionRecompileTest extends AutomatedTestBase
private void runFunctionTest( boolean recompile, boolean IPA )
{
- boolean oldFlagRecompile = OptimizerUtils.ALLOW_DYN_RECOMPILATION;
+ boolean oldFlagRecompile = CompilerConfig.FLAG_DYN_RECOMPILE;
boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
try
@@ -103,7 +102,7 @@ public class FunctionRecompileTest extends AutomatedTestBase
double[][] V = getRandomMatrix(rows, cols, 0, 1, sparsity, seed);
writeInputMatrix("V", V, true);
- OptimizerUtils.ALLOW_DYN_RECOMPILATION = recompile;
+ CompilerConfig.FLAG_DYN_RECOMPILE = recompile;
OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = IPA;
boolean exceptionExpected = false;
@@ -133,7 +132,7 @@ public class FunctionRecompileTest extends AutomatedTestBase
}
finally
{
- OptimizerUtils.ALLOW_DYN_RECOMPILATION = oldFlagRecompile;
+ CompilerConfig.FLAG_DYN_RECOMPILE = oldFlagRecompile;
OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/recompile/PredicateRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/PredicateRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/PredicateRecompileTest.java
index c439138..29660aa 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/PredicateRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/PredicateRecompileTest.java
@@ -23,7 +23,7 @@ import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-
+import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
import org.apache.sysml.test.integration.AutomatedTestBase;
@@ -278,7 +278,7 @@ public class PredicateRecompileTest extends AutomatedTestBase
private void runRecompileTest( String testname, boolean recompile, boolean evalExpr, boolean constFold, boolean IPA )
{
- boolean oldFlagRecompile = OptimizerUtils.ALLOW_DYN_RECOMPILATION;
+ boolean oldFlagRecompile = CompilerConfig.FLAG_DYN_RECOMPILE;
boolean oldFlagEval = OptimizerUtils.ALLOW_SIZE_EXPRESSION_EVALUATION;
boolean oldFlagFold = OptimizerUtils.ALLOW_CONSTANT_FOLDING;
boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
@@ -301,7 +301,7 @@ public class PredicateRecompileTest extends AutomatedTestBase
Integer.toString(val),
output("R") };
- OptimizerUtils.ALLOW_DYN_RECOMPILATION = recompile;
+ CompilerConfig.FLAG_DYN_RECOMPILE = recompile;
OptimizerUtils.ALLOW_SIZE_EXPRESSION_EVALUATION = evalExpr;
OptimizerUtils.ALLOW_CONSTANT_FOLDING = constFold;
OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = IPA;
@@ -350,7 +350,7 @@ public class PredicateRecompileTest extends AutomatedTestBase
}
finally
{
- OptimizerUtils.ALLOW_DYN_RECOMPILATION = oldFlagRecompile;
+ CompilerConfig.FLAG_DYN_RECOMPILE = oldFlagRecompile;
OptimizerUtils.ALLOW_SIZE_EXPRESSION_EVALUATION = oldFlagEval;
OptimizerUtils.ALLOW_CONSTANT_FOLDING = oldFlagFold;
OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandRecompileTest.java
index 515b43c..a1ed7f2 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RandRecompileTest.java
@@ -21,7 +21,7 @@ package org.apache.sysml.test.integration.functions.recompile;
import org.junit.Assert;
import org.junit.Test;
-
+import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.test.integration.AutomatedTestBase;
import org.apache.sysml.test.integration.TestConfiguration;
@@ -113,7 +113,7 @@ public class RandRecompileTest extends AutomatedTestBase
private void runRandTest( String testName, boolean recompile, boolean IPA )
{
- boolean oldFlagRecompile = OptimizerUtils.ALLOW_DYN_RECOMPILATION;
+ boolean oldFlagRecompile = CompilerConfig.FLAG_DYN_RECOMPILE;
boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
boolean oldFlagRand1 = OptimizerUtils.ALLOW_RAND_JOB_RECOMPILE;
@@ -132,7 +132,7 @@ public class RandRecompileTest extends AutomatedTestBase
fullDMLScriptName = HOME + testName + ".dml";
programArgs = new String[]{"-args", Integer.toString(rows), Integer.toString(cols) };
- OptimizerUtils.ALLOW_DYN_RECOMPILATION = recompile;
+ CompilerConfig.FLAG_DYN_RECOMPILE = recompile;
OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = IPA;
//disable rand specific recompile
@@ -160,7 +160,7 @@ public class RandRecompileTest extends AutomatedTestBase
}
finally
{
- OptimizerUtils.ALLOW_DYN_RECOMPILATION = oldFlagRecompile;
+ CompilerConfig.FLAG_DYN_RECOMPILE = oldFlagRecompile;
OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA;
OptimizerUtils.ALLOW_RAND_JOB_RECOMPILE = oldFlagRand1;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/recompile/ReblockRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/ReblockRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/ReblockRecompileTest.java
index 531a5b7..54c8fed 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/ReblockRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/ReblockRecompileTest.java
@@ -23,10 +23,8 @@ import java.io.IOException;
import java.util.HashMap;
import org.junit.Assert;
-
import org.junit.Test;
-
-import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.matrix.data.InputInfo;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
@@ -120,7 +118,7 @@ public class ReblockRecompileTest extends AutomatedTestBase
//compare matrices
try
{
- MatrixBlock mo = DataConverter.readMatrixFromHDFS(output("R"), InputInfo.BinaryBlockInputInfo, rows, 1, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ MatrixBlock mo = DataConverter.readMatrixFromHDFS(output("R"), InputInfo.BinaryBlockInputInfo, rows, 1, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE);
HashMap<CellIndex, Double> dmlfile = new HashMap<CellIndex,Double>();
for( int i=0; i<mo.getNumRows(); i++ )
for( int j=0; j<mo.getNumColumns(); j++ )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityFunctionRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityFunctionRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityFunctionRecompileTest.java
index 54f207c..5cafd6a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityFunctionRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityFunctionRecompileTest.java
@@ -23,9 +23,8 @@ import java.util.HashMap;
import org.junit.Assert;
import org.junit.Test;
-
+import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.hops.OptimizerUtils;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -166,7 +165,7 @@ public class SparsityFunctionRecompileTest extends AutomatedTestBase
private void runRecompileTest( String testname, boolean recompile, boolean IPA )
{
- boolean oldFlagRecompile = OptimizerUtils.ALLOW_DYN_RECOMPILATION;
+ boolean oldFlagRecompile = CompilerConfig.FLAG_DYN_RECOMPILE;
boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS;
boolean oldFlagBranchRemoval = OptimizerUtils.ALLOW_BRANCH_REMOVAL;
@@ -180,12 +179,12 @@ public class SparsityFunctionRecompileTest extends AutomatedTestBase
programArgs = new String[]{"-args",
input("V"), Double.toString(val), output("R") };
- OptimizerUtils.ALLOW_DYN_RECOMPILATION = recompile;
+ CompilerConfig.FLAG_DYN_RECOMPILE = recompile;
OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = IPA;
OptimizerUtils.ALLOW_BRANCH_REMOVAL = false;
MatrixBlock mb = MatrixBlock.randOperations((int)rows, (int)cols, sparsity, 0, 1, "uniform", 732);
- MatrixCharacteristics mc = new MatrixCharacteristics(rows,cols,DMLTranslator.DMLBlockSize,DMLTranslator.DMLBlockSize,(long)(rows*cols*sparsity));
+ MatrixCharacteristics mc = new MatrixCharacteristics(rows,cols,OptimizerUtils.DEFAULT_BLOCKSIZE,OptimizerUtils.DEFAULT_BLOCKSIZE,(long)(rows*cols*sparsity));
DataConverter.writeMatrixToHDFS(mb, input("V"), OutputInfo.TextCellOutputInfo, mc);
MapReduceTool.writeMetaDataFile(input("V.mtd"), ValueType.DOUBLE, mc, OutputInfo.TextCellOutputInfo);
@@ -221,7 +220,7 @@ public class SparsityFunctionRecompileTest extends AutomatedTestBase
}
finally
{
- OptimizerUtils.ALLOW_DYN_RECOMPILATION = oldFlagRecompile;
+ CompilerConfig.FLAG_DYN_RECOMPILE = oldFlagRecompile;
OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA;
OptimizerUtils.ALLOW_BRANCH_REMOVAL = oldFlagBranchRemoval;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityRecompileTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityRecompileTest.java
index dc01e34..357b15a 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityRecompileTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/SparsityRecompileTest.java
@@ -22,11 +22,9 @@ package org.apache.sysml.test.integration.functions.recompile;
import java.util.HashMap;
import org.junit.Assert;
-
import org.junit.Test;
-
+import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.hops.OptimizerUtils;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
@@ -135,7 +133,7 @@ public class SparsityRecompileTest extends AutomatedTestBase
private void runRecompileTest( String testname, boolean recompile )
{
- boolean oldFlagRecompile = OptimizerUtils.ALLOW_DYN_RECOMPILATION;
+ boolean oldFlagRecompile = CompilerConfig.FLAG_DYN_RECOMPILE;
try
{
@@ -146,10 +144,10 @@ public class SparsityRecompileTest extends AutomatedTestBase
programArgs = new String[]{"-explain", "-args",
input("V"), Double.toString(val), output("R") };
- OptimizerUtils.ALLOW_DYN_RECOMPILATION = recompile;
+ CompilerConfig.FLAG_DYN_RECOMPILE = recompile;
MatrixBlock mb = MatrixBlock.randOperations((int)rows, (int)cols, sparsity, 0, 1, "uniform", System.currentTimeMillis());
- MatrixCharacteristics mc = new MatrixCharacteristics(rows,cols,DMLTranslator.DMLBlockSize,DMLTranslator.DMLBlockSize,(long)(rows*cols*sparsity));
+ MatrixCharacteristics mc = new MatrixCharacteristics(rows,cols,OptimizerUtils.DEFAULT_BLOCKSIZE,OptimizerUtils.DEFAULT_BLOCKSIZE,(long)(rows*cols*sparsity));
DataConverter.writeMatrixToHDFS(mb, input("V"), OutputInfo.TextCellOutputInfo, mc);
MapReduceTool.writeMetaDataFile(input("V.mtd"), ValueType.DOUBLE, mc, OutputInfo.TextCellOutputInfo);
@@ -183,7 +181,7 @@ public class SparsityRecompileTest extends AutomatedTestBase
}
finally
{
- OptimizerUtils.ALLOW_DYN_RECOMPILATION = oldFlagRecompile;
+ CompilerConfig.FLAG_DYN_RECOMPILE = oldFlagRecompile;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/transform/ScalingTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/ScalingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/ScalingTest.java
index e08c8dd..058308b 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/transform/ScalingTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/ScalingTest.java
@@ -34,7 +34,6 @@ import org.junit.Test;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.io.ReaderBinaryBlock;
import org.apache.sysml.runtime.io.ReaderTextCSV;
@@ -229,8 +228,8 @@ public class ScalingTest extends AutomatedTestBase
ReaderBinaryBlock bbReader = new ReaderBinaryBlock(false);
out = bbReader.readMatrixFromHDFS(
outputFile, exp.getNumRows(), exp.getNumColumns(),
- ConfigurationManager.getConfig().getIntValue( DMLConfig.DEFAULT_BLOCK_SIZE ),
- ConfigurationManager.getConfig().getIntValue( DMLConfig.DEFAULT_BLOCK_SIZE ),
+ ConfigurationManager.getBlocksize(),
+ ConfigurationManager.getBlocksize(),
-1);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformTest.java
index e607ee2..3d799f0 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformTest.java
@@ -26,7 +26,6 @@ import org.junit.Test;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.runtime.io.ReaderBinaryBlock;
import org.apache.sysml.runtime.io.ReaderTextCSV;
import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
@@ -666,13 +665,13 @@ public class TransformTest extends AutomatedTestBase
ReaderBinaryBlock bbReader = new ReaderBinaryBlock(false);
out = bbReader.readMatrixFromHDFS(
output("tfout"), exp.getNumRows(), exp.getNumColumns(),
- ConfigurationManager.getConfig().getIntValue( DMLConfig.DEFAULT_BLOCK_SIZE ),
- ConfigurationManager.getConfig().getIntValue( DMLConfig.DEFAULT_BLOCK_SIZE ),
+ ConfigurationManager.getBlocksize(),
+ ConfigurationManager.getBlocksize(),
-1);
out2 = bbReader.readMatrixFromHDFS(
output("test_tfout"), exp.getNumRows(), exp.getNumColumns(),
- ConfigurationManager.getConfig().getIntValue( DMLConfig.DEFAULT_BLOCK_SIZE ),
- ConfigurationManager.getConfig().getIntValue( DMLConfig.DEFAULT_BLOCK_SIZE ),
+ ConfigurationManager.getBlocksize(),
+ ConfigurationManager.getBlocksize(),
-1);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/test/java/org/apache/sysml/test/integration/functions/updateinplace/UpdateInPlaceTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/functions/updateinplace/UpdateInPlaceTest.java b/src/test/java/org/apache/sysml/test/integration/functions/updateinplace/UpdateInPlaceTest.java
index 062a680..cb19d04 100644
--- a/src/test/java/org/apache/sysml/test/integration/functions/updateinplace/UpdateInPlaceTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/functions/updateinplace/UpdateInPlaceTest.java
@@ -24,7 +24,6 @@ import java.util.List;
import org.junit.Assert;
import org.junit.Test;
-import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.controlprogram.parfor.opt.OptimizerRuleBased;
import org.apache.sysml.test.integration.AutomatedTestBase;
import org.apache.sysml.test.integration.TestConfiguration;
@@ -95,10 +94,8 @@ public class UpdateInPlaceTest extends AutomatedTestBase
//Note: In order to run these tests against ParFor loop, parfor's DEBUG flag needs to be set in the script.
@Override
- public void setUp()
- {
+ public void setUp() {
TestUtils.clearAssertionInformation();
- OptimizerUtils.ALLOW_DYN_RECOMPILATION = true;
addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, null));
}
[4/4] incubator-systemml git commit: [SYSTEMML-584] New thread-local
configuration handling (dml/compiler)
Posted by mb...@apache.org.
[SYSTEMML-584] New thread-local configuration handling (dml/compiler)
So far we did not fully support concurrent script executions in a shared
JVM process (e.g., as necessary for JMLC scoring in spark executors)
because global static state updates caused potential side effects. This
patch introduces thread-local configuration handling for dml and
compiler configurations. All these functionalities are encapsulated
within the ConfigurationManager behind a very generic API. This cleanup
covers all relevant configurations for JMLC - down the road we could
extend this as required. Note that we also support 1-1 thread mappings
in parfor, which will be required in JMLC for parfor scripts with
disabled parallelism.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/257793d8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/257793d8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/257793d8
Branch: refs/heads/master
Commit: 257793d8309056b86096f5ed7c8528ad83f5d8fc
Parents: 59a4a50
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sat Mar 19 01:41:13 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sat Mar 19 15:20:39 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/sysml/api/DMLScript.java | 26 +--
.../java/org/apache/sysml/api/MLContext.java | 56 +++---
.../java/org/apache/sysml/api/MLMatrix.java | 5 +-
.../org/apache/sysml/api/jmlc/Connection.java | 72 ++++---
.../apache/sysml/api/jmlc/PreparedScript.java | 9 +-
.../org/apache/sysml/conf/CompilerConfig.java | 157 +++++++++++++++
.../apache/sysml/conf/ConfigurationManager.java | 190 ++++++++++++++++---
.../java/org/apache/sysml/conf/DMLConfig.java | 90 ++++-----
.../java/org/apache/sysml/hops/AggBinaryOp.java | 3 +-
.../java/org/apache/sysml/hops/AggUnaryOp.java | 3 +-
.../java/org/apache/sysml/hops/BinaryOp.java | 5 +-
.../java/org/apache/sysml/hops/DataGenOp.java | 13 +-
src/main/java/org/apache/sysml/hops/DataOp.java | 9 +-
src/main/java/org/apache/sysml/hops/Hop.java | 5 +-
.../java/org/apache/sysml/hops/IndexingOp.java | 3 +-
.../org/apache/sysml/hops/LeftIndexingOp.java | 10 +-
.../org/apache/sysml/hops/OptimizerUtils.java | 151 ++++++---------
.../sysml/hops/ParameterizedBuiltinOp.java | 3 +-
.../org/apache/sysml/hops/QuaternaryOp.java | 3 +-
.../java/org/apache/sysml/hops/ReorgOp.java | 3 +-
.../java/org/apache/sysml/hops/TernaryOp.java | 3 +-
.../java/org/apache/sysml/hops/UnaryOp.java | 3 +-
.../apache/sysml/hops/cost/CostEstimator.java | 5 +-
.../hops/cost/CostEstimatorStaticRuntime.java | 8 +-
.../sysml/hops/globalopt/GDFEnumOptimizer.java | 7 +-
.../sysml/hops/ipa/InterProceduralAnalysis.java | 5 +-
.../apache/sysml/hops/recompile/Recompiler.java | 37 ++--
.../sysml/hops/rewrite/ProgramRewriter.java | 5 +-
.../RewriteAlgebraicSimplificationDynamic.java | 6 +-
.../rewrite/RewriteBlockSizeAndReblock.java | 10 +-
.../hops/rewrite/RewriteConstantFolding.java | 2 +-
.../RewriteSplitDagDataDependentOperators.java | 4 +-
.../java/org/apache/sysml/lops/compile/Dag.java | 3 +-
.../apache/sysml/lops/runtime/RunMRJobs.java | 8 +-
.../org/apache/sysml/parser/AParserWrapper.java | 16 +-
.../org/apache/sysml/parser/DMLTranslator.java | 26 +--
.../org/apache/sysml/parser/DataExpression.java | 17 +-
.../apache/sysml/parser/ForStatementBlock.java | 5 +-
.../apache/sysml/parser/IfStatementBlock.java | 4 +-
.../org/apache/sysml/parser/StatementBlock.java | 5 +-
.../sysml/parser/WhileStatementBlock.java | 4 +-
.../sysml/parser/dml/DmlSyntacticValidator.java | 5 +-
.../parser/pydml/PydmlSyntacticValidator.java | 5 +-
.../ExternalFunctionProgramBlock.java | 16 +-
.../ExternalFunctionProgramBlockCP.java | 4 +-
.../controlprogram/FunctionProgramBlock.java | 4 +-
.../controlprogram/ParForProgramBlock.java | 30 ++-
.../runtime/controlprogram/ProgramBlock.java | 7 +-
.../controlprogram/caching/MatrixObject.java | 11 +-
.../parfor/DataPartitionerRemoteMR.java | 2 +-
.../controlprogram/parfor/LocalParWorker.java | 11 +-
.../controlprogram/parfor/ProgramConverter.java | 15 +-
.../controlprogram/parfor/RemoteDPParForMR.java | 2 +-
.../controlprogram/parfor/RemoteParForMR.java | 2 +-
.../parfor/ResultMergeRemoteMR.java | 2 +-
.../parfor/opt/OptimizationWrapper.java | 4 +-
.../parfor/opt/OptimizerConstrained.java | 5 +-
.../parfor/opt/OptimizerRuleBased.java | 8 +-
.../controlprogram/parfor/opt/PerfTestTool.java | 12 +-
.../parfor/opt/ProgramRecompiler.java | 6 +-
.../ParameterizedBuiltinCPFileInstruction.java | 9 +-
.../sysml/runtime/io/MatrixReaderFactory.java | 15 +-
.../sysml/runtime/io/MatrixWriterFactory.java | 9 +-
.../apache/sysml/runtime/matrix/CMCOVMR.java | 2 +-
.../sysml/runtime/matrix/CSVReblockMR.java | 4 +-
.../apache/sysml/runtime/matrix/CombineMR.java | 2 +-
.../apache/sysml/runtime/matrix/DataGenMR.java | 2 +-
.../org/apache/sysml/runtime/matrix/GMR.java | 2 +-
.../sysml/runtime/matrix/GroupedAggMR.java | 2 +-
.../org/apache/sysml/runtime/matrix/MMCJMR.java | 2 +-
.../org/apache/sysml/runtime/matrix/MMRJMR.java | 2 +-
.../apache/sysml/runtime/matrix/ReblockMR.java | 2 +-
.../org/apache/sysml/runtime/matrix/SortMR.java | 2 +-
.../apache/sysml/runtime/matrix/WriteCSVMR.java | 2 +-
.../sysml/runtime/matrix/data/MatrixBlock.java | 9 +-
.../matrix/mapred/MRJobConfiguration.java | 6 +-
.../sysml/runtime/transform/ApplyTfBBMR.java | 3 +-
.../sysml/runtime/transform/DataTransform.java | 4 +-
.../sysml/runtime/util/LocalFileUtils.java | 2 +-
src/main/java/org/apache/sysml/udf/Matrix.java | 6 +-
.../sysml/udf/lib/DynamicReadMatrixCP.java | 4 +-
.../sysml/udf/lib/DynamicReadMatrixRcCP.java | 4 +-
.../org/apache/sysml/yarn/DMLAppMaster.java | 2 +-
.../apache/sysml/yarn/DMLAppMasterUtils.java | 9 +-
.../sysml/yarn/ropt/ResourceOptimizer.java | 4 +-
.../sysml/yarn/ropt/YarnOptimizerUtils.java | 3 +-
.../test/integration/AutomatedTestBase.java | 5 +-
.../test/integration/applications/GLMTest.java | 4 +-
.../functions/append/AppendMatrixTest.java | 5 +-
.../functions/append/AppendVectorTest.java | 7 +-
.../FullIntegerDivisionTest.java | 10 +-
.../binary/matrix_full_other/FullPowerTest.java | 7 +-
.../caching/CachingPWriteExportTest.java | 5 +-
.../data/FullStringInitializeTest.java | 5 +-
.../integration/functions/data/ReadMMTest.java | 17 +-
.../functions/io/SeqParReadTest.java | 10 +-
.../functions/io/csv/ReadCSVTest.java | 9 +-
.../functions/io/matrixmarket/ReadMMTest.java | 9 +-
.../functions/misc/DataTypeChangeTest.java | 2 +-
.../functions/mlcontext/GNMFTest.java | 2 +-
.../parfor/ParForDependencyAnalysisTest.java | 2 +-
.../ParForReplaceThreadIDRecompileTest.java | 10 +-
.../recompile/FunctionRecompileTest.java | 9 +-
.../recompile/PredicateRecompileTest.java | 8 +-
.../functions/recompile/RandRecompileTest.java | 8 +-
.../recompile/ReblockRecompileTest.java | 6 +-
.../SparsityFunctionRecompileTest.java | 11 +-
.../recompile/SparsityRecompileTest.java | 12 +-
.../functions/transform/ScalingTest.java | 5 +-
.../functions/transform/TransformTest.java | 9 +-
.../updateinplace/UpdateInPlaceTest.java | 5 +-
111 files changed, 832 insertions(+), 613 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/api/DMLScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java b/src/main/java/org/apache/sysml/api/DMLScript.java
index 101b229..a3278c0 100644
--- a/src/main/java/org/apache/sysml/api/DMLScript.java
+++ b/src/main/java/org/apache/sysml/api/DMLScript.java
@@ -46,7 +46,7 @@ import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.xml.sax.SAXException;
-
+import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.debug.DMLDebugger;
@@ -571,13 +571,15 @@ public class DMLScript
printStartExecInfo( dmlScriptStr );
//Step 1: parse configuration files
- DMLConfig conf = DMLConfig.readAndMergeConfigurationFiles(fnameOptConfig);
- ConfigurationManager.setConfig(conf);
- LOG.debug("\nDML config: \n" + conf.getConfigInfo());
+ DMLConfig dmlconf = DMLConfig.readAndMergeConfigurationFiles(fnameOptConfig);
+ ConfigurationManager.setGlobalConfig(dmlconf);
+ CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf);
+ ConfigurationManager.setGlobalConfig(cconf);
+ LOG.debug("\nDML config: \n" + dmlconf.getConfigInfo());
//Step 2: set local/remote memory if requested (for compile in AM context)
- if( conf.getBooleanValue(DMLConfig.YARN_APPMASTER) ){
- DMLAppMasterUtils.setupConfigRemoteMaxMemory(conf);
+ if( dmlconf.getBooleanValue(DMLConfig.YARN_APPMASTER) ){
+ DMLAppMasterUtils.setupConfigRemoteMaxMemory(dmlconf);
}
//Step 3: parse dml script
@@ -621,7 +623,7 @@ public class DMLScript
}
//Step 7: generate runtime program
- Program rtprog = prog.getRuntimeProgram(conf);
+ Program rtprog = prog.getRuntimeProgram(dmlconf);
if (LOG.isDebugEnabled()) {
LOG.info("********************** Instructions *******************");
@@ -638,8 +640,8 @@ public class DMLScript
}
//launch SystemML appmaster (if requested and not already in launched AM)
- if( conf.getBooleanValue(DMLConfig.YARN_APPMASTER) ){
- if( !isActiveAM() && DMLYarnClientProxy.launchDMLYarnAppmaster(dmlScriptStr, conf, allArgs, rtprog) )
+ if( dmlconf.getBooleanValue(DMLConfig.YARN_APPMASTER) ){
+ if( !isActiveAM() && DMLYarnClientProxy.launchDMLYarnAppmaster(dmlScriptStr, dmlconf, allArgs, rtprog) )
return; //if AM launch unsuccessful, fall back to normal execute
if( isActiveAM() ) //in AM context (not failed AM launch)
DMLAppMasterUtils.setupProgramMappingRemoteMaxMemory(rtprog);
@@ -669,7 +671,7 @@ public class DMLScript
ExecutionContext ec = null;
try
{
- initHadoopExecution( conf );
+ initHadoopExecution( dmlconf );
//run execute (w/ exception handling to ensure proper shutdown)
ec = ExecutionContextFactory.createContext(rtprog);
@@ -688,7 +690,7 @@ public class DMLScript
LOG.info("END DML run " + getDateTime() );
//cleanup scratch_space and all working dirs
- cleanupHadoopExecution( conf );
+ cleanupHadoopExecution( dmlconf );
}
}
@@ -755,7 +757,7 @@ public class DMLScript
//Step 1: parse configuration files
dbprog.conf = DMLConfig.readAndMergeConfigurationFiles(fnameOptConfig);
- ConfigurationManager.setConfig(dbprog.conf);
+ ConfigurationManager.setGlobalConfig(dbprog.conf);
//Step 2: parse dml script
AParserWrapper parser = AParserWrapper.createParser(parsePyDML);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/api/MLContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/MLContext.java b/src/main/java/org/apache/sysml/api/MLContext.java
index 247cd69..f3d0bbb 100644
--- a/src/main/java/org/apache/sysml/api/MLContext.java
+++ b/src/main/java/org/apache/sysml/api/MLContext.java
@@ -36,6 +36,8 @@ import org.apache.spark.rdd.RDD;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
import org.apache.sysml.api.jmlc.JMLCUtils;
import org.apache.sysml.api.monitoring.SparkMonitoringUtil;
+import org.apache.sysml.conf.CompilerConfig;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.OptimizerUtils;
@@ -486,14 +488,16 @@ public class MLContext {
MatrixObject mo = null;
if( format.equals("csv") ) {
- MatrixCharacteristics mc = new MatrixCharacteristics(rlen, clen, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, nnz);
+ //TODO replace default block size
+ MatrixCharacteristics mc = new MatrixCharacteristics(rlen, clen, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE, nnz);
mo = new MatrixObject(ValueType.DOUBLE, null, new MatrixFormatMetaData(mc, OutputInfo.CSVOutputInfo, InputInfo.CSVInputInfo));
}
else if( format.equals("text") ) {
if(rlen == -1 || clen == -1) {
throw new DMLRuntimeException("The metadata is required in registerInput for format:" + format);
}
- MatrixCharacteristics mc = new MatrixCharacteristics(rlen, clen, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, nnz);
+ //TODO replace default block size
+ MatrixCharacteristics mc = new MatrixCharacteristics(rlen, clen, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE, nnz);
mo = new MatrixObject(ValueType.DOUBLE, null, new MatrixFormatMetaData(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo));
}
else if( format.equals("mm") ) {
@@ -532,7 +536,8 @@ public class MLContext {
* @throws DMLRuntimeException
*/
public void registerInput(String varName, JavaPairRDD<MatrixIndexes,MatrixBlock> rdd, long rlen, long clen) throws DMLRuntimeException {
- registerInput(varName, rdd, rlen, clen, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ //TODO replace default blocksize
+ registerInput(varName, rdd, rlen, clen, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE);
}
/**
@@ -1189,13 +1194,6 @@ public class MLContext {
// Set active MLContext.
_activeMLContext = this;
- // Setup parser parameters
- // TODO In the process of hardening mlcontext, we should also reinvestigate if we
- // could be more restrictive and require known dimensions (rm REJECT_READ_WRITE_UNKNOWNS).
- AParserWrapper.IGNORE_UNSPECIFIED_ARGS = true;
- DataExpression.REJECT_READ_WRITE_UNKNOWNS = false;
- OptimizerUtils.ALLOW_CSE_PERSISTENT_READS = false;
-
if(_monitorUtils != null) {
_monitorUtils.resetMonitoringData();
}
@@ -1254,13 +1252,7 @@ public class MLContext {
}
finally {
// Reset active MLContext.
- _activeMLContext = null;
-
- // Reset parser parameters
- AParserWrapper.IGNORE_UNSPECIFIED_ARGS = false;
- DataExpression.REJECT_READ_WRITE_UNKNOWNS = true;
- OptimizerUtils.ALLOW_CSE_PERSISTENT_READS =
- OptimizerUtils.ALLOW_COMMON_SUBEXPRESSION_ELIMINATION;
+ _activeMLContext = null;
}
}
@@ -1282,27 +1274,25 @@ public class MLContext {
* @throws ParseException
*/
private ExecutionContext executeUsingSimplifiedCompilationChain(String dmlScriptFilePath, boolean isFile, HashMap<String, String> argVals, boolean parsePyDML,
- String[] inputs, String[] outputs, LocalVariableMap inputSymbolTable, String configFilePath) throws IOException, DMLException, ParseException {
- DMLConfig config = null;
- if(configFilePath == null) {
- config = new DMLConfig();
- }
- else {
- config = new DMLConfig(configFilePath);
- }
-
+ String[] inputs, String[] outputs, LocalVariableMap inputSymbolTable, String configFilePath)
+ throws IOException, DMLException, ParseException
+ {
+ //construct dml configuration
+ DMLConfig config = (configFilePath == null) ? new DMLConfig() : new DMLConfig(configFilePath);
for(Entry<String, String> param : _additionalConfigs.entrySet()) {
config.setTextValue(param.getKey(), param.getValue());
}
- ConfigurationManager.setConfig(config);
+ //set global dml and specialized compiler configurations
+ ConfigurationManager.setGlobalConfig(config);
+ CompilerConfig cconf = new CompilerConfig();
+ cconf.set(ConfigType.IGNORE_UNSPECIFIED_ARGS, true);
+ cconf.set(ConfigType.REJECT_READ_WRITE_UNKNOWNS, false);
+ cconf.set(ConfigType.ALLOW_CSE_PERSISTENT_READS, false);
+ ConfigurationManager.setGlobalConfig(cconf);
- String dmlScriptStr = null;
- if(isFile)
- dmlScriptStr = DMLScript.readDMLScript("-f", dmlScriptFilePath);
- else
- dmlScriptStr = DMLScript.readDMLScript("-s", dmlScriptFilePath);
-
+ //read dml script string
+ String dmlScriptStr = DMLScript.readDMLScript( isFile?"-f":"-s", dmlScriptFilePath);
if(_monitorUtils != null) {
_monitorUtils.setDMLString(dmlScriptStr);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/api/MLMatrix.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/MLMatrix.java b/src/main/java/org/apache/sysml/api/MLMatrix.java
index 84e15fa..b0c0afb 100644
--- a/src/main/java/org/apache/sysml/api/MLMatrix.java
+++ b/src/main/java/org/apache/sysml/api/MLMatrix.java
@@ -34,7 +34,7 @@ import org.apache.spark.sql.types.StructType;
import scala.Tuple2;
-import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.parser.ParseException;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.instructions.spark.functions.GetMIMBFromRow;
@@ -91,7 +91,8 @@ public class MLMatrix extends DataFrame {
this.ml = ml;
}
- static String writeStmt = "write(output, \"tmp\", format=\"binary\", rows_in_block=" + DMLTranslator.DMLBlockSize + ", cols_in_block=" + DMLTranslator.DMLBlockSize + ");";
+ //TODO replace default blocksize
+ static String writeStmt = "write(output, \"tmp\", format=\"binary\", rows_in_block=" + OptimizerUtils.DEFAULT_BLOCKSIZE + ", cols_in_block=" + OptimizerUtils.DEFAULT_BLOCKSIZE + ");";
// ------------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/api/jmlc/Connection.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
index 43b56d0..bd764ac 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
@@ -29,19 +29,18 @@ import java.util.HashMap;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-
import org.apache.sysml.api.DMLException;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.CompilerConfig;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
-import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.rewrite.ProgramRewriter;
import org.apache.sysml.hops.rewrite.RewriteRemovePersistentReadWrite;
import org.apache.sysml.parser.AParserWrapper;
import org.apache.sysml.parser.DMLProgram;
import org.apache.sysml.parser.DMLTranslator;
-import org.apache.sysml.parser.DataExpression;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.Program;
import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
@@ -59,9 +58,9 @@ import org.apache.sysml.runtime.util.DataConverter;
* * See JUnit test cases (org.apache.sysml.test.integration.functions.jmlc) for examples.
*/
public class Connection
-{
-
- private DMLConfig _conf = null;
+{
+ private DMLConfig _dmlconf = null;
+ private CompilerConfig _cconf = null;
/**
* Connection constructor, starting point for any other JMLC API calls.
@@ -69,26 +68,31 @@ public class Connection
*/
public Connection()
{
- //setup basic parameters for embedded execution
- //parser parameters
- AParserWrapper.IGNORE_UNSPECIFIED_ARGS = true;
- DataExpression.IGNORE_READ_WRITE_METADATA = true;
- DataExpression.REJECT_READ_WRITE_UNKNOWNS = false;
- //runtime parameters
DMLScript.rtplatform = RUNTIME_PLATFORM.SINGLE_NODE;
- OptimizerUtils.ALLOW_CSE_PERSISTENT_READS = false;
- OptimizerUtils.PARALLEL_CP_MATRIX_OPERATIONS = false;
- OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR = false;
- OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS = false;
- OptimizerUtils.PARALLEL_CP_WRITE_TEXTFORMATS = false;
- OptimizerUtils.PARALLEL_CP_READ_BINARYFORMATS = false;
- OptimizerUtils.PARALLEL_CP_WRITE_BINARYFORMATS = false;
- OptimizerUtils.ALLOW_DYN_RECOMPILATION = false;
+
+ //setup basic parameters for embedded execution
+ //(parser, compiler, and runtime parameters)
+ _cconf = new CompilerConfig();
+ _cconf.set(ConfigType.IGNORE_UNSPECIFIED_ARGS, true);
+ _cconf.set(ConfigType.IGNORE_READ_WRITE_METADATA, true);
+ _cconf.set(ConfigType.REJECT_READ_WRITE_UNKNOWNS, false);
+ _cconf.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, false);
+ _cconf.set(ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS, false);
+ _cconf.set(ConfigType.PARALLEL_CP_READ_BINARYFORMATS, false);
+ _cconf.set(ConfigType.PARALLEL_CP_WRITE_BINARYFORMATS, false);
+ _cconf.set(ConfigType.PARALLEL_CP_MATRIX_OPERATIONS, false);
+ _cconf.set(ConfigType.PARALLEL_LOCAL_OR_REMOTE_PARFOR, false);
+ _cconf.set(ConfigType.ALLOW_DYN_RECOMPILATION, false);
+ _cconf.set(ConfigType.ALLOW_INDIVIDUAL_SB_SPECIFIC_OPS, false);
+ _cconf.set(ConfigType.ALLOW_CSE_PERSISTENT_READS, false);
+ ConfigurationManager.setLocalConfig(_cconf);
+
+ //disable caching globally
CacheableData.disableCaching();
- //create default configuration
- _conf = new DMLConfig();
- ConfigurationManager.setConfig(_conf);
+ //create thread-local default configuration
+ _dmlconf = new DMLConfig();
+ ConfigurationManager.setLocalConfig(_dmlconf);
}
/**
@@ -143,7 +147,7 @@ public class Connection
//lop construct and runtime prog generation
dmlt.constructLops(prog);
- rtprog = prog.getRuntimeProgram(_conf);
+ rtprog = prog.getRuntimeProgram(_dmlconf);
//final cleanup runtime prog
JMLCUtils.cleanupRuntimeProgram(rtprog, outputs);
@@ -162,21 +166,9 @@ public class Connection
/**
*
*/
- public void close()
- {
- //reset parameters for embedded execution
- AParserWrapper.IGNORE_UNSPECIFIED_ARGS = false;
- DataExpression.IGNORE_READ_WRITE_METADATA = false;
- DataExpression.REJECT_READ_WRITE_UNKNOWNS = true;
- OptimizerUtils.ALLOW_CSE_PERSISTENT_READS =
- OptimizerUtils.ALLOW_COMMON_SUBEXPRESSION_ELIMINATION;
- OptimizerUtils.PARALLEL_CP_MATRIX_OPERATIONS = true;
- OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR = true;
- OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS = true;
- OptimizerUtils.PARALLEL_CP_WRITE_TEXTFORMATS = true;
- OptimizerUtils.PARALLEL_CP_READ_BINARYFORMATS = true;
- OptimizerUtils.PARALLEL_CP_WRITE_BINARYFORMATS = true;
- OptimizerUtils.ALLOW_DYN_RECOMPILATION = true;
+ public void close() {
+ //clear thread-local dml / compiler configs
+ ConfigurationManager.clearLocalConfigs();
}
/**
@@ -250,7 +242,7 @@ public class Connection
//read input matrix
InputStream is = new ByteArrayInputStream(input.getBytes("UTF-8"));
ReaderTextCell reader = (ReaderTextCell)MatrixReaderFactory.createMatrixReader(InputInfo.TextCellInputInfo);
- MatrixBlock mb = reader.readMatrixFromInputStream(is, rows, cols, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, (long)rows*cols);
+ MatrixBlock mb = reader.readMatrixFromInputStream(is, rows, cols, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), (long)rows*cols);
//convert to double array
ret = DataConverter.convertToDoubleMatrix( mb );
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
index e29cca4..498cd24 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
@@ -28,7 +28,6 @@ import java.util.Map.Entry;
import org.apache.sysml.api.DMLException;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
import org.apache.sysml.runtime.controlprogram.Program;
@@ -168,9 +167,8 @@ public class PreparedScript
if( !_inVarnames.contains(varname) )
throw new DMLException("Unspecified input variable: "+varname);
- DMLConfig conf = ConfigurationManager.getConfig();
- String scratch_space = conf.getTextValue(DMLConfig.SCRATCH_SPACE);
- int blocksize = conf.getIntValue(DMLConfig.DEFAULT_BLOCK_SIZE);
+ String scratch_space = ConfigurationManager.getScratchSpace();
+ int blocksize = ConfigurationManager.getBlocksize();
//create new matrix object
MatrixCharacteristics mc = new MatrixCharacteristics(matrix.getNumRows(), matrix.getNumColumns(), blocksize, blocksize);
@@ -233,8 +231,7 @@ public class PreparedScript
if( !_inVarnames.contains(varname) )
throw new DMLException("Unspecified input variable: "+varname);
- DMLConfig conf = ConfigurationManager.getConfig();
- String scratch_space = conf.getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratch_space = ConfigurationManager.getScratchSpace();
//create new frame object
FrameObject fo = new FrameObject(scratch_space+"/"+varname);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/conf/CompilerConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/conf/CompilerConfig.java b/src/main/java/org/apache/sysml/conf/CompilerConfig.java
new file mode 100644
index 0000000..d0c7f7c
--- /dev/null
+++ b/src/main/java/org/apache/sysml/conf/CompilerConfig.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.conf;
+
+import java.util.HashMap;
+
+import org.apache.sysml.hops.OptimizerUtils;
+
+/**
+ * Basic wrapper for all compiler configurations that are configured
+ * dynamically on a per script invocation basis. This allows us to
+ * provide thread-local compiler configurations to prevent side-effects
+ * between multiple scripts running in the same JVM process.
+ *
+ */
+public class CompilerConfig
+{
+ public enum ConfigType {
+ //Configured compiler optimization level (see OptimizerUtils for defails)
+ OPT_LEVEL,
+ //Configured or automatically determined binary matrix blocksize
+ BLOCK_SIZE,
+ //Enables parallel read/write of text (textcell, csv, mm) and binary formats
+ PARALLEL_CP_READ_TEXTFORMATS,
+ PARALLEL_CP_WRITE_TEXTFORMATS,
+ PARALLEL_CP_READ_BINARYFORMATS,
+ PARALLEL_CP_WRITE_BINARYFORMATS,
+ //Enables multi-threaded operations for mm, mmchain, and tsmm, rand, wdivmm,
+ //wsloss, wumm, wcemm, uagg, tak, and groupedaggregate.
+ PARALLEL_CP_MATRIX_OPERATIONS,
+ //Enables multi-threaded local or distributed remote parfor operators. Otherwise
+ //parfor is restricted to parfor local with par=1.
+ PARALLEL_LOCAL_OR_REMOTE_PARFOR,
+ //Enables dynamic re-compilation of lops/instructions. If enabled, we recompile
+ //each program block that contains at least one hop that requires re-compilation
+ //(e.g., unknown statistics during compilation, or program blocks in functions).
+ ALLOW_DYN_RECOMPILATION,
+ ALLOW_PARALLEL_DYN_RECOMPILATION,
+ //Enables to put operations with data-dependent output size into individual
+ //statement blocks / program blocks. Since recompilation is done on the granularity
+ //of program blocks this enables recompilation of subsequent operations according
+ //to the actual output size. This rewrite might limit the opportunity for piggybacking
+ //and therefore should only be applied if dyanmic recompilation is enabled as well.
+ ALLOW_INDIVIDUAL_SB_SPECIFIC_OPS,
+ //Enables common subexpression elimination in dags for persistent reads based on
+ //filenames and other relevant read meta data. Disabled for jmlc to allow binding of
+ //in-memory objects without specifying read properties.
+ ALLOW_CSE_PERSISTENT_READS,
+
+ //Global parser configuration (dml/pydml) to skip errors on unspecified args
+ // (modified by mlcontext / jmlc)
+ IGNORE_UNSPECIFIED_ARGS,
+ //Data expression configuration (modified by mlcontext, jmlc apis); no read of meta
+ //data on mlcontext (local) /jmlc (global); ignore unknowns on jmlc
+ IGNORE_READ_WRITE_METADATA, // global skip meta data reads
+ REJECT_READ_WRITE_UNKNOWNS, // ignore missing meta data
+ }
+
+ //default flags (exposed for testing purposes only)
+ public static boolean FLAG_DYN_RECOMPILE = true;
+ public static boolean FLAG_PARREAD_TEXT = true;
+
+ private HashMap<ConfigType, Boolean> _bmap = null;
+ private HashMap<ConfigType, Integer> _imap = null;
+
+ public CompilerConfig() {
+ _bmap = new HashMap<ConfigType, Boolean>();
+ _bmap.put(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, FLAG_PARREAD_TEXT);
+ _bmap.put(ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS, true);
+ _bmap.put(ConfigType.PARALLEL_CP_READ_BINARYFORMATS, true);
+ _bmap.put(ConfigType.PARALLEL_CP_WRITE_BINARYFORMATS, true);
+ _bmap.put(ConfigType.PARALLEL_CP_MATRIX_OPERATIONS, true);
+ _bmap.put(ConfigType.PARALLEL_LOCAL_OR_REMOTE_PARFOR, true);
+ _bmap.put(ConfigType.ALLOW_DYN_RECOMPILATION, FLAG_DYN_RECOMPILE);
+ _bmap.put(ConfigType.ALLOW_PARALLEL_DYN_RECOMPILATION, FLAG_DYN_RECOMPILE);
+ _bmap.put(ConfigType.ALLOW_INDIVIDUAL_SB_SPECIFIC_OPS, FLAG_DYN_RECOMPILE);
+ _bmap.put(ConfigType.ALLOW_CSE_PERSISTENT_READS, true);
+ _bmap.put(ConfigType.IGNORE_UNSPECIFIED_ARGS, false);
+ _bmap.put(ConfigType.IGNORE_READ_WRITE_METADATA, false);
+ _bmap.put(ConfigType.REJECT_READ_WRITE_UNKNOWNS, true);
+
+ _imap = new HashMap<CompilerConfig.ConfigType, Integer>();
+ _imap.put(ConfigType.BLOCK_SIZE, OptimizerUtils.DEFAULT_BLOCKSIZE);
+ _imap.put(ConfigType.OPT_LEVEL, OptimizerUtils.DEFAULT_OPTLEVEL.ordinal());
+ }
+
+ @SuppressWarnings("unchecked")
+ public CompilerConfig( CompilerConfig conf ) {
+ _bmap = (HashMap<ConfigType, Boolean>) conf._bmap.clone();
+ _imap = (HashMap<ConfigType, Integer>) conf._imap.clone();
+ }
+
+ /**
+ *
+ * @param key
+ * @param value
+ */
+ public void set( ConfigType key, boolean value ) {
+ _bmap.put(key, value);
+ }
+
+ /**
+ *
+ * @param key
+ * @param value
+ */
+ public void set( ConfigType key, int value ) {
+ _imap.put(key, value);
+ }
+
+ /**
+ *
+ * @param key
+ * @return
+ */
+ public boolean getBool( ConfigType key ) {
+ if( _bmap.containsKey(key) )
+ return _bmap.get(key);
+ return false;
+ }
+
+ /**
+ *
+ * @param key
+ * @return
+ */
+ public int getInt( ConfigType key ) {
+ if( _imap.containsKey(key) )
+ return _imap.get(key);
+ return -1;
+ }
+
+
+ /**
+ *
+ */
+ public CompilerConfig clone() {
+ return new CompilerConfig(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/conf/ConfigurationManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/conf/ConfigurationManager.java b/src/main/java/org/apache/sysml/conf/ConfigurationManager.java
index d6bf881..7227bbc 100644
--- a/src/main/java/org/apache/sysml/conf/ConfigurationManager.java
+++ b/src/main/java/org/apache/sysml/conf/ConfigurationManager.java
@@ -20,6 +20,7 @@
package org.apache.sysml.conf;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
@@ -31,51 +32,194 @@ import org.apache.hadoop.mapred.JobConf;
*/
public class ConfigurationManager
{
+ /** Global cached job conf for read-only operations */
+ private static JobConf _rJob = null;
- private static DMLConfig _conf = null; //read systemml configuration
- private static JobConf _rJob = null; //cached job conf for read-only operations
+ /** Global DML configuration (read or defaults) */
+ private static DMLConfig _dmlconf = null;
- static{
+ /** Local DML configuration for thread-local config updates */
+ private static ThreadLocalDMLConfig _ldmlconf = new ThreadLocalDMLConfig(null);
+
+ /** Global compiler configuration (defaults) */
+ private static CompilerConfig _cconf = null;
+
+ /** Local compiler configuration for thead-local config updates */
+ private static ThreadLocalCompilerConfig _lcconf = new ThreadLocalCompilerConfig(null);
+
+ //global static initialization
+ static {
_rJob = new JobConf();
+ _cconf = new CompilerConfig();
+ }
+
+
+ /**
+ * Returns a cached JobConf object, intended for global use by all operations
+ * with read-only access to job conf. This prevents to read the hadoop conf files
+ * over and over again from classpath. However,
+ *
+ * @return
+ */
+ public static JobConf getCachedJobConf() {
+ return _rJob;
+ }
+
+ /**
+ *
+ * @param job
+ */
+ public static void setCachedJobConf(JobConf job) {
+ _rJob = job;
}
+ /**
+ * Sets a global configuration as a basis for any thread-local configurations.
+ * NOTE: This global configuration should never be accessed directly but only
+ * through its thread-local derivatives.
+ *
+ * @param conf
+ */
+ public synchronized static void setGlobalConfig( DMLConfig conf ) {
+ _dmlconf = conf;
+
+ //reinitialize thread-local dml configs
+ _ldmlconf = new ThreadLocalDMLConfig(_dmlconf);
+ }
/**
+ * Sets the current thread-local dml configuration to the given config.
*
* @param conf
*/
- public synchronized static void setConfig( DMLConfig conf )
- {
- _conf = conf;
+ public static void setLocalConfig( DMLConfig conf ) {
+ _ldmlconf.set(conf);
}
/**
+ * Gets the current thread-local dml configuration.
*
* @return
*/
- public synchronized static DMLConfig getConfig()
- {
- return _conf;
+ public static DMLConfig getDMLConfig() {
+ return _ldmlconf.get();
}
- /**
- * Returns a cached JobConf object, intended for global use by all operations
- * with read-only access to job conf. This prevents to read the hadoop conf files
- * over and over again from classpath. However,
- *
- * @return
- */
- public static JobConf getCachedJobConf()
- {
- return _rJob;
+ /**
+ *
+ * @param conf
+ */
+ public synchronized static void setGlobalConfig( CompilerConfig conf ) {
+ _cconf = conf;
+
+ //reinitialize thread-local compiler configs
+ _lcconf = new ThreadLocalCompilerConfig(_cconf);
}
/**
+ * Sets the current thread-local compiler configuration to the given config.
*
- * @param job
+ * @param conf
*/
- public static void setCachedJobConf(JobConf job)
- {
- _rJob = job;
+ public static void setLocalConfig( CompilerConfig conf ) {
+ _lcconf.set(conf);
}
+
+ /**
+ * Removes the thread-local dml and compiler configurations, leading to
+ * a reinitialization on the next get unless set in between.
+ */
+ public static void clearLocalConfigs() {
+ _ldmlconf.remove();
+ _lcconf.remove();
+ }
+
+ /**
+ * Gets the current thread-local compiler configuration.
+ *
+ * @return
+ */
+ public static CompilerConfig getCompilerConfig() {
+ return _lcconf.get();
+ }
+
+ /**
+ * Get a boolean compiler config in a robust manner,
+ * returning false if config not existing.
+ *
+ * @param key
+ * @return
+ */
+ public static boolean getCompilerConfigFlag(ConfigType key) {
+ CompilerConfig cconf = getCompilerConfig();
+ return (cconf!=null) ? cconf.getBool(key) : false;
+ }
+
+ /////////////////////////////////////
+ // shorthand methods for common local configurations
+
+ public static String getScratchSpace() {
+ return getDMLConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
+ }
+
+ public static int getBlocksize() {
+ return getCompilerConfig().getInt(ConfigType.BLOCK_SIZE);
+ }
+
+ public static int getNumReducers() {
+ return getDMLConfig().getIntValue(DMLConfig.NUM_REDUCERS);
+ }
+
+ public static boolean isDynamicRecompilation() {
+ return getCompilerConfigFlag(ConfigType.ALLOW_DYN_RECOMPILATION);
+ }
+
+ public static boolean isParallelMatrixOperations() {
+ return getCompilerConfigFlag(ConfigType.PARALLEL_CP_MATRIX_OPERATIONS);
+ }
+
+ public static boolean isParallelParFor() {
+ return getCompilerConfigFlag(ConfigType.PARALLEL_LOCAL_OR_REMOTE_PARFOR);
+ }
+
+
+ ///////////////////////////////////////
+ // Thread-local classes
+
+ /**
+ *
+ */
+ private static class ThreadLocalDMLConfig extends ThreadLocal<DMLConfig> {
+ private DMLConfig _source = null;
+
+ private ThreadLocalDMLConfig( DMLConfig source ) {
+ _source = source;
+ }
+
+ @Override
+ protected DMLConfig initialValue() {
+ //currently initialize by reference to avoid unnecessary deep copy via clone.
+ if( _source != null )
+ return _source;
+ return null;
+ }
+ }
+
+ /**
+ *
+ */
+ private static class ThreadLocalCompilerConfig extends ThreadLocal<CompilerConfig> {
+ private CompilerConfig _source = null;
+
+ private ThreadLocalCompilerConfig( CompilerConfig source ) {
+ _source = source;
+ }
+
+ @Override
+ protected CompilerConfig initialValue() {
+ if( _source != null )
+ return _source.clone();
+ return null;
+ }
+ };
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/conf/DMLConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/conf/DMLConfig.java b/src/main/java/org/apache/sysml/conf/DMLConfig.java
index df682c8..d65bba1 100644
--- a/src/main/java/org/apache/sysml/conf/DMLConfig.java
+++ b/src/main/java/org/apache/sysml/conf/DMLConfig.java
@@ -43,6 +43,7 @@ import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.parser.ParseException;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.util.LocalFileUtils;
@@ -81,22 +82,22 @@ public class DMLConfig
//configuration default values
private static HashMap<String, String> _defaultVals = null;
- private String config_file_name = null;
- private Element xml_root = null;
+ private String _fileName = null;
+ private Element _xmlRoot = null;
static
{
_defaultVals = new HashMap<String, String>();
- _defaultVals.put(LOCAL_TMP_DIR, "/tmp/systemml" );
- _defaultVals.put(SCRATCH_SPACE, "scratch_space" );
- _defaultVals.put(OPTIMIZATION_LEVEL, "2" );
- _defaultVals.put(NUM_REDUCERS, "10" );
- _defaultVals.put(JVM_REUSE, "false" );
- _defaultVals.put(DEFAULT_BLOCK_SIZE, "1000" );
- _defaultVals.put(YARN_APPMASTER, "false" );
- _defaultVals.put(YARN_APPMASTERMEM, "2048" );
- _defaultVals.put(YARN_MAPREDUCEMEM, "-1" );
- _defaultVals.put(YARN_APPQUEUE, "default" );
+ _defaultVals.put(LOCAL_TMP_DIR, "/tmp/systemml" );
+ _defaultVals.put(SCRATCH_SPACE, "scratch_space" );
+ _defaultVals.put(OPTIMIZATION_LEVEL, String.valueOf(OptimizerUtils.DEFAULT_OPTLEVEL.ordinal()) );
+ _defaultVals.put(NUM_REDUCERS, "10" );
+ _defaultVals.put(JVM_REUSE, "false" );
+ _defaultVals.put(DEFAULT_BLOCK_SIZE, String.valueOf(OptimizerUtils.DEFAULT_BLOCKSIZE) );
+ _defaultVals.put(YARN_APPMASTER, "false" );
+ _defaultVals.put(YARN_APPMASTERMEM, "2048" );
+ _defaultVals.put(YARN_MAPREDUCEMEM, "-1" );
+ _defaultVals.put(YARN_APPQUEUE, "default" );
_defaultVals.put(CP_PARALLEL_MATRIXMULT, "true" );
_defaultVals.put(CP_PARALLEL_TEXTIO, "true" );
}
@@ -128,7 +129,7 @@ public class DMLConfig
public DMLConfig(String fileName, boolean silent)
throws ParseException, FileNotFoundException
{
- config_file_name = fileName;
+ _fileName = fileName;
try {
parseConfig();
} catch (FileNotFoundException fnfe) {
@@ -144,15 +145,9 @@ public class DMLConfig
LOCAL_MR_MODE_STAGING_DIR = getTextValue(LOCAL_TMP_DIR) + "/hadoop/mapred/staging";
}
-
- public String getConfig_file_name()
- {
- return config_file_name;
- }
-
public DMLConfig( Element root )
{
- xml_root = root;
+ _xmlRoot = root;
}
public void merge(DMLConfig otherConfig)
@@ -163,7 +158,7 @@ public class DMLConfig
try {
// for each element in otherConfig, either overwrite existing value OR add to defaultConfig
- NodeList otherConfigNodeList = otherConfig.xml_root.getChildNodes();
+ NodeList otherConfigNodeList = otherConfig._xmlRoot.getChildNodes();
if (otherConfigNodeList != null && otherConfigNodeList.getLength() > 0){
for (int i=0; i<otherConfigNodeList.getLength(); i++){
org.w3c.dom.Node optionalConfigNode = otherConfigNodeList.item(i);
@@ -174,18 +169,18 @@ public class DMLConfig
String paramName = optionalConfigNode.getNodeName();
String paramValue = ((Element)optionalConfigNode).getFirstChild().getNodeValue();
- if (this.xml_root.getElementsByTagName(paramName) != null)
+ if (_xmlRoot.getElementsByTagName(paramName) != null)
LOG.info("Updating " + paramName + " with value " + paramValue);
else
LOG.info("Defining new attribute" + paramName + " with value " + paramValue);
- DMLConfig.setTextValue(this.xml_root, paramName, paramValue);
+ DMLConfig.setTextValue(_xmlRoot, paramName, paramValue);
}
}
} // end if (otherConfigNodeList != null && otherConfigNodeList.getLength() > 0){
} catch (Exception e){
LOG.error("Failed in merge default config file with optional config file",e);
- throw new ParseException("ERROR: error merging config file" + otherConfig.config_file_name + " with " + config_file_name);
+ throw new ParseException("ERROR: error merging config file" + otherConfig._fileName + " with " + _fileName);
}
}
@@ -201,23 +196,23 @@ public class DMLConfig
factory.setIgnoringComments(true); //ignore XML comments
DocumentBuilder builder = factory.newDocumentBuilder();
Document domTree = null;
- if (config_file_name.startsWith("hdfs:") ||
- config_file_name.startsWith("gpfs:") ) // config file from DFS
+ if (_fileName.startsWith("hdfs:") ||
+ _fileName.startsWith("gpfs:") ) // config file from DFS
{
- if( !LocalFileUtils.validateExternalFilename(config_file_name, true) )
+ if( !LocalFileUtils.validateExternalFilename(_fileName, true) )
throw new IOException("Invalid (non-trustworthy) hdfs config filename.");
FileSystem DFS = FileSystem.get(ConfigurationManager.getCachedJobConf());
- Path configFilePath = new Path(config_file_name);
+ Path configFilePath = new Path(_fileName);
domTree = builder.parse(DFS.open(configFilePath));
}
else // config from local file system
{
- if( !LocalFileUtils.validateExternalFilename(config_file_name, false) )
+ if( !LocalFileUtils.validateExternalFilename(_fileName, false) )
throw new IOException("Invalid (non-trustworthy) local config filename.");
- domTree = builder.parse(config_file_name);
+ domTree = builder.parse(_fileName);
}
- xml_root = domTree.getDocumentElement();
+ _xmlRoot = domTree.getDocumentElement();
}
/**
@@ -229,7 +224,7 @@ public class DMLConfig
public String getTextValue(String tagName)
{
//get the actual value
- String retVal = (xml_root!=null)?getTextValue(xml_root,tagName):null;
+ String retVal = (_xmlRoot!=null)?getTextValue(_xmlRoot,tagName):null;
if (retVal == null)
{
@@ -291,8 +286,8 @@ public class DMLConfig
* @param paramValue
*/
public void setTextValue(String paramName, String paramValue) throws DMLRuntimeException {
- if(this.xml_root != null)
- DMLConfig.setTextValue(this.xml_root, paramName, paramValue);
+ if(_xmlRoot != null)
+ DMLConfig.setTextValue(_xmlRoot, paramName, paramValue);
else {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setIgnoringComments(true); //ignore XML comments
@@ -301,7 +296,7 @@ public class DMLConfig
builder = factory.newDocumentBuilder();
String configString = "<root><" + paramName + ">"+paramValue+"</" + paramName + "></root>";
Document domTree = builder.parse(new ByteArrayInputStream(configString.getBytes("UTF-8")));
- this.xml_root = domTree.getDocumentElement();
+ _xmlRoot = domTree.getDocumentElement();
} catch (Exception e) {
throw new DMLRuntimeException("Unable to set config value", e);
}
@@ -319,11 +314,11 @@ public class DMLConfig
HashMap<String, String> ret = new HashMap<String, String>();
//check for non-existing config xml tree
- if( xml_root == null )
+ if( _xmlRoot == null )
return ret;
//get all mapred.* and mapreduce.* tag / value pairs
- NodeList list = xml_root.getElementsByTagName("*");
+ NodeList list = _xmlRoot.getElementsByTagName("*");
for( int i=0; list!=null && i<list.getLength(); i++ ) {
if( list.item(i) instanceof Element &&
( ((Element)list.item(i)).getNodeName().startsWith(PREFIX_MAPRED)
@@ -353,7 +348,7 @@ public class DMLConfig
transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
//transformer.setOutputProperty(OutputKeys.INDENT, "yes");
StreamResult result = new StreamResult(new StringWriter());
- DOMSource source = new DOMSource(xml_root);
+ DOMSource source = new DOMSource(_xmlRoot);
transformer.transform(source, result);
ret = result.getWriter().toString();
}
@@ -487,14 +482,14 @@ public class DMLConfig
public void updateYarnMemorySettings(String amMem, String mrMem)
{
//app master memory
- NodeList list1 = xml_root.getElementsByTagName(YARN_APPMASTERMEM);
+ NodeList list1 = _xmlRoot.getElementsByTagName(YARN_APPMASTERMEM);
if (list1 != null && list1.getLength() > 0) {
Element elem = (Element) list1.item(0);
elem.getFirstChild().setNodeValue(String.valueOf(amMem));
}
//mapreduce memory
- NodeList list2 = xml_root.getElementsByTagName(YARN_MAPREDUCEMEM);
+ NodeList list2 = _xmlRoot.getElementsByTagName(YARN_MAPREDUCEMEM);
if (list2 != null && list2.getLength() > 0) {
Element elem = (Element) list2.item(0);
elem.getFirstChild().setNodeValue(String.valueOf(mrMem));
@@ -509,7 +504,7 @@ public class DMLConfig
public void makeQualifiedScratchSpacePath()
throws IOException
{
- NodeList list2 = xml_root.getElementsByTagName(SCRATCH_SPACE);
+ NodeList list2 = _xmlRoot.getElementsByTagName(SCRATCH_SPACE);
if (list2 != null && list2.getLength() > 0) {
Element elem = (Element) list2.item(0);
@@ -526,9 +521,18 @@ public class DMLConfig
* @param key
* @return
*/
- public static String getDefaultTextValue( String key )
- {
+ public static String getDefaultTextValue( String key ) {
return _defaultVals.get( key );
}
+ /**
+ *
+ */
+ public DMLConfig clone() {
+ DMLConfig conf = new DMLConfig();
+ conf._fileName = _fileName;
+ conf._xmlRoot = (Element) _xmlRoot.cloneNode(true);
+
+ return conf;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
index f8e1541..02d32ff 100644
--- a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
@@ -21,6 +21,7 @@ package org.apache.sysml.hops;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.lops.Aggregate;
import org.apache.sysml.lops.Binary;
@@ -449,7 +450,7 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE ) {
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE ) {
setRequiresRecompile();
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
index 4c4d7c8..0152d54 100644
--- a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
@@ -19,6 +19,7 @@
package org.apache.sysml.hops;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
import org.apache.sysml.hops.Hop.MultiThreadedHop;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
@@ -441,7 +442,7 @@ public class AggUnaryOp extends Hop implements MultiThreadedHop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE ) {
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE ) {
setRequiresRecompile();
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/BinaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/BinaryOp.java b/src/main/java/org/apache/sysml/hops/BinaryOp.java
index 7fd0e97..e434197 100644
--- a/src/main/java/org/apache/sysml/hops/BinaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/BinaryOp.java
@@ -19,6 +19,7 @@
package org.apache.sysml.hops;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.lops.Aggregate;
import org.apache.sysml.lops.AppendGAlignedSP;
@@ -766,7 +767,7 @@ public class BinaryOp extends Hop
if( dimsKnown() && _nnz<0 ) //never after inference
nnz = -1;
- if((op==OpOp2.CBIND || op==OpOp2.RBIND) && !OptimizerUtils.ALLOW_DYN_RECOMPILATION && !(getDataType()==DataType.SCALAR) ) {
+ if((op==OpOp2.CBIND || op==OpOp2.RBIND) && !ConfigurationManager.isDynamicRecompilation() && !(getDataType()==DataType.SCALAR) ) {
ret = OptimizerUtils.DEFAULT_SIZE;
}
else
@@ -997,7 +998,7 @@ public class BinaryOp extends Hop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE ) {
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE ) {
setRequiresRecompile();
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/DataGenOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/DataGenOp.java b/src/main/java/org/apache/sysml/hops/DataGenOp.java
index c584d9c..4c81f77 100644
--- a/src/main/java/org/apache/sysml/hops/DataGenOp.java
+++ b/src/main/java/org/apache/sysml/hops/DataGenOp.java
@@ -25,14 +25,12 @@ import java.util.Map.Entry;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.hops.Hop.MultiThreadedHop;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.DataGen;
import org.apache.sysml.lops.LopsException;
import org.apache.sysml.lops.LopProperties.ExecType;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.DataIdentifier;
import org.apache.sysml.parser.DataExpression;
import org.apache.sysml.parser.Expression.DataType;
@@ -109,7 +107,7 @@ public class DataGenOp extends Hop implements MultiThreadedHop
_sparsity = Double.valueOf(((LiteralOp)inputParameters.get(DataExpression.RAND_SPARSITY)).getName());
//generate base dir
- String scratch = ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratch = ConfigurationManager.getScratchSpace();
_baseDir = scratch + Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID() + Lop.FILE_SEPARATOR +
Lop.FILE_SEPARATOR + ProgramConverter.CP_ROOT_THREAD_ID + Lop.FILE_SEPARATOR;
@@ -165,8 +163,8 @@ public class DataGenOp extends Hop implements MultiThreadedHop
rnd.getOutputParameters().setDimensions(
getDim1(), getDim2(),
//robust handling for blocksize (important for -exec singlenode; otherwise incorrect results)
- (getRowsInBlock()>0)?getRowsInBlock():DMLTranslator.DMLBlockSize,
- (getColsInBlock()>0)?getColsInBlock():DMLTranslator.DMLBlockSize,
+ (getRowsInBlock()>0)?getRowsInBlock():ConfigurationManager.getBlocksize(),
+ (getColsInBlock()>0)?getColsInBlock():ConfigurationManager.getBlocksize(),
//actual rand nnz might differ (in cp/mr they are corrected after execution)
(_op==DataGenMethod.RAND && et==ExecType.SPARK && getNnz()!=0) ? -1 : getNnz(),
getUpdateInPlace());
@@ -225,7 +223,8 @@ public class DataGenOp extends Hop implements MultiThreadedHop
protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
{
if ( _op == DataGenMethod.RAND && dimsKnown() ) {
- long numBlocks = (long) (Math.ceil((double)dim1/DMLTranslator.DMLBlockSize) * Math.ceil((double)dim2/DMLTranslator.DMLBlockSize));
+ long numBlocks = (long) (Math.ceil((double)dim1/ConfigurationManager.getBlocksize())
+ * Math.ceil((double)dim2/ConfigurationManager.getBlocksize()));
return 32 + numBlocks*8.0; // 32 bytes of overhead for an array of long & numBlocks long values.
}
else
@@ -296,7 +295,7 @@ public class DataGenOp extends Hop implements MultiThreadedHop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
setRequiresRecompile();
//always force string initialization into CP (not supported in MR)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/DataOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/DataOp.java b/src/main/java/org/apache/sysml/hops/DataOp.java
index ca447ec..bdaa3b9 100644
--- a/src/main/java/org/apache/sysml/hops/DataOp.java
+++ b/src/main/java/org/apache/sysml/hops/DataOp.java
@@ -19,6 +19,8 @@
package org.apache.sysml.hops;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.lops.Data;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.LopsException;
@@ -461,14 +463,14 @@ public class DataOp extends Hop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE ) {
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE ) {
setRequiresRecompile();
}
}
else //READ
{
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && letype==REMOTE
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && letype==REMOTE
&& (_recompileRead || _requiresCheckpoint) )
{
setRequiresRecompile();
@@ -541,7 +543,8 @@ public class DataOp extends Hop
//with multiple piggybacked csvreblock of the same input w/ unknown input sizes
DataOp that2 = (DataOp)that;
- boolean ret = ( OptimizerUtils.ALLOW_CSE_PERSISTENT_READS
+ boolean ret = ( OptimizerUtils.ALLOW_COMMON_SUBEXPRESSION_ELIMINATION
+ && ConfigurationManager.getCompilerConfigFlag(ConfigType.ALLOW_CSE_PERSISTENT_READS)
&&_dataop == that2._dataop
&& _dataop == DataOpTypes.PERSISTENTREAD
&& _fileName.equals(that2._fileName)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index a175817..adf1828 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -27,7 +27,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.lops.CSVReBlock;
import org.apache.sysml.lops.Checkpoint;
import org.apache.sysml.lops.Data;
@@ -414,7 +413,7 @@ public abstract class Hop
{
Lop offset = null;
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && hop.dimsKnown() )
+ if( ConfigurationManager.isDynamicRecompilation() && hop.dimsKnown() )
{
// If dynamic recompilation is enabled and dims are known, we can replace the ncol with
// a literal in order to increase the piggybacking potential. This is safe because append
@@ -1798,7 +1797,7 @@ public abstract class Hop
public String constructBaseDir()
{
StringBuilder sb = new StringBuilder();
- sb.append( ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE) );
+ sb.append( ConfigurationManager.getScratchSpace() );
sb.append( Lop.FILE_SEPARATOR );
sb.append( Lop.PROCESS_PREFIX );
sb.append( DMLScript.getUUID() );
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/IndexingOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/IndexingOp.java b/src/main/java/org/apache/sysml/hops/IndexingOp.java
index fe32370..4b9901c 100644
--- a/src/main/java/org/apache/sysml/hops/IndexingOp.java
+++ b/src/main/java/org/apache/sysml/hops/IndexingOp.java
@@ -19,6 +19,7 @@
package org.apache.sysml.hops;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.lops.Aggregate;
@@ -363,7 +364,7 @@ public class IndexingOp extends Hop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
setRequiresRecompile();
return _etype;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java b/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
index dc4d912..07091e7 100644
--- a/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
+++ b/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
@@ -19,6 +19,7 @@
package org.apache.sysml.hops;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.lops.Binary;
import org.apache.sysml.lops.Group;
import org.apache.sysml.lops.LeftIndex;
@@ -29,7 +30,6 @@ import org.apache.sysml.lops.UnaryCP;
import org.apache.sysml.lops.ZeroOut;
import org.apache.sysml.lops.LopProperties.ExecType;
import org.apache.sysml.lops.UnaryCP.OperationTypes;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
@@ -133,8 +133,8 @@ public class LeftIndexingOp extends Hop
OperationTypes.CAST_AS_MATRIX,
DataType.MATRIX, ValueType.DOUBLE);
rightInput.getOutputParameters().setDimensions( (long)1, (long)1,
- (long)DMLTranslator.DMLBlockSize,
- (long)DMLTranslator.DMLBlockSize,
+ (long)ConfigurationManager.getBlocksize(),
+ (long)ConfigurationManager.getBlocksize(),
(long)-1);
}
else
@@ -194,7 +194,7 @@ public class LeftIndexingOp extends Hop
Lop rightInput = right.constructLops();
if (isRightHandSideScalar()) {
rightInput = new UnaryCP(rightInput, OperationTypes.CAST_AS_MATRIX, DataType.MATRIX, ValueType.DOUBLE);
- long bsize = (long)DMLTranslator.DMLBlockSize;
+ long bsize = ConfigurationManager.getBlocksize();
rightInput.getOutputParameters().setDimensions( 1, 1, bsize, bsize, -1);
}
@@ -391,7 +391,7 @@ public class LeftIndexingOp extends Hop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
setRequiresRecompile();
return _etype;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
index 66ab82b..b4ab7e8 100644
--- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
@@ -26,6 +26,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.CompilerConfig;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.Hop.DataOpTypes;
@@ -62,12 +64,20 @@ public class OptimizerUtils
*/
public static double MEM_UTIL_FACTOR = 0.7d;
+ /** Default blocksize if unspecified or for testing purposes */
+ public static final int DEFAULT_BLOCKSIZE = 1000;
+
+ /** Default optimization level if unspecified */
+ public static final OptimizationLevel DEFAULT_OPTLEVEL =
+ OptimizationLevel.O2_LOCAL_MEMORY_DEFAULT;
+
/**
- * Default memory size, which is used the actual estimate can not be computed
- * -- for example, when input/output dimensions are unknown. In case of ROBUST,
- * the default is set to a large value so that operations are scheduled on MR.
+ * Default memory size, which is used if the actual estimate can not be computed
+ * e.g., when input/output dimensions are unknown. The default is set to a large
+ * value so that operations are scheduled on MR while avoiding overflows as well.
*/
- public static double DEFAULT_SIZE;
+ public static double DEFAULT_SIZE = getDefaultSize();
+
public static final long DOUBLE_SIZE = 8;
public static final long INT_SIZE = 4;
@@ -80,26 +90,6 @@ public class OptimizerUtils
public static final long MAX_NUMCELLS_CP_DENSE = Integer.MAX_VALUE;
public static final long MAX_NNZ_CP_SPARSE = (MatrixBlock.DEFAULT_SPARSEBLOCK ==
SparseBlock.Type.MCSR) ? Long.MAX_VALUE : Integer.MAX_VALUE;
-
- /**
- * Enables/disables dynamic re-compilation of lops/instructions.
- * If enabled, we recompile each program block that contains at least
- * one hop that requires re-compilation (e.g., unknown statistics
- * during compilation, or program blocks in functions).
- */
- public static boolean ALLOW_DYN_RECOMPILATION = true;
- public static boolean ALLOW_PARALLEL_DYN_RECOMPILATION = ALLOW_DYN_RECOMPILATION && true;
-
- /**
- * Enables/disables to put operations with data-dependent output
- * size into individual statement blocks / program blocks.
- * Since recompilation is done on the granularity of program blocks
- * this enables recompilation of subsequent operations according
- * to the actual output size. This rewrite might limit the opportunity
- * for piggybacking and therefore should only be applied if
- * dyanmic recompilation is enabled as well.
- */
- public static boolean ALLOW_INDIVIDUAL_SB_SPECIFIC_OPS = ALLOW_DYN_RECOMPILATION && true;
/**
* Enables common subexpression elimination in dags. There is however, a potential tradeoff
@@ -109,13 +99,6 @@ public class OptimizerUtils
public static boolean ALLOW_COMMON_SUBEXPRESSION_ELIMINATION = true;
/**
- * Enables common subexpression elimination in dags for persistent reads based on filenames
- * and other relevant read meta data. Disabled for jmlc to allow binding of in-memory objects
- * without specifying read properties.
- */
- public static boolean ALLOW_CSE_PERSISTENT_READS = ALLOW_COMMON_SUBEXPRESSION_ELIMINATION && true;
-
- /**
* Enables constant folding in dags. Constant folding computes simple expressions of binary
* operations and literals and replaces the hop sub-DAG with a new literal operator.
*/
@@ -188,17 +171,6 @@ public class OptimizerUtils
public static boolean ALLOW_SPLIT_HOP_DAGS = true;
- /**
- * Enables parallel read/write of all text formats (textcell, csv, mm)
- * and binary formats (binary block).
- *
- */
- public static boolean PARALLEL_CP_READ_TEXTFORMATS = true;
- public static boolean PARALLEL_CP_WRITE_TEXTFORMATS = true;
- public static boolean PARALLEL_CP_READ_BINARYFORMATS = true;
- public static boolean PARALLEL_CP_WRITE_BINARYFORMATS = true;
-
-
/**
* Specifies a multiplier computing the degree of parallelism of parallel
@@ -210,19 +182,6 @@ public class OptimizerUtils
public static final double PARALLEL_CP_WRITE_PARALLELISM_MULTIPLIER = 1.0;
/**
- * Enables multi-threaded operations for mm, mmchain, and tsmm,
- * rand, wdivmm, wsloss, wumm, wcemm, uagg, tak, and groupedaggregate.
- *
- */
- public static boolean PARALLEL_CP_MATRIX_OPERATIONS = true;
-
- /**
- * Enables multi-threaded local or distributed remote parfor operators.
- * Otherwise parfor is restricted to parfor local with par=1.
- */
- public static boolean PARALLEL_LOCAL_OR_REMOTE_PARFOR = true;
-
- /**
* Enables the use of CombineSequenceFileInputFormat with splitsize = 2x hdfs blocksize,
* if sort buffer size large enough and parallelism not hurt. This solves to issues:
* (1) it combines small files (depending on producers), and (2) it reduces task
@@ -236,8 +195,6 @@ public class OptimizerUtils
// Optimizer levels //
//////////////////////
- private static OptimizationLevel _optLevel = OptimizationLevel.O2_LOCAL_MEMORY_DEFAULT;
-
/**
* Optimization Types for Compilation
*
@@ -284,15 +241,16 @@ public class OptimizerUtils
};
public static OptimizationLevel getOptLevel() {
- return _optLevel;
+ int optlevel = ConfigurationManager.getCompilerConfig().getInt(ConfigType.OPT_LEVEL);
+ return OptimizationLevel.values()[optlevel];
}
public static boolean isMemoryBasedOptLevel() {
- return (_optLevel != OptimizationLevel.O0_LOCAL_STATIC);
+ return (getOptLevel() != OptimizationLevel.O0_LOCAL_STATIC);
}
public static boolean isOptLevel( OptimizationLevel level ){
- return (_optLevel == level);
+ return (getOptLevel() == level);
}
/**
@@ -300,9 +258,17 @@ public class OptimizerUtils
* @param optlevel
* @throws DMLRuntimeException
*/
- public static void setOptimizationLevel( int optlevel )
+ public static CompilerConfig constructCompilerConfig( DMLConfig dmlconf )
throws DMLRuntimeException
{
+ //create default compiler configuration
+ CompilerConfig cconf = new CompilerConfig();
+
+ //each script sets its own block size, opt level etc
+ cconf.set(ConfigType.BLOCK_SIZE, dmlconf.getIntValue( DMLConfig.DEFAULT_BLOCK_SIZE ));
+
+ //handle optimization level
+ int optlevel = dmlconf.getIntValue(DMLConfig.OPTIMIZATION_LEVEL);
if( optlevel < 0 || optlevel > 5 )
throw new DMLRuntimeException("Error: invalid optimization level '"+optlevel+"' (valid values: 0-5).");
@@ -317,7 +283,7 @@ public class OptimizerUtils
{
// opt level 0: static dimensionality
case 0:
- _optLevel = OptimizationLevel.O0_LOCAL_STATIC;
+ cconf.set(ConfigType.OPT_LEVEL, OptimizationLevel.O0_LOCAL_STATIC.ordinal());
ALLOW_CONSTANT_FOLDING = false;
ALLOW_COMMON_SUBEXPRESSION_ELIMINATION = false;
ALLOW_ALGEBRAIC_SIMPLIFICATION = false;
@@ -328,7 +294,7 @@ public class OptimizerUtils
break;
// opt level 1: memory-based (no advanced rewrites)
case 1:
- _optLevel = OptimizationLevel.O1_LOCAL_MEMORY_MIN;
+ cconf.set(ConfigType.OPT_LEVEL, OptimizationLevel.O1_LOCAL_MEMORY_MIN.ordinal());
ALLOW_CONSTANT_FOLDING = false;
ALLOW_COMMON_SUBEXPRESSION_ELIMINATION = false;
ALLOW_ALGEBRAIC_SIMPLIFICATION = false;
@@ -339,68 +305,71 @@ public class OptimizerUtils
break;
// opt level 2: memory-based (all advanced rewrites)
case 2:
- _optLevel = OptimizationLevel.O2_LOCAL_MEMORY_DEFAULT;
+ cconf.set(ConfigType.OPT_LEVEL, OptimizationLevel.O2_LOCAL_MEMORY_DEFAULT.ordinal());
break;
// opt level 3: resource optimization, time- and memory-based (2 w/ resource optimizat)
case 3:
- _optLevel = OptimizationLevel.O3_LOCAL_RESOURCE_TIME_MEMORY;
+ cconf.set(ConfigType.OPT_LEVEL, OptimizationLevel.O3_LOCAL_RESOURCE_TIME_MEMORY.ordinal());
break;
// opt level 3: global, time- and memory-based (all advanced rewrites)
case 4:
- _optLevel = OptimizationLevel.O4_GLOBAL_TIME_MEMORY;
+ cconf.set(ConfigType.OPT_LEVEL, OptimizationLevel.O4_GLOBAL_TIME_MEMORY.ordinal());
break;
// opt level 4: debug mode (no interfering rewrites)
case 5:
- _optLevel = OptimizationLevel.O5_DEBUG_MODE;
+ cconf.set(ConfigType.OPT_LEVEL, OptimizationLevel.O5_DEBUG_MODE.ordinal());
ALLOW_CONSTANT_FOLDING = false;
ALLOW_COMMON_SUBEXPRESSION_ELIMINATION = false;
ALLOW_ALGEBRAIC_SIMPLIFICATION = false;
ALLOW_INTER_PROCEDURAL_ANALYSIS = false;
ALLOW_BRANCH_REMOVAL = false;
- ALLOW_DYN_RECOMPILATION = false;
ALLOW_SIZE_EXPRESSION_EVALUATION = false;
ALLOW_WORSTCASE_SIZE_EXPRESSION_EVALUATION = false;
ALLOW_RAND_JOB_RECOMPILE = false;
ALLOW_SUM_PRODUCT_REWRITES = false;
ALLOW_SPLIT_HOP_DAGS = false;
+ cconf.set(ConfigType.ALLOW_DYN_RECOMPILATION, false);
+ cconf.set(ConfigType.ALLOW_INDIVIDUAL_SB_SPECIFIC_OPS, false);
break;
}
- setDefaultSize();
//handle parallel text io (incl awareness of thread contention in <jdk8)
- if (!ConfigurationManager.getConfig().getBooleanValue(DMLConfig.CP_PARALLEL_TEXTIO)) {
- PARALLEL_CP_READ_TEXTFORMATS = false;
- PARALLEL_CP_WRITE_TEXTFORMATS = false;
- PARALLEL_CP_READ_BINARYFORMATS = false;
- PARALLEL_CP_WRITE_BINARYFORMATS = false;
+ if (!dmlconf.getBooleanValue(DMLConfig.CP_PARALLEL_TEXTIO)) {
+ cconf.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, false);
+ cconf.set(ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS, false);
+ cconf.set(ConfigType.PARALLEL_CP_READ_BINARYFORMATS, false);
+ cconf.set(ConfigType.PARALLEL_CP_WRITE_BINARYFORMATS, false);
}
else if( InfrastructureAnalyzer.isJavaVersionLessThanJDK8()
&& InfrastructureAnalyzer.getLocalParallelism() > 1 )
{
LOG.warn("Auto-disable multi-threaded text read for 'text' and 'csv' due to thread contention on JRE < 1.8"
- + " (java.version="+ System.getProperty("java.version")+").");
-
- //disable parallel text read
- PARALLEL_CP_READ_TEXTFORMATS = false;
+ + " (java.version="+ System.getProperty("java.version")+").");
+ cconf.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, false);
}
//handle parallel matrix mult / rand configuration
- if (!ConfigurationManager.getConfig().getBooleanValue(DMLConfig.CP_PARALLEL_MATRIXMULT)) {
- PARALLEL_CP_MATRIX_OPERATIONS = false;
+ if (!dmlconf.getBooleanValue(DMLConfig.CP_PARALLEL_MATRIXMULT)) {
+ cconf.set(ConfigType.PARALLEL_CP_MATRIX_OPERATIONS, false);
}
+
+ return cconf;
}
/**
*
*/
- public static void setDefaultSize()
- {
+ public static long getDefaultSize() {
//we need to set default_size larger than any execution context
//memory budget, however, it should not produce overflows on sum
- DEFAULT_SIZE = Math.max( InfrastructureAnalyzer.getLocalMaxMemory(),
- Math.max(InfrastructureAnalyzer.getRemoteMaxMemoryMap(),
- InfrastructureAnalyzer.getRemoteMaxMemoryReduce()));
+ return Math.max( InfrastructureAnalyzer.getLocalMaxMemory(),
+ Math.max(InfrastructureAnalyzer.getRemoteMaxMemoryMap(),
+ InfrastructureAnalyzer.getRemoteMaxMemoryReduce()));
+ }
+
+ public static void resetDefaultSize() {
+ DEFAULT_SIZE = getDefaultSize();
}
/**
@@ -544,7 +513,7 @@ public class OptimizerUtils
*/
public static int getNumReducers( boolean configOnly )
{
- int ret = ConfigurationManager.getConfig().getIntValue(DMLConfig.NUM_REDUCERS);
+ int ret = ConfigurationManager.getNumReducers();
if( !configOnly ) {
ret = Math.min(ret,InfrastructureAnalyzer.getRemoteParallelReduceTasks());
@@ -599,7 +568,7 @@ public class OptimizerUtils
*/
public static int getParallelTextReadParallelism()
{
- if( !PARALLEL_CP_READ_TEXTFORMATS )
+ if( !ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_TEXTFORMATS) )
return 1; // sequential execution
//compute degree of parallelism for parallel text read
@@ -614,7 +583,7 @@ public class OptimizerUtils
*/
public static int getParallelBinaryReadParallelism()
{
- if( !PARALLEL_CP_READ_BINARYFORMATS )
+ if( !ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_BINARYFORMATS) )
return 1; // sequential execution
//compute degree of parallelism for parallel text read
@@ -633,7 +602,7 @@ public class OptimizerUtils
*/
public static int getParallelTextWriteParallelism()
{
- if( !PARALLEL_CP_WRITE_TEXTFORMATS )
+ if( !ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS) )
return 1; // sequential execution
//compute degree of parallelism for parallel text read
@@ -648,7 +617,7 @@ public class OptimizerUtils
*/
public static int getParallelBinaryWriteParallelism()
{
- if( !PARALLEL_CP_WRITE_BINARYFORMATS )
+ if( !ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_WRITE_BINARYFORMATS) )
return 1; // sequential execution
//compute degree of parallelism for parallel text read
@@ -954,7 +923,7 @@ public class OptimizerUtils
}
//apply global multi-threading constraint
- if( !PARALLEL_CP_MATRIX_OPERATIONS ) {
+ if( !ConfigurationManager.isParallelMatrixOperations() ) {
ret = 1;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
index 70915fd..0da75a0 100644
--- a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
+++ b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
@@ -22,6 +22,7 @@ package org.apache.sysml.hops;
import java.util.HashMap;
import java.util.Map.Entry;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop.MultiThreadedHop;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.lops.Aggregate;
@@ -995,7 +996,7 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
setRequiresRecompile();
return _etype;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/QuaternaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/QuaternaryOp.java b/src/main/java/org/apache/sysml/hops/QuaternaryOp.java
index b011b8e..484a84a 100644
--- a/src/main/java/org/apache/sysml/hops/QuaternaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/QuaternaryOp.java
@@ -19,6 +19,7 @@
package org.apache.sysml.hops;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop.MultiThreadedHop;
import org.apache.sysml.lops.Aggregate;
import org.apache.sysml.lops.DataPartition;
@@ -1613,7 +1614,7 @@ public class QuaternaryOp extends Hop implements MultiThreadedHop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
setRequiresRecompile();
return _etype;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/ReorgOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ReorgOp.java b/src/main/java/org/apache/sysml/hops/ReorgOp.java
index cfd2f65..a88a098 100644
--- a/src/main/java/org/apache/sysml/hops/ReorgOp.java
+++ b/src/main/java/org/apache/sysml/hops/ReorgOp.java
@@ -21,6 +21,7 @@ package org.apache.sysml.hops;
import java.util.ArrayList;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.lops.Aggregate;
import org.apache.sysml.lops.Group;
@@ -486,7 +487,7 @@ public class ReorgOp extends Hop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
setRequiresRecompile();
return _etype;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/TernaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/TernaryOp.java b/src/main/java/org/apache/sysml/hops/TernaryOp.java
index 562189d..e353273 100644
--- a/src/main/java/org/apache/sysml/hops/TernaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/TernaryOp.java
@@ -19,6 +19,7 @@
package org.apache.sysml.hops;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.lops.Aggregate;
import org.apache.sysml.lops.CentralMoment;
@@ -784,7 +785,7 @@ public class TernaryOp extends Hop
// Necessary condition for recompilation is unknown dimensions.
// When execType=CP, it is marked for recompilation only when additional
// dimension inputs are provided (and those values are unknown at initial compile time).
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) ) {
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) ) {
if ( _etype==REMOTE || (_etype == ExecType.CP && _dimInputsPresent))
setRequiresRecompile();
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/UnaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/UnaryOp.java b/src/main/java/org/apache/sysml/hops/UnaryOp.java
index 4437cdb..fa0e401 100644
--- a/src/main/java/org/apache/sysml/hops/UnaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/UnaryOp.java
@@ -21,6 +21,7 @@ package org.apache.sysml.hops;
import java.util.ArrayList;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.lops.Aggregate;
import org.apache.sysml.lops.Aggregate.OperationTypes;
import org.apache.sysml.lops.CombineUnary;
@@ -670,7 +671,7 @@ public class UnaryOp extends Hop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
setRequiresRecompile();
//ensure cp exec type for single-node operations
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
index 30cd6d7..00be756 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
@@ -28,14 +28,13 @@ import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.HopsException;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.LopsException;
import org.apache.sysml.parser.DMLProgram;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.DMLUnsupportedOperationException;
import org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlock;
@@ -373,7 +372,7 @@ public abstract class CostEstimator
String varname = iinst.output.getName();
long rlen = iinst.getRows();
long clen = iinst.getCols();
- VarStats vs = new VarStats(rlen, clen, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, rlen*clen, true);
+ VarStats vs = new VarStats(rlen, clen, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), rlen*clen, true);
stats.put(varname, vs);
}
else if( inst instanceof FunctionCallCPInstruction )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java
index b68285f..a4128ac 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java
@@ -23,14 +23,12 @@ import java.util.ArrayList;
import java.util.HashSet;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.lops.DataGen;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.MapMult;
import org.apache.sysml.lops.LopProperties.ExecType;
import org.apache.sysml.lops.MMTSJ.MMTSJType;
import org.apache.sysml.lops.compile.JobType;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.DMLUnsupportedOperationException;
import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
@@ -167,7 +165,7 @@ public class CostEstimatorStaticRuntime extends CostEstimator
boolean localJob = InfrastructureAnalyzer.isLocalMode();
int maxPMap = InfrastructureAnalyzer.getRemoteParallelMapTasks();
int maxPRed = Math.min( InfrastructureAnalyzer.getRemoteParallelReduceTasks(),
- ConfigurationManager.getConfig().getIntValue(DMLConfig.NUM_REDUCERS) );
+ ConfigurationManager.getNumReducers() );
double blocksize = ((double)InfrastructureAnalyzer.getHDFSBlockSize())/(1024*1024);
//correction max number of mappers/reducers on yarn clusters
@@ -633,8 +631,8 @@ public class CostEstimatorStaticRuntime extends CostEstimator
default: {
for( int i=0; i<mapOutIx.length; i++ )
{
- int lret = (int) Math.ceil((double)vs[mapOutIx[i]]._rlen/DMLTranslator.DMLBlockSize)
- *(int) Math.ceil((double)vs[mapOutIx[i]]._clen/DMLTranslator.DMLBlockSize);
+ int lret = (int) Math.ceil((double)vs[mapOutIx[i]]._rlen/ConfigurationManager.getBlocksize())
+ *(int) Math.ceil((double)vs[mapOutIx[i]]._clen/ConfigurationManager.getBlocksize());
ret = Math.max(lret, ret);
}
break;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/globalopt/GDFEnumOptimizer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/GDFEnumOptimizer.java b/src/main/java/org/apache/sysml/hops/globalopt/GDFEnumOptimizer.java
index 678ce92..f3a9188 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/GDFEnumOptimizer.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/GDFEnumOptimizer.java
@@ -24,7 +24,7 @@ import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.DataOp;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.Hop.DataOpTypes;
@@ -44,7 +44,6 @@ import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.lops.LopsException;
import org.apache.sysml.lops.LopProperties.ExecType;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.DMLUnsupportedOperationException;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
@@ -579,8 +578,8 @@ public class GDFEnumOptimizer extends GlobalOptimizer
if( hop!=null )
{
hop.setForcedExecType(null);
- hop.setRowsInBlock(DMLTranslator.DMLBlockSize);
- hop.setColsInBlock(DMLTranslator.DMLBlockSize);
+ hop.setRowsInBlock(ConfigurationManager.getBlocksize());
+ hop.setColsInBlock(ConfigurationManager.getBlocksize());
if( !HopRewriteUtils.alwaysRequiresReblock(hop) ) {
hop.setRequiresReblock(false);
}