You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/03/19 23:21:41 UTC
[3/4] incubator-systemml git commit: [SYSTEMML-584] New thread-local
configuration handling (dml/compiler)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
index a7b4776..dcee148 100644
--- a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
+++ b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java
@@ -31,6 +31,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.BinaryOp;
import org.apache.sysml.hops.DataGenOp;
import org.apache.sysml.hops.DataOp;
@@ -692,7 +693,7 @@ public class InterProceduralAnalysis
MatrixObject mo = new MatrixObject(ValueType.DOUBLE, null);
MatrixCharacteristics mc = new MatrixCharacteristics(
input.getDim1(), input.getDim2(),
- DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize,
+ ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(),
inputSafeNNZ.contains(input.getHopID())?input.getNnz():-1 );
MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
mo.setMetaData(meta);
@@ -873,7 +874,7 @@ public class InterProceduralAnalysis
MatrixObject moOut = new MatrixObject(ValueType.DOUBLE, null);
MatrixCharacteristics mc = new MatrixCharacteristics(
dim1, dim2,
- DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize,
+ ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(),
nnz);
MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
moOut.setMetaData(meta);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
index aa0eb03..3edf7f7 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.wink.json4j.JSONObject;
-
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.DataGenOp;
@@ -62,7 +61,6 @@ import org.apache.sysml.lops.LopsException;
import org.apache.sysml.lops.ReBlock;
import org.apache.sysml.lops.compile.Dag;
import org.apache.sysml.parser.DMLProgram;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.DataExpression;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
@@ -125,15 +123,18 @@ public class Recompiler
private static final long CP_CSV_REBLOCK_UNKNOWN_THRESHOLD_SIZE = (long)256*1024*1024;
private static final long CP_TRANSFORM_UNKNOWN_THRESHOLD_SIZE = (long)1024*1024*1024;
- //reused rewriter for dynamic rewrites during recompile
- private static ProgramRewriter rewriter = new ProgramRewriter(false, true);
+ /** Local reused rewriter for dynamic rewrites during recompile */
+
+ /** Local DML configuration for thread-local config updates */
+ private static ThreadLocal<ProgramRewriter> _rewriter = new ThreadLocal<ProgramRewriter>() {
+ @Override protected ProgramRewriter initialValue() { return new ProgramRewriter(false, true); }
+ };
/**
* Re-initializes the recompiler according to the current optimizer flags.
*/
- public static void reinitRecompiler()
- {
- rewriter = new ProgramRewriter(false, true);
+ public static void reinitRecompiler() {
+ _rewriter.set(new ProgramRewriter(false, true));
}
/**
@@ -190,7 +191,7 @@ public class Recompiler
// dynamic hop rewrites
if( !inplace )
- rewriter.rewriteHopDAGs( hops, null );
+ _rewriter.get().rewriteHopDAGs( hops, null );
// refresh memory estimates (based on updated stats,
// before: init memo table with propagated worst-case estimates,
@@ -211,7 +212,7 @@ public class Recompiler
}
// generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(sb, ConfigurationManager.getConfig());
+ newInst = dag.getJobs(sb, ConfigurationManager.getDMLConfig());
}
// replace thread ids in new instructions
@@ -286,7 +287,7 @@ public class Recompiler
// dynamic hop rewrites
if( !inplace )
- rewriter.rewriteHopDAG( hops, null );
+ _rewriter.get().rewriteHopDAG( hops, null );
// refresh memory estimates (based on updated stats)
MemoTable memo = new MemoTable();
@@ -301,7 +302,7 @@ public class Recompiler
lops.addToDag(dag);
// generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(null, ConfigurationManager.getConfig());
+ newInst = dag.getJobs(null, ConfigurationManager.getDMLConfig());
}
// replace thread ids in new instructions
@@ -363,7 +364,7 @@ public class Recompiler
}
// generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(sb, ConfigurationManager.getConfig());
+ newInst = dag.getJobs(sb, ConfigurationManager.getDMLConfig());
}
// replace thread ids in new instructions
@@ -415,7 +416,7 @@ public class Recompiler
lops.addToDag(dag);
// generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(null, ConfigurationManager.getConfig());
+ newInst = dag.getJobs(null, ConfigurationManager.getDMLConfig());
}
// replace thread ids in new instructions
@@ -461,7 +462,7 @@ public class Recompiler
}
// generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(sb, ConfigurationManager.getConfig());
+ newInst = dag.getJobs(sb, ConfigurationManager.getDMLConfig());
}
// explain recompiled hops / instructions
@@ -508,7 +509,7 @@ public class Recompiler
lops.addToDag(dag);
// generate runtime instructions (incl piggybacking)
- newInst = dag.getJobs(null, ConfigurationManager.getConfig());
+ newInst = dag.getJobs(null, ConfigurationManager.getDMLConfig());
}
// explain recompiled instructions
@@ -1164,7 +1165,7 @@ public class Recompiler
MatrixObject moOut = new MatrixObject(ValueType.DOUBLE, null);
MatrixCharacteristics mc = new MatrixCharacteristics(
dim1, dim2,
- DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize,
+ ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(),
nnz);
MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
moOut.setMetaData(meta);
@@ -1482,7 +1483,7 @@ public class Recompiler
MatrixObject mo = new MatrixObject(ValueType.DOUBLE, null);
MatrixCharacteristics mc = new MatrixCharacteristics(
hop.getDim1(), hop.getDim2(),
- DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize,
+ ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(),
hop.getNnz());
MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,null,null);
mo.setMetaData(meta);
@@ -1925,7 +1926,7 @@ public class Recompiler
long nnz = mc.getNonZeros();
//check valid cp reblock recompilation hook
- if( !OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ if( !ConfigurationManager.isDynamicRecompilation()
|| !OptimizerUtils.isHybridExecutionMode() )
{
return false;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
index 5c36074..cfc5855 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
@@ -25,7 +25,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.HopsException;
import org.apache.sysml.hops.OptimizerUtils;
@@ -109,7 +110,7 @@ public class ProgramRewriter
_sbRuleSet.add( new RewriteRemoveUnnecessaryBranches() ); //dependency: constant folding
if( OptimizerUtils.ALLOW_SPLIT_HOP_DAGS )
_sbRuleSet.add( new RewriteSplitDagUnknownCSVRead() ); //dependency: reblock
- if( OptimizerUtils.ALLOW_INDIVIDUAL_SB_SPECIFIC_OPS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.ALLOW_INDIVIDUAL_SB_SPECIFIC_OPS) )
_sbRuleSet.add( new RewriteSplitDagDataDependentOperators() );
if( OptimizerUtils.ALLOW_AUTO_VECTORIZATION )
_sbRuleSet.add( new RewriteForLoopVectorization() ); //dependency: reblock (reblockop)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
index f23f87d..1d7e1db 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.AggBinaryOp;
import org.apache.sysml.hops.AggUnaryOp;
import org.apache.sysml.hops.BinaryOp;
@@ -44,7 +45,6 @@ import org.apache.sysml.hops.Hop.OpOp2;
import org.apache.sysml.hops.ReorgOp;
import org.apache.sysml.hops.UnaryOp;
import org.apache.sysml.lops.MapMultChain.ChainType;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.DataExpression;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
@@ -612,7 +612,7 @@ public class RewriteAlgebraicSimplificationDynamic extends HopRewriteRule
//create cast to keep same output datatype
UnaryOp cast = new UnaryOp(uhi.getName(), DataType.MATRIX, ValueType.DOUBLE,
OpOp1.CAST_AS_MATRIX, uhi);
- HopRewriteUtils.setOutputParameters(cast, 1, 1, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, -1);
+ HopRewriteUtils.setOutputParameters(cast, 1, 1, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), -1);
//rehang cast under all parents
for( Hop p : parents ) {
@@ -700,7 +700,7 @@ public class RewriteAlgebraicSimplificationDynamic extends HopRewriteRule
//create cast to keep same output datatype
UnaryOp cast = new UnaryOp(uhi.getName(), DataType.MATRIX, ValueType.DOUBLE,
OpOp1.CAST_AS_MATRIX, uhi);
- HopRewriteUtils.setOutputParameters(cast, 1, 1, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, -1);
+ HopRewriteUtils.setOutputParameters(cast, 1, 1, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), -1);
//rehang cast under all parents
for( Hop p : parents ) {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java
index 96b20e0..ce09a6f 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteBlockSizeAndReblock.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.DataOp;
import org.apache.sysml.hops.FunctionOp;
import org.apache.sysml.hops.Hop;
@@ -31,7 +32,6 @@ import org.apache.sysml.hops.Hop.FileFormatTypes;
import org.apache.sysml.hops.Hop.ParamBuiltinOp;
import org.apache.sysml.hops.HopsException;
import org.apache.sysml.hops.ParameterizedBuiltinOp;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.Expression.DataType;
/**
@@ -52,11 +52,11 @@ public class RewriteBlockSizeAndReblock extends HopRewriteRule
//maintain rewrite status
if( isReblockValid() )
- state.setBlocksize(DMLTranslator.DMLBlockSize);
+ state.setBlocksize(ConfigurationManager.getBlocksize());
//perform reblock and blocksize rewrite
for( Hop h : roots )
- rule_BlockSizeAndReblock(h, DMLTranslator.DMLBlockSize);
+ rule_BlockSizeAndReblock(h, ConfigurationManager.getBlocksize());
return roots;
}
@@ -70,10 +70,10 @@ public class RewriteBlockSizeAndReblock extends HopRewriteRule
//maintain rewrite status
if( isReblockValid() )
- state.setBlocksize(DMLTranslator.DMLBlockSize);
+ state.setBlocksize(ConfigurationManager.getBlocksize());
//perform reblock and blocksize rewrite
- rule_BlockSizeAndReblock(root, DMLTranslator.DMLBlockSize);
+ rule_BlockSizeAndReblock(root, ConfigurationManager.getBlocksize());
return root;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java
index a0d8e7b..945b21f 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java
@@ -208,7 +208,7 @@ public class RewriteConstantFolding extends HopRewriteRule
Recompiler.rClearLops(tmpWrite); //prevent lops reuse
Lop lops = tmpWrite.constructLops(); //reconstruct lops
lops.addToDag( dag );
- ArrayList<Instruction> inst = dag.getJobs(null, ConfigurationManager.getConfig());
+ ArrayList<Instruction> inst = dag.getJobs(null, ConfigurationManager.getDMLConfig());
//execute instructions
ExecutionContext ec = getExecutionContext();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
index 1785a51..6f9ab63 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java
@@ -22,6 +22,7 @@ package org.apache.sysml.hops.rewrite;
import java.util.ArrayList;
import java.util.HashSet;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.AggBinaryOp;
import org.apache.sysml.hops.DataOp;
import org.apache.sysml.hops.Hop;
@@ -33,7 +34,6 @@ import org.apache.sysml.hops.Hop.ReOrgOp;
import org.apache.sysml.hops.Hop.VisitStatus;
import org.apache.sysml.hops.HopsException;
import org.apache.sysml.hops.LiteralOp;
-import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.ParameterizedBuiltinOp;
import org.apache.sysml.hops.ReorgOp;
import org.apache.sysml.hops.TernaryOp;
@@ -276,7 +276,7 @@ public class RewriteSplitDagDataDependentOperators extends StatementBlockRewrite
//configure rmEmpty to directly output selection vector
//(only applied if dynamic recompilation enabled)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION )
+ if( ConfigurationManager.isDynamicRecompilation() )
pbhop.setOutputPermutationMatrix(true);
for( Hop p : hop.getParent() )
((AggBinaryOp)p).setHasLeftPMInput(true);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/lops/compile/Dag.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/compile/Dag.java b/src/main/java/org/apache/sysml/lops/compile/Dag.java
index 2944444..472149e 100644
--- a/src/main/java/org/apache/sysml/lops/compile/Dag.java
+++ b/src/main/java/org/apache/sysml/lops/compile/Dag.java
@@ -202,8 +202,7 @@ public class Dag<N extends Lop>
IDMap = new HashMap<Long, Integer>();
// get number of reducers from dml config
- DMLConfig conf = ConfigurationManager.getConfig();
- total_reducers = conf.getIntValue(DMLConfig.NUM_REDUCERS);
+ total_reducers = ConfigurationManager.getNumReducers();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java b/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
index bf40415..a49c61b 100644
--- a/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
+++ b/src/main/java/org/apache/sysml/lops/runtime/RunMRJobs.java
@@ -178,7 +178,7 @@ public class RunMRJobs
String shuffleInst = inst.getIv_shuffleInstructions();
String aggInst = inst.getIv_aggInstructions();
String otherInst = inst.getIv_otherInstructions();
- boolean jvmReuse = ConfigurationManager.getConfig().getBooleanValue(DMLConfig.JVM_REUSE);
+ boolean jvmReuse = ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.JVM_REUSE);
switch(inst.getJobType()) {
@@ -193,7 +193,7 @@ public class RunMRJobs
break;
case DATAGEN:
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ if( ConfigurationManager.isDynamicRecompilation()
&& OptimizerUtils.ALLOW_RAND_JOB_RECOMPILE
&& DMLScript.rtplatform != RUNTIME_PLATFORM.HADOOP
&& Recompiler.checkCPDataGen( inst, rdInst ) )
@@ -229,7 +229,7 @@ public class RunMRJobs
case REBLOCK:
case CSV_REBLOCK:
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ if( ConfigurationManager.isDynamicRecompilation()
&& DMLScript.rtplatform != RUNTIME_PLATFORM.HADOOP
&& Recompiler.checkCPReblock( inst, inputMatrices ) )
{
@@ -310,7 +310,7 @@ public class RunMRJobs
case TRANSFORM:
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ if( ConfigurationManager.isDynamicRecompilation()
&& OptimizerUtils.ALLOW_TRANSFORM_RECOMPILE
&& DMLScript.rtplatform != RUNTIME_PLATFORM.HADOOP
&& Recompiler.checkCPTransform( inst, inputMatrices ) )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/AParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/AParserWrapper.java b/src/main/java/org/apache/sysml/parser/AParserWrapper.java
index 7b7a797..b22e60b 100644
--- a/src/main/java/org/apache/sysml/parser/AParserWrapper.java
+++ b/src/main/java/org/apache/sysml/parser/AParserWrapper.java
@@ -39,12 +39,16 @@ import org.apache.sysml.runtime.util.LocalFileUtils;
*/
public abstract class AParserWrapper
{
- //global parser configuration dml/pydml:
- //1) skip errors on unspecified args (modified by mlcontext / jmlc)
- public static boolean IGNORE_UNSPECIFIED_ARGS = false;
-
-
- public abstract DMLProgram parse(String fileName, String dmlScript, HashMap<String, String> argVals) throws ParseException;
+ /**
+ *
+ * @param fileName
+ * @param dmlScript
+ * @param argVals
+ * @return
+ * @throws ParseException
+ */
+ public abstract DMLProgram parse(String fileName, String dmlScript, HashMap<String, String> argVals)
+ throws ParseException;
/**
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/DMLTranslator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
index 133b816..4e69819 100644
--- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java
+++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
@@ -26,9 +26,7 @@ import java.util.Iterator;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.AggBinaryOp;
import org.apache.sysml.hops.AggUnaryOp;
import org.apache.sysml.hops.BinaryOp;
@@ -70,31 +68,16 @@ import org.apache.sysml.runtime.DMLRuntimeException;
public class DMLTranslator
{
-
- public static int DMLBlockSize = 1000;
-
private static final Log LOG = LogFactory.getLog(DMLTranslator.class.getName());
private DMLProgram _dmlProg = null;
-
public DMLTranslator(DMLProgram dmlp)
throws DMLRuntimeException
{
_dmlProg = dmlp;
- //each script sets its own block size, opt level etc
- DMLConfig conf = ConfigurationManager.getConfig();
- DMLTranslator.setDMLBlockSize( conf.getIntValue( DMLConfig.DEFAULT_BLOCK_SIZE ) );
- OptimizerUtils.setOptimizationLevel( conf.getIntValue(DMLConfig.OPTIMIZATION_LEVEL) );
- Recompiler.reinitRecompiler(); //reinit rewriter according to opt level flags
- }
-
- /**
- *
- * @param blocksize
- */
- public static void setDMLBlockSize(int blocksize) {
- DMLBlockSize = blocksize;
+ //reinit rewriter according to opt level flags
+ Recompiler.reinitRecompiler();
}
/**
@@ -1022,7 +1005,7 @@ public class DMLTranslator
case BINARY:
// write output in binary block format
- ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateInPlace(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ ae.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateInPlace(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
break;
default:
@@ -1097,11 +1080,10 @@ public class DMLTranslator
// preserve data type matrix of any index identifier
// (required for scalar input to left indexing)
- // TODO Doug, please verify this (required for scalar assignments, otherwise TWrite of LIX target becomes scalar as well)
if( target.getDataType() != DataType.MATRIX ) {
target.setDataType(DataType.MATRIX);
target.setValueType(ValueType.DOUBLE);
- target.setBlockDimensions(DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ target.setBlockDimensions(ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
}
Integer statementId = liveOutToTemp.get(target.getName());
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/DataExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DataExpression.java b/src/main/java/org/apache/sysml/parser/DataExpression.java
index 29ffb18..84e7c43 100644
--- a/src/main/java/org/apache/sysml/parser/DataExpression.java
+++ b/src/main/java/org/apache/sysml/parser/DataExpression.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.wink.json4j.JSONArray;
import org.apache.wink.json4j.JSONObject;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.DataGenOp;
import org.apache.sysml.parser.LanguageException.LanguageErrorCodes;
@@ -43,11 +44,6 @@ import org.apache.sysml.utils.JSONHelper;
public class DataExpression extends DataIdentifier
{
- //internal configuration (modified by mlcontext, jmlc apis)
- //no read of meta data on mlcontext (local) /jmlc (global); ignore unknowns on jmlc
- public static boolean IGNORE_READ_WRITE_METADATA = false; // global skip meta data reads
- public static boolean REJECT_READ_WRITE_UNKNOWNS = true; // ignore missing meta data
-
public static final String RAND_ROWS = "rows";
public static final String RAND_COLS = "cols";
public static final String RAND_MIN = "min";
@@ -624,7 +620,8 @@ public class DataExpression extends DataIdentifier
String mtdFileName = getMTDFileName(inputFileName);
// track whether should attempt to read MTD file or not
- boolean shouldReadMTD = _checkMetadata && !IGNORE_READ_WRITE_METADATA;
+ boolean shouldReadMTD = _checkMetadata && !ConfigurationManager
+ .getCompilerConfigFlag(ConfigType.IGNORE_READ_WRITE_METADATA);
// Check for file existence (before metadata parsing for meaningful error messages)
if( shouldReadMTD //skip check for jmlc/mlcontext
@@ -894,7 +891,8 @@ public class DataExpression extends DataIdentifier
// initialize size of target data identifier to UNKNOWN
getOutput().setDimensions(-1, -1);
- if ( !isCSV && REJECT_READ_WRITE_UNKNOWNS //skip check for csv format / jmlc api
+ if ( !isCSV && ConfigurationManager.getCompilerConfig()
+ .getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) //skip check for csv format / jmlc api
&& (getVarParam(READROWPARAM) == null || getVarParam(READCOLPARAM) == null) ) {
raiseValidateError("Missing or incomplete dimension information in read statement: "
+ mtdFileName, conditional, LanguageErrorCodes.INVALID_PARAMETERS);
@@ -906,7 +904,8 @@ public class DataExpression extends DataIdentifier
// these are strings that are long values
Long dim1 = (getVarParam(READROWPARAM) == null) ? null : Long.valueOf( getVarParam(READROWPARAM).toString());
Long dim2 = (getVarParam(READCOLPARAM) == null) ? null : Long.valueOf( getVarParam(READCOLPARAM).toString());
- if ( !isCSV && (dim1 <= 0 || dim2 <= 0) && REJECT_READ_WRITE_UNKNOWNS ) {
+ if ( !isCSV && (dim1 <= 0 || dim2 <= 0) && ConfigurationManager
+ .getCompilerConfig().getBool(ConfigType.REJECT_READ_WRITE_UNKNOWNS) ) {
raiseValidateError("Invalid dimension information in read statement", conditional, LanguageErrorCodes.INVALID_PARAMETERS);
}
@@ -1065,7 +1064,7 @@ public class DataExpression extends DataIdentifier
if (getVarParam(FORMAT_TYPE) == null || getVarParam(FORMAT_TYPE).toString().equalsIgnoreCase("text"))
getOutput().setBlockDimensions(-1, -1);
else if (getVarParam(FORMAT_TYPE).toString().equalsIgnoreCase("binary"))
- getOutput().setBlockDimensions(DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ getOutput().setBlockDimensions(ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
else if (getVarParam(FORMAT_TYPE).toString().equalsIgnoreCase(FORMAT_TYPE_VALUE_MATRIXMARKET) || (getVarParam(FORMAT_TYPE).toString().equalsIgnoreCase(FORMAT_TYPE_VALUE_CSV)))
getOutput().setBlockDimensions(-1, -1);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/ForStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/ForStatementBlock.java b/src/main/java/org/apache/sysml/parser/ForStatementBlock.java
index 2469131..a13db39 100644
--- a/src/main/java/org/apache/sysml/parser/ForStatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/ForStatementBlock.java
@@ -23,9 +23,9 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.HopsException;
-import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.runtime.instructions.cp.BooleanObject;
@@ -420,8 +420,7 @@ public class ForStatementBlock extends StatementBlock
public void updatePredicateRecompilationFlags()
throws HopsException
{
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION )
- {
+ if( ConfigurationManager.isDynamicRecompilation() ) {
_requiresFromRecompile = Recompiler.requiresRecompilation(getFromHops());
_requiresToRecompile = Recompiler.requiresRecompilation(getToHops());
_requiresIncrementRecompile = Recompiler.requiresRecompilation(getIncrementHops());
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/IfStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/IfStatementBlock.java b/src/main/java/org/apache/sysml/parser/IfStatementBlock.java
index 99f5151..c172ffa 100644
--- a/src/main/java/org/apache/sysml/parser/IfStatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/IfStatementBlock.java
@@ -24,9 +24,9 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map.Entry;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.HopsException;
-import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.parser.Expression.DataType;
@@ -528,7 +528,7 @@ public class IfStatementBlock extends StatementBlock
public void updatePredicateRecompilationFlag()
throws HopsException
{
- _requiresPredicateRecompile = OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ _requiresPredicateRecompile = ConfigurationManager.isDynamicRecompilation()
&& Recompiler.requiresRecompilation(getPredicateHops());
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/StatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/StatementBlock.java b/src/main/java/org/apache/sysml/parser/StatementBlock.java
index 8f8de82..ef38989 100644
--- a/src/main/java/org/apache/sysml/parser/StatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/StatementBlock.java
@@ -25,12 +25,11 @@ import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.MLContextProxy;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.HopsException;
-import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.parser.Expression.DataType;
@@ -1052,7 +1051,7 @@ public class StatementBlock extends LiveVariableAnalysis
public void updateRecompilationFlag()
throws HopsException
{
- _requiresRecompile = OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ _requiresRecompile = ConfigurationManager.isDynamicRecompilation()
&& Recompiler.requiresRecompilation(get_hops());
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java b/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java
index f5852ae..866bf1c 100644
--- a/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java
@@ -23,9 +23,9 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.HopsException;
-import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.lops.Lop;
@@ -325,7 +325,7 @@ public class WhileStatementBlock extends StatementBlock
public void updatePredicateRecompilationFlag()
throws HopsException
{
- _requiresPredicateRecompile = OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ _requiresPredicateRecompile = ConfigurationManager.isDynamicRecompilation()
&& Recompiler.requiresRecompilation(getPredicateHops());
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java
index 02c8b75..7381c73 100644
--- a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java
+++ b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java
@@ -30,7 +30,8 @@ import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.TerminalNode;
-import org.apache.sysml.parser.AParserWrapper;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.parser.AssignmentStatement;
import org.apache.sysml.parser.ConditionalPredicate;
import org.apache.sysml.parser.DMLProgram;
@@ -301,7 +302,7 @@ public class DmlSyntacticValidator extends CommonSyntacticValidator implements D
if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) {
String msg = "The parameter " + varName + " either needs to be passed "
+ "through commandline or initialized to default value.";
- if( AParserWrapper.IGNORE_UNSPECIFIED_ARGS ) {
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.IGNORE_UNSPECIFIED_ARGS) ) {
ctx.dataInfo.expr = getConstIdFromString(" ", ctx.start);
raiseWarning(msg, ctx.start);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java
index fd12d53..2da5196 100644
--- a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java
+++ b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java
@@ -31,7 +31,8 @@ import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.apache.commons.lang.StringUtils;
-import org.apache.sysml.parser.AParserWrapper;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.parser.AssignmentStatement;
import org.apache.sysml.parser.BinaryExpression;
import org.apache.sysml.parser.ConditionalPredicate;
@@ -349,7 +350,7 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements
if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) {
String msg = "The parameter " + varName + " either needs to be passed "
+ "through commandline or initialized to default value.";
- if( AParserWrapper.IGNORE_UNSPECIFIED_ARGS ) {
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.IGNORE_UNSPECIFIED_ARGS) ) {
ctx.dataInfo.expr = getConstIdFromString(" ", ctx.start);
raiseWarning(msg, ctx.start);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
index 0ea80d8..b36bd59 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java
@@ -28,11 +28,9 @@ import java.util.TreeMap;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.ReBlock;
import org.apache.sysml.lops.compile.JobType;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.DataIdentifier;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
@@ -361,7 +359,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
String reblock = "";
String reblockStr = ""; //Keep a copy of a single MR reblock instruction
- String scratchSpaceLoc = ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
try {
// create a RBLK job that transforms each output matrix from cell to block
@@ -380,14 +378,14 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
reblock += "MR" + ReBlock.OPERAND_DELIMITOR + "rblk" + ReBlock.OPERAND_DELIMITOR +
i + ReBlock.DATATYPE_PREFIX + matrices.get(i).getDataType() + ReBlock.VALUETYPE_PREFIX + matrices.get(i).getValueType() + ReBlock.OPERAND_DELIMITOR +
i + ReBlock.DATATYPE_PREFIX + matrices.get(i).getDataType() + ReBlock.VALUETYPE_PREFIX + matrices.get(i).getValueType() + ReBlock.OPERAND_DELIMITOR +
- DMLTranslator.DMLBlockSize + ReBlock.OPERAND_DELIMITOR + DMLTranslator.DMLBlockSize + ReBlock.OPERAND_DELIMITOR + "true";
+ ConfigurationManager.getBlocksize() + ReBlock.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize() + ReBlock.OPERAND_DELIMITOR + "true";
if(DMLScript.ENABLE_DEBUG_MODE) {
//Create a copy of reblock instruction but as a single instruction (FOR DEBUGGER)
reblockStr = "MR" + ReBlock.OPERAND_DELIMITOR + "rblk" + ReBlock.OPERAND_DELIMITOR +
i + ReBlock.DATATYPE_PREFIX + matrices.get(i).getDataType() + ReBlock.VALUETYPE_PREFIX + matrices.get(i).getValueType() + ReBlock.OPERAND_DELIMITOR +
i + ReBlock.DATATYPE_PREFIX + matrices.get(i).getDataType() + ReBlock.VALUETYPE_PREFIX + matrices.get(i).getValueType() + ReBlock.OPERAND_DELIMITOR +
- DMLTranslator.DMLBlockSize + ReBlock.OPERAND_DELIMITOR + DMLTranslator.DMLBlockSize + ReBlock.OPERAND_DELIMITOR + "true";
+ ConfigurationManager.getBlocksize() + ReBlock.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize() + ReBlock.OPERAND_DELIMITOR + "true";
//Set MR reblock instruction line number (FOR DEBUGGER)
if (!MRJobLineNumbers.containsKey(matrices.get(i).getBeginLine())) {
MRJobLineNumbers.put(matrices.get(i).getBeginLine(), new ArrayList<String>());
@@ -487,7 +485,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
String[] outputs = new String[matrices.size()];
byte[] resultIndex = new byte[matrices.size()];
- String scratchSpaceLoc = ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
try {
@@ -514,7 +512,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
gmrStr = "MR" + Lop.OPERAND_DELIMITOR + "gmr" + Lop.OPERAND_DELIMITOR +
i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR +
i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR +
- DMLTranslator.DMLBlockSize + Lop.OPERAND_DELIMITOR + DMLTranslator.DMLBlockSize;
+ ConfigurationManager.getBlocksize() + Lop.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize();
//Set MR gmr instruction line number (FOR DEBUGGER)
if (!MRJobLineNumbers.containsKey(matrices.get(i).getBeginLine())) {
@@ -579,7 +577,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
for( int i=0; i<matricesNoReblock.size(); i++ )
{
- String scratchSpaceLoc = ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
try{
String filename = scratchSpaceLoc +
@@ -765,7 +763,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock
protected MatrixObject createOutputMatrixObject( Matrix m )
throws CacheException
{
- MatrixCharacteristics mc = new MatrixCharacteristics(m.getNumRows(),m.getNumCols(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ MatrixCharacteristics mc = new MatrixCharacteristics(m.getNumRows(),m.getNumCols(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
MatrixFormatMetaData mfmd = new MatrixFormatMetaData(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo);
return new MatrixObject(ValueType.DOUBLE, m.getFilePath(), mfmd);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
index ad0dbcd..3740e83 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java
@@ -22,7 +22,7 @@ package org.apache.sysml.runtime.controlprogram;
import java.util.ArrayList;
import java.util.HashMap;
-import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.parser.DataIdentifier;
import org.apache.sysml.parser.ExternalFunctionStatement;
import org.apache.sysml.parser.Expression.ValueType;
@@ -164,7 +164,7 @@ public class ExternalFunctionProgramBlockCP extends ExternalFunctionProgramBlock
if( ret == null ) //otherwise, pass in-memory matrix from extfunct back to invoking program
{
- MatrixCharacteristics mc = new MatrixCharacteristics(m.getNumRows(),m.getNumCols(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ MatrixCharacteristics mc = new MatrixCharacteristics(m.getNumRows(),m.getNumCols(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
MatrixFormatMetaData mfmd = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
ret = new MatrixObject(ValueType.DOUBLE, m.getFilePath(), mfmd);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java
index ff9113f..c28a6cc 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java
@@ -22,7 +22,7 @@ package org.apache.sysml.runtime.controlprogram;
import java.util.ArrayList;
import org.apache.sysml.api.DMLScript;
-import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.parser.DataIdentifier;
import org.apache.sysml.runtime.DMLRuntimeException;
@@ -84,7 +84,7 @@ public class FunctionProgramBlock extends ProgramBlock
{
//dynamically recompile entire function body (according to function inputs)
try {
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ if( ConfigurationManager.isDynamicRecompilation()
&& isRecompileOnce()
&& ParForProgramBlock.RESET_RECOMPILATION_FLAGs )
{
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
index 5d5b4ca..eb85306 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java
@@ -31,9 +31,9 @@ import java.util.List;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Level;
-
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.OptimizerUtils;
@@ -314,7 +314,7 @@ public class ParForProgramBlock extends ForProgramBlock
super(prog, iterPredVars);
//init internal flags according to DML config
- initInternalConfigurations(ConfigurationManager.getConfig());
+ initInternalConfigurations(ConfigurationManager.getDMLConfig());
//ID generation and setting
setParForProgramBlockIDs( ID );
@@ -1404,8 +1404,7 @@ public class ParForProgramBlock extends ForProgramBlock
try
{
- //create deep copies of required elements
- //child blocks
+ //create deep copies of required elements child blocks
ArrayList<ProgramBlock> cpChildBlocks = null;
HashSet<String> fnNames = new HashSet<String>();
if( USE_PB_CACHE )
@@ -1425,12 +1424,15 @@ public class ParForProgramBlock extends ForProgramBlock
cpChildBlocks = ProgramConverter.rcreateDeepCopyProgramBlocks(_childBlocks, pwID, _IDPrefix, new HashSet<String>(), fnNames, false, false);
}
- // Deep copy Execution Context
+ //deep copy execution context
ExecutionContext cpEc = ProgramConverter.createDeepCopyExecutionContext(ec);
+ //copy compiler configuration (for jmlc w/o global config)
+ CompilerConfig cconf = ConfigurationManager.getCompilerConfig();
+
//create the actual parallel worker
ParForBody body = new ParForBody( cpChildBlocks, _resultVars, cpEc );
- pw = new LocalParWorker( pwID, queue, body, MAX_RETRYS_ON_ERROR, _monitor );
+ pw = new LocalParWorker( pwID, queue, body, cconf, MAX_RETRYS_ON_ERROR, _monitor );
pw.setFunctionNames(fnNames);
}
catch(Exception ex)
@@ -1506,7 +1508,7 @@ public class ParForProgramBlock extends ForProgramBlock
DataPartitioner dp = null;
//determine max degree of parallelism
- int numReducers = ConfigurationManager.getConfig().getIntValue(DMLConfig.NUM_REDUCERS);
+ int numReducers = ConfigurationManager.getNumReducers();
int maxNumRed = InfrastructureAnalyzer.getRemoteParallelReduceTasks();
//correction max number of reducers on yarn clusters
if( InfrastructureAnalyzer.isYarnEnabled() )
@@ -1550,7 +1552,7 @@ public class ParForProgramBlock extends ForProgramBlock
ResultMerge rm = null;
//determine degree of parallelism
- int numReducers = ConfigurationManager.getConfig().getIntValue(DMLConfig.NUM_REDUCERS);
+ int numReducers = ConfigurationManager.getNumReducers();
int maxMap = InfrastructureAnalyzer.getRemoteParallelMapTasks();
int maxRed = InfrastructureAnalyzer.getRemoteParallelReduceTasks();
//correction max number of reducers on yarn clusters
@@ -1920,8 +1922,7 @@ public class ParForProgramBlock extends ForProgramBlock
*/
private String constructTaskFileName()
{
- String scratchSpaceLoc = ConfigurationManager.getConfig()
- .getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
StringBuilder sb = new StringBuilder();
sb.append(scratchSpaceLoc);
@@ -1941,8 +1942,7 @@ public class ParForProgramBlock extends ForProgramBlock
*/
private String constructResultFileName()
{
- String scratchSpaceLoc = ConfigurationManager.getConfig()
- .getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
StringBuilder sb = new StringBuilder();
sb.append(scratchSpaceLoc);
@@ -1960,8 +1960,7 @@ public class ParForProgramBlock extends ForProgramBlock
*/
private String constructResultMergeFileName()
{
- String scratchSpaceLoc = ConfigurationManager.getConfig()
- .getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
String fname = PARFOR_MR_RESULTMERGE_FNAME;
fname = fname.replaceAll("%ID%", String.valueOf(_ID)); //replace workerID
@@ -1983,8 +1982,7 @@ public class ParForProgramBlock extends ForProgramBlock
*/
private String constructDataPartitionsFileName()
{
- String scratchSpaceLoc = ConfigurationManager.getConfig()
- .getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
String fname = PARFOR_DATAPARTITIONS_FNAME;
fname = fname.replaceAll("%ID%", String.valueOf(_ID)); //replace workerID
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
index 209b68d..02e3e0c 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java
@@ -23,11 +23,10 @@ import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.MLContextProxy;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop;
-import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.parser.StatementBlock;
import org.apache.sysml.parser.Expression.ValueType;
@@ -144,7 +143,7 @@ public class ProgramBlock
DMLAppMasterUtils.setupProgramBlockRemoteMaxMemory(this);
long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ if( ConfigurationManager.isDynamicRecompilation()
&& _sb != null
&& _sb.requiresRecompilation() )
{
@@ -186,7 +185,7 @@ public class ProgramBlock
//dynamically recompile instructions if enabled and required
try {
long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION
+ if( ConfigurationManager.isDynamicRecompilation()
&& requiresRecompile )
{
tmp = Recompiler.recompileHopsDag(hops, ec.getVariables(), null, false, _tid);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
index 57c3da8..4f9d886 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java
@@ -23,12 +23,11 @@ import java.io.IOException;
import java.lang.ref.SoftReference;
import org.apache.commons.lang.mutable.MutableBoolean;
-
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.lops.Lop;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
@@ -1500,9 +1499,9 @@ public class MatrixObject extends CacheableData<MatrixBlock>
// when outputFormat is binaryblock, make sure that matrixCharacteristics has correct blocking dimensions
// note: this is only required if singlenode (due to binarycell default)
if ( oinfo == OutputInfo.BinaryBlockOutputInfo && DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE &&
- (mc.getRowsPerBlock() != DMLTranslator.DMLBlockSize || mc.getColsPerBlock() != DMLTranslator.DMLBlockSize) )
+ (mc.getRowsPerBlock() != ConfigurationManager.getBlocksize() || mc.getColsPerBlock() != ConfigurationManager.getBlocksize()) )
{
- DataConverter.writeMatrixToHDFS(_data, filePathAndName, oinfo, new MatrixCharacteristics(mc.getRows(), mc.getCols(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, mc.getNonZeros()), replication, formatProperties);
+ DataConverter.writeMatrixToHDFS(_data, filePathAndName, oinfo, new MatrixCharacteristics(mc.getRows(), mc.getCols(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), mc.getNonZeros()), replication, formatProperties);
}
else {
DataConverter.writeMatrixToHDFS(_data, filePathAndName, oinfo, mc, replication, formatProperties);
@@ -1545,9 +1544,9 @@ public class MatrixObject extends CacheableData<MatrixBlock>
// when outputFormat is binaryblock, make sure that matrixCharacteristics has correct blocking dimensions
// note: this is only required if singlenode (due to binarycell default)
if ( oinfo == OutputInfo.BinaryBlockOutputInfo && DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE &&
- (mc.getRowsPerBlock() != DMLTranslator.DMLBlockSize || mc.getColsPerBlock() != DMLTranslator.DMLBlockSize) )
+ (mc.getRowsPerBlock() != ConfigurationManager.getBlocksize() || mc.getColsPerBlock() != ConfigurationManager.getBlocksize()) )
{
- mc = new MatrixCharacteristics(mc.getRows(), mc.getCols(), DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, mc.getNonZeros());
+ mc = new MatrixCharacteristics(mc.getRows(), mc.getCols(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), mc.getNonZeros());
}
MapReduceTool.writeMetaDataFile (filePathAndName + ".mtd", valueType, mc, oinfo, formatProperties);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java
index 9f609bb..f666feb 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteMR.java
@@ -176,7 +176,7 @@ public class DataPartitionerRemoteMR extends DataPartitioner
job.setInt(MRConfigurationNames.DFS_REPLICATION, _replication);
//set up map/reduce memory configurations (if in AM context)
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
DMLAppMasterUtils.setupMRJobRemoteMaxMemory(job, config);
//set up custom map/reduce configurations
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java
index 1c66c11..e907c46 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalParWorker.java
@@ -21,6 +21,8 @@ package org.apache.sysml.runtime.controlprogram.parfor;
import java.util.Collection;
+import org.apache.sysml.conf.CompilerConfig;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysml.runtime.controlprogram.parfor.stat.Stat;
@@ -41,16 +43,18 @@ public class LocalParWorker extends ParWorker implements Runnable
protected Collection<String> _fnNames = null;
+ protected CompilerConfig _cconf = null;
protected boolean _stopped = false;
protected int _max_retry = -1;
- public LocalParWorker( long ID, LocalTaskQueue<Task> q, ParForBody body, int max_retry, boolean monitor )
+ public LocalParWorker( long ID, LocalTaskQueue<Task> q, ParForBody body, CompilerConfig cconf, int max_retry, boolean monitor )
{
super(ID, body, monitor);
_taskQueue = q;
- _stopped = false;
+ _cconf = cconf;
+ _stopped = false;
_max_retry = max_retry;
}
@@ -82,6 +86,9 @@ public class LocalParWorker extends ParWorker implements Runnable
sec.setThreadLocalSchedulerPool("parforPool"+_workerID);
}
+ //setup compiler config for worker thread
+ ConfigurationManager.setLocalConfig(_cconf);
+
// continuous execution (execute tasks until (1) stopped or (2) no more tasks)
Task lTask = null;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
index 004e980..a684c08 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java
@@ -29,12 +29,11 @@ import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapred.JobConf;
-
import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.Hop;
-import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.parser.DMLProgram;
import org.apache.sysml.parser.DataIdentifier;
@@ -625,7 +624,7 @@ public class ProgramConverter
try
{
- if( OptimizerUtils.ALLOW_PARALLEL_DYN_RECOMPILATION
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.ALLOW_PARALLEL_DYN_RECOMPILATION)
&& sb != null //forced deep copy for function recompilation
&& (Recompiler.requiresRecompilation( sb.get_hops() ) || forceDeepCopy) )
{
@@ -673,7 +672,7 @@ public class ProgramConverter
try
{
- if( OptimizerUtils.ALLOW_PARALLEL_DYN_RECOMPILATION
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.ALLOW_PARALLEL_DYN_RECOMPILATION)
&& sb != null //forced deep copy for function recompile
&& (Recompiler.requiresRecompilation( sb.getPredicateHops() ) || forceDeepCopy) )
{
@@ -722,7 +721,7 @@ public class ProgramConverter
try
{
- if( OptimizerUtils.ALLOW_PARALLEL_DYN_RECOMPILATION
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.ALLOW_PARALLEL_DYN_RECOMPILATION)
&& sb != null //forced deep copy for function recompile
&& (Recompiler.requiresRecompilation( sb.getPredicateHops() ) || forceDeepCopy) )
{
@@ -771,7 +770,7 @@ public class ProgramConverter
try
{
- if( OptimizerUtils.ALLOW_PARALLEL_DYN_RECOMPILATION
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.ALLOW_PARALLEL_DYN_RECOMPILATION)
&& sb != null
&& ( Recompiler.requiresRecompilation(sb.getFromHops()) ||
Recompiler.requiresRecompilation(sb.getToHops()) ||
@@ -852,7 +851,7 @@ public class ProgramConverter
sb.append( NEWLINE );
//handle DML config
- sb.append( ConfigurationManager.getConfig().serializeDMLConfig() );
+ sb.append( ConfigurationManager.getDMLConfig().serializeDMLConfig() );
sb.append( COMPONENTS_DELIM );
sb.append( NEWLINE );
@@ -1558,7 +1557,7 @@ public class ProgramConverter
JobConf job = ConfigurationManager.getCachedJobConf();
if( !InfrastructureAnalyzer.isLocalMode(job) ) {
DMLConfig config = DMLConfig.parseDMLConfig(confStr);
- ConfigurationManager.setConfig(config);
+ ConfigurationManager.setLocalConfig(config);
ParForProgramBlock.initInternalConfigurations(config);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java
index 2cbfd36..15b1fb3 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java
@@ -160,7 +160,7 @@ public class RemoteDPParForMR
MRJobConfiguration.addBinaryBlockSerializationFramework( job );
//set up map/reduce memory configurations (if in AM context)
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
DMLAppMasterUtils.setupMRJobRemoteMaxMemory(job, config);
//set up custom map/reduce configurations
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java
index 94be120..7710fe8 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java
@@ -164,7 +164,7 @@ public class RemoteParForMR
job.setMapSpeculativeExecution(false);
//set up map/reduce memory configurations (if in AM context)
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
DMLAppMasterUtils.setupMRJobRemoteMaxMemory(job, config);
//set up custom map/reduce configurations
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
index 356ae3c..185acbb 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java
@@ -294,7 +294,7 @@ public class ResultMergeRemoteMR extends ResultMerge
MRJobConfiguration.addBinaryBlockSerializationFramework( job );
//set up custom map/reduce configurations
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
MRJobConfiguration.setupCustomMRConfigurations(job, config);
//enables the reuse of JVMs (multiple tasks per MR task)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
index 360bbd2..edae6a1 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizationWrapper.java
@@ -29,8 +29,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-
import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.ipa.InterProceduralAnalysis;
import org.apache.sysml.hops.rewrite.HopRewriteRule;
@@ -228,7 +228,7 @@ public class OptimizationWrapper
OptTree tree = null;
//recompile parfor body
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION )
+ if( ConfigurationManager.isDynamicRecompilation() )
{
ForStatement fs = (ForStatement) sb.getStatement(0);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
index a6b7763..a2efb93 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
@@ -22,6 +22,7 @@ package org.apache.sysml.runtime.controlprogram.parfor.opt;
import java.util.HashMap;
import java.util.HashSet;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.lops.LopProperties;
import org.apache.sysml.parser.ParForStatementBlock;
@@ -265,7 +266,7 @@ public class OptimizerConstrained extends OptimizerRuleBased
boolean ret = false;
// constraint awareness
- if( n.getExecType() != null && OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR )
+ if( n.getExecType() != null && ConfigurationManager.isParallelParFor() )
{
ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter
.getAbstractPlanMapping().getMappedProg(n.getID())[1];
@@ -298,7 +299,7 @@ public class OptimizerConstrained extends OptimizerRuleBased
throws DMLRuntimeException
{
// constraint awareness
- if( n.getK() > 0 && OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR )
+ if( n.getK() > 0 && ConfigurationManager.isParallelParFor() )
{
ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter
.getAbstractPlanMapping().getMappedProg(n.getID())[1];
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index 4e871a1..98fb6bb 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -963,7 +963,7 @@ public class OptimizerRuleBased extends Optimizer
PDataPartitioner REMOTE_DP = OptimizerUtils.isSparkExecutionMode() ? PDataPartitioner.REMOTE_SPARK : PDataPartitioner.REMOTE_MR;
//deciding on the execution strategy
- if( OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR //allowed remote parfor execution
+ if( ConfigurationManager.isParallelParFor() //allowed remote parfor execution
&& ( (isCPOnly && M <= _rm ) //Required: all instruction can be be executed in CP
||(isCPOnlyPossible && M2 <= _rm)) ) //Required: cp inst fit into remote JVM mem
{
@@ -1436,7 +1436,7 @@ public class OptimizerRuleBased extends Optimizer
if( type == ExecType.CP )
{
//determine local max parallelism constraint
- int kMax = OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR ?
+ int kMax = ConfigurationManager.isParallelParFor() ?
(n.isCPOnly() ? _lkmaxCP : _lkmaxMR) : 1;
//ensure local memory constraint (for spark more conservative in order to
@@ -1535,7 +1535,7 @@ public class OptimizerRuleBased extends Optimizer
{
//set degree of parallelism for multi-threaded leaf nodes
Hop h = OptTreeConverter.getAbstractPlanMapping().getMappedHop(c.getID());
- if( OptimizerUtils.PARALLEL_CP_MATRIX_OPERATIONS
+ if( ConfigurationManager.isParallelMatrixOperations()
&& h instanceof MultiThreadedHop //abop, datagenop, qop, paramop
&& !( h instanceof ParameterizedBuiltinOp //only paramop-grpagg
&& ((ParameterizedBuiltinOp)h).getOp()!=ParamBuiltinOp.GROUPEDAGG) )
@@ -1583,7 +1583,7 @@ public class OptimizerRuleBased extends Optimizer
if( flagNested && flagLIX )
LOG.warn(getOptMode()+" OPT: Task partitioner decision has conflicting input from rewrites 'nested parallelism' and 'result partitioning'.");
- boolean jvmreuse = ConfigurationManager.getConfig().getBooleanValue(DMLConfig.JVM_REUSE);
+ boolean jvmreuse = ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.JVM_REUSE);
//set task partitioner
if( flagNested )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java
index c88f657..5289ce2 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/PerfTestTool.java
@@ -49,10 +49,10 @@ import au.com.bytecode.opencsv.CSVWriter;
import org.apache.sysml.api.DMLException;
import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.MMTSJ.MMTSJType;
import org.apache.sysml.parser.DMLProgram;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.DataIdentifier;
import org.apache.sysml.parser.ExternalFunctionStatement;
import org.apache.sysml.parser.ParseException;
@@ -1118,7 +1118,7 @@ public class PerfTestTool
MapReduceTool.deleteFileIfExistOnHDFS(fname);
- MatrixCharacteristics mc = new MatrixCharacteristics(dim, dim, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ MatrixCharacteristics mc = new MatrixCharacteristics(dim, dim, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
MatrixFormatMetaData md = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
MatrixObject mo = new MatrixObject(ValueType.DOUBLE,fname,md);
mo.acquireModify(mb);
@@ -1170,7 +1170,7 @@ public class PerfTestTool
MapReduceTool.deleteFileIfExistOnHDFS(fname);
- MatrixCharacteristics mc = new MatrixCharacteristics(d1, d2, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ MatrixCharacteristics mc = new MatrixCharacteristics(d1, d2, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
MatrixFormatMetaData md = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
MatrixObject mo = new MatrixObject(ValueType.DOUBLE,fname,md);
mo.acquireModify(mb);
@@ -1205,8 +1205,8 @@ public class PerfTestTool
mb = new MatrixBlock(dim,dim,true);
break;
}*/
-
- MatrixCharacteristics mc = new MatrixCharacteristics(dim, dim, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+
+ MatrixCharacteristics mc = new MatrixCharacteristics(dim, dim, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
MatrixFormatMetaData md = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
MatrixObject mo = new MatrixObject(ValueType.DOUBLE,fname,md);
@@ -1241,7 +1241,7 @@ public class PerfTestTool
break;
}*/
- MatrixCharacteristics mc = new MatrixCharacteristics(d1, d2, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ MatrixCharacteristics mc = new MatrixCharacteristics(d1, d2, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
MatrixFormatMetaData md = new MatrixFormatMetaData(mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
MatrixObject mo = new MatrixObject(ValueType.DOUBLE,fname,md);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
index 72b8c71..a914c0b 100644
--- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
+++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java
@@ -82,7 +82,7 @@ public class ProgramRecompiler
throws LopsException, DMLRuntimeException, DMLUnsupportedOperationException, IOException, HopsException
{
ArrayList<ProgramBlock> ret = new ArrayList<ProgramBlock>();
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
//construct lops from hops if not existing
DMLTranslator dmlt = new DMLTranslator(sbs.get(0).getDMLProg());
@@ -599,7 +599,7 @@ public class ProgramRecompiler
}
//construct new instructions
- ArrayList<Instruction> newInst = dag.getJobs(sbOld, ConfigurationManager.getConfig());
+ ArrayList<Instruction> newInst = dag.getJobs(sbOld, ConfigurationManager.getDMLConfig());
//exchange instructions
@@ -654,7 +654,7 @@ public class ProgramRecompiler
}
//construct new instructions
- ArrayList<Instruction> newInst = dag.getJobs(sbOld, ConfigurationManager.getConfig());
+ ArrayList<Instruction> newInst = dag.getJobs(sbOld, ConfigurationManager.getDMLConfig());
//exchange instructions
pbNew = new ProgramBlock(pbOld.getProgram());
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
index e598e34..ba511c7 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.DMLRuntimeException;
@@ -326,14 +325,14 @@ public class ParameterizedBuiltinCPFileInstruction extends ParameterizedBuiltinC
if( buffer.size() > StagingFileUtils.CELL_BUFFER_SIZE )
{
- appendCellBufferToStagingArea(stagingDir, buffer, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ appendCellBufferToStagingArea(stagingDir, buffer, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
buffer.clear();
}
}
if( !buffer.isEmpty() )
{
- appendCellBufferToStagingArea(stagingDir, buffer, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ appendCellBufferToStagingArea(stagingDir, buffer, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
buffer.clear();
}
}
@@ -383,14 +382,14 @@ public class ParameterizedBuiltinCPFileInstruction extends ParameterizedBuiltinC
if( buffer.size() > StagingFileUtils.CELL_BUFFER_SIZE )
{
- appendCellBufferToStagingArea(stagingDir, buffer, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ appendCellBufferToStagingArea(stagingDir, buffer, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
buffer.clear();
}
}
if( !buffer.isEmpty() )
{
- appendCellBufferToStagingArea(stagingDir, buffer, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ appendCellBufferToStagingArea(stagingDir, buffer, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize());
buffer.clear();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/io/MatrixReaderFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/MatrixReaderFactory.java b/src/main/java/org/apache/sysml/runtime/io/MatrixReaderFactory.java
index d1065f8..8da8fd7 100644
--- a/src/main/java/org/apache/sysml/runtime/io/MatrixReaderFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/io/MatrixReaderFactory.java
@@ -19,7 +19,8 @@
package org.apache.sysml.runtime.io;
-import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.InputInfo;
@@ -43,14 +44,14 @@ public class MatrixReaderFactory
if( iinfo == InputInfo.TextCellInputInfo || iinfo == InputInfo.MatrixMarketInputInfo )
{
- if( OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_TEXTFORMATS) )
reader = new ReaderTextCellParallel( iinfo );
else
reader = new ReaderTextCell( iinfo );
}
else if( iinfo == InputInfo.CSVInputInfo )
{
- if( OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_TEXTFORMATS) )
reader = new ReaderTextCSVParallel(new CSVFileFormatProperties());
else
reader = new ReaderTextCSV(new CSVFileFormatProperties());
@@ -58,7 +59,7 @@ public class MatrixReaderFactory
else if( iinfo == InputInfo.BinaryCellInputInfo )
reader = new ReaderBinaryCell();
else if( iinfo == InputInfo.BinaryBlockInputInfo ) {
- if( OptimizerUtils.PARALLEL_CP_READ_BINARYFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_BINARYFORMATS) )
reader = new ReaderBinaryBlockParallel( false );
else
reader = new ReaderBinaryBlock( false );
@@ -88,13 +89,13 @@ public class MatrixReaderFactory
InputInfo iinfo = props.inputInfo;
if( iinfo == InputInfo.TextCellInputInfo || iinfo == InputInfo.MatrixMarketInputInfo ) {
- if( OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_TEXTFORMATS) )
reader = new ReaderTextCellParallel( iinfo );
else
reader = new ReaderTextCell( iinfo );
}
else if( iinfo == InputInfo.CSVInputInfo ) {
- if( OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_TEXTFORMATS) )
reader = new ReaderTextCSVParallel( props.formatProperties!=null ? (CSVFileFormatProperties)props.formatProperties : new CSVFileFormatProperties());
else
reader = new ReaderTextCSV( props.formatProperties!=null ? (CSVFileFormatProperties)props.formatProperties : new CSVFileFormatProperties());
@@ -102,7 +103,7 @@ public class MatrixReaderFactory
else if( iinfo == InputInfo.BinaryCellInputInfo )
reader = new ReaderBinaryCell();
else if( iinfo == InputInfo.BinaryBlockInputInfo ) {
- if( OptimizerUtils.PARALLEL_CP_READ_BINARYFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_BINARYFORMATS) )
reader = new ReaderBinaryBlockParallel( props.localFS );
else
reader = new ReaderBinaryBlock( props.localFS );
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/io/MatrixWriterFactory.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/io/MatrixWriterFactory.java b/src/main/java/org/apache/sysml/runtime/io/MatrixWriterFactory.java
index 85ff514..fa56fef 100644
--- a/src/main/java/org/apache/sysml/runtime/io/MatrixWriterFactory.java
+++ b/src/main/java/org/apache/sysml/runtime/io/MatrixWriterFactory.java
@@ -19,7 +19,8 @@
package org.apache.sysml.runtime.io;
-import org.apache.sysml.hops.OptimizerUtils;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties;
import org.apache.sysml.runtime.matrix.data.FileFormatProperties;
@@ -58,7 +59,7 @@ public class MatrixWriterFactory
MatrixWriter writer = null;
if( oinfo == OutputInfo.TextCellOutputInfo ) {
- if( OptimizerUtils.PARALLEL_CP_WRITE_TEXTFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS) )
writer = new WriterTextCellParallel();
else
writer = new WriterTextCell();
@@ -75,7 +76,7 @@ public class MatrixWriterFactory
else if( oinfo == OutputInfo.CSVOutputInfo ) {
if( props!=null && !(props instanceof CSVFileFormatProperties) )
throw new DMLRuntimeException("Wrong type of file format properties for CSV writer.");
- if( OptimizerUtils.PARALLEL_CP_WRITE_TEXTFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS) )
writer = new WriterTextCSVParallel((CSVFileFormatProperties)props);
else
writer = new WriterTextCSV((CSVFileFormatProperties)props);
@@ -84,7 +85,7 @@ public class MatrixWriterFactory
writer = new WriterBinaryCell();
}
else if( oinfo == OutputInfo.BinaryBlockOutputInfo ) {
- if( OptimizerUtils.PARALLEL_CP_WRITE_BINARYFORMATS )
+ if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_WRITE_BINARYFORMATS) )
writer = new WriterBinaryBlockParallel(replication);
else
writer = new WriterBinaryBlock(replication);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/matrix/CMCOVMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/CMCOVMR.java b/src/main/java/org/apache/sysml/runtime/matrix/CMCOVMR.java
index 1747214..06f4054 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/CMCOVMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/CMCOVMR.java
@@ -90,7 +90,7 @@ public class CMCOVMR
job.setInt(MRConfigurationNames.DFS_REPLICATION, replication);
//set up custom map/reduce configurations
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
MRJobConfiguration.setupCustomMRConfigurations(job, config);
//set up what matrices are needed to pass from the mapper to reducer
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java b/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java
index 68611bb..0cae638 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java
@@ -316,7 +316,7 @@ public class CSVReblockMR
job.setInt(MRConfigurationNames.DFS_REPLICATION, replication);
//set up custom map/reduce configurations
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
MRJobConfiguration.setupCustomMRConfigurations(job, config);
//set up the number of reducers
@@ -412,7 +412,7 @@ public class CSVReblockMR
MRJobConfiguration.addBinaryBlockSerializationFramework( job );
//set up custom map/reduce configurations
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
MRJobConfiguration.setupCustomMRConfigurations(job, config);
//set up what matrices are needed to pass from the mapper to reducer
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java b/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
index 99d561f..e48c868 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java
@@ -332,7 +332,7 @@ public class CombineMR
job.setInt(MRConfigurationNames.DFS_REPLICATION, replication);
//set up custom map/reduce configurations
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
MRJobConfiguration.setupCustomMRConfigurations(job, config);
//set up what matrices are needed to pass from the mapper to reducer
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java b/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java
index 4cf2472..974c7df 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java
@@ -297,7 +297,7 @@ public class DataGenMR
job.setInt(MRConfigurationNames.DFS_REPLICATION, replication);
//set up map/reduce memory configurations (if in AM context)
- DMLConfig config = ConfigurationManager.getConfig();
+ DMLConfig config = ConfigurationManager.getDMLConfig();
DMLAppMasterUtils.setupMRJobRemoteMaxMemory(job, config);
//set up custom map/reduce configurations