You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/03/19 23:21:42 UTC
[4/4] incubator-systemml git commit: [SYSTEMML-584] New thread-local
configuration handling (dml/compiler)
[SYSTEMML-584] New thread-local configuration handling (dml/compiler)
So far we did not fully support concurrent script executions in a shared
JVM process (e.g., as necessary for JMLC scoring in spark executors)
because global static state updates caused potential side effects. This
patch introduces thread-local configuration handling for dml and
compiler configurations. All these functionalities are encapsulated
within the ConfigurationManager behind a very generic API. This cleanup
covers all relevant configurations for JMLC - down the road we could
extend this as required. Note that we also support 1-1 thread mappings
in parfor, which will be required in JMLC for parfor scripts with
disabled parallelism.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/257793d8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/257793d8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/257793d8
Branch: refs/heads/master
Commit: 257793d8309056b86096f5ed7c8528ad83f5d8fc
Parents: 59a4a50
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sat Mar 19 01:41:13 2016 -0700
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sat Mar 19 15:20:39 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/sysml/api/DMLScript.java | 26 +--
.../java/org/apache/sysml/api/MLContext.java | 56 +++---
.../java/org/apache/sysml/api/MLMatrix.java | 5 +-
.../org/apache/sysml/api/jmlc/Connection.java | 72 ++++---
.../apache/sysml/api/jmlc/PreparedScript.java | 9 +-
.../org/apache/sysml/conf/CompilerConfig.java | 157 +++++++++++++++
.../apache/sysml/conf/ConfigurationManager.java | 190 ++++++++++++++++---
.../java/org/apache/sysml/conf/DMLConfig.java | 90 ++++-----
.../java/org/apache/sysml/hops/AggBinaryOp.java | 3 +-
.../java/org/apache/sysml/hops/AggUnaryOp.java | 3 +-
.../java/org/apache/sysml/hops/BinaryOp.java | 5 +-
.../java/org/apache/sysml/hops/DataGenOp.java | 13 +-
src/main/java/org/apache/sysml/hops/DataOp.java | 9 +-
src/main/java/org/apache/sysml/hops/Hop.java | 5 +-
.../java/org/apache/sysml/hops/IndexingOp.java | 3 +-
.../org/apache/sysml/hops/LeftIndexingOp.java | 10 +-
.../org/apache/sysml/hops/OptimizerUtils.java | 151 ++++++---------
.../sysml/hops/ParameterizedBuiltinOp.java | 3 +-
.../org/apache/sysml/hops/QuaternaryOp.java | 3 +-
.../java/org/apache/sysml/hops/ReorgOp.java | 3 +-
.../java/org/apache/sysml/hops/TernaryOp.java | 3 +-
.../java/org/apache/sysml/hops/UnaryOp.java | 3 +-
.../apache/sysml/hops/cost/CostEstimator.java | 5 +-
.../hops/cost/CostEstimatorStaticRuntime.java | 8 +-
.../sysml/hops/globalopt/GDFEnumOptimizer.java | 7 +-
.../sysml/hops/ipa/InterProceduralAnalysis.java | 5 +-
.../apache/sysml/hops/recompile/Recompiler.java | 37 ++--
.../sysml/hops/rewrite/ProgramRewriter.java | 5 +-
.../RewriteAlgebraicSimplificationDynamic.java | 6 +-
.../rewrite/RewriteBlockSizeAndReblock.java | 10 +-
.../hops/rewrite/RewriteConstantFolding.java | 2 +-
.../RewriteSplitDagDataDependentOperators.java | 4 +-
.../java/org/apache/sysml/lops/compile/Dag.java | 3 +-
.../apache/sysml/lops/runtime/RunMRJobs.java | 8 +-
.../org/apache/sysml/parser/AParserWrapper.java | 16 +-
.../org/apache/sysml/parser/DMLTranslator.java | 26 +--
.../org/apache/sysml/parser/DataExpression.java | 17 +-
.../apache/sysml/parser/ForStatementBlock.java | 5 +-
.../apache/sysml/parser/IfStatementBlock.java | 4 +-
.../org/apache/sysml/parser/StatementBlock.java | 5 +-
.../sysml/parser/WhileStatementBlock.java | 4 +-
.../sysml/parser/dml/DmlSyntacticValidator.java | 5 +-
.../parser/pydml/PydmlSyntacticValidator.java | 5 +-
.../ExternalFunctionProgramBlock.java | 16 +-
.../ExternalFunctionProgramBlockCP.java | 4 +-
.../controlprogram/FunctionProgramBlock.java | 4 +-
.../controlprogram/ParForProgramBlock.java | 30 ++-
.../runtime/controlprogram/ProgramBlock.java | 7 +-
.../controlprogram/caching/MatrixObject.java | 11 +-
.../parfor/DataPartitionerRemoteMR.java | 2 +-
.../controlprogram/parfor/LocalParWorker.java | 11 +-
.../controlprogram/parfor/ProgramConverter.java | 15 +-
.../controlprogram/parfor/RemoteDPParForMR.java | 2 +-
.../controlprogram/parfor/RemoteParForMR.java | 2 +-
.../parfor/ResultMergeRemoteMR.java | 2 +-
.../parfor/opt/OptimizationWrapper.java | 4 +-
.../parfor/opt/OptimizerConstrained.java | 5 +-
.../parfor/opt/OptimizerRuleBased.java | 8 +-
.../controlprogram/parfor/opt/PerfTestTool.java | 12 +-
.../parfor/opt/ProgramRecompiler.java | 6 +-
.../ParameterizedBuiltinCPFileInstruction.java | 9 +-
.../sysml/runtime/io/MatrixReaderFactory.java | 15 +-
.../sysml/runtime/io/MatrixWriterFactory.java | 9 +-
.../apache/sysml/runtime/matrix/CMCOVMR.java | 2 +-
.../sysml/runtime/matrix/CSVReblockMR.java | 4 +-
.../apache/sysml/runtime/matrix/CombineMR.java | 2 +-
.../apache/sysml/runtime/matrix/DataGenMR.java | 2 +-
.../org/apache/sysml/runtime/matrix/GMR.java | 2 +-
.../sysml/runtime/matrix/GroupedAggMR.java | 2 +-
.../org/apache/sysml/runtime/matrix/MMCJMR.java | 2 +-
.../org/apache/sysml/runtime/matrix/MMRJMR.java | 2 +-
.../apache/sysml/runtime/matrix/ReblockMR.java | 2 +-
.../org/apache/sysml/runtime/matrix/SortMR.java | 2 +-
.../apache/sysml/runtime/matrix/WriteCSVMR.java | 2 +-
.../sysml/runtime/matrix/data/MatrixBlock.java | 9 +-
.../matrix/mapred/MRJobConfiguration.java | 6 +-
.../sysml/runtime/transform/ApplyTfBBMR.java | 3 +-
.../sysml/runtime/transform/DataTransform.java | 4 +-
.../sysml/runtime/util/LocalFileUtils.java | 2 +-
src/main/java/org/apache/sysml/udf/Matrix.java | 6 +-
.../sysml/udf/lib/DynamicReadMatrixCP.java | 4 +-
.../sysml/udf/lib/DynamicReadMatrixRcCP.java | 4 +-
.../org/apache/sysml/yarn/DMLAppMaster.java | 2 +-
.../apache/sysml/yarn/DMLAppMasterUtils.java | 9 +-
.../sysml/yarn/ropt/ResourceOptimizer.java | 4 +-
.../sysml/yarn/ropt/YarnOptimizerUtils.java | 3 +-
.../test/integration/AutomatedTestBase.java | 5 +-
.../test/integration/applications/GLMTest.java | 4 +-
.../functions/append/AppendMatrixTest.java | 5 +-
.../functions/append/AppendVectorTest.java | 7 +-
.../FullIntegerDivisionTest.java | 10 +-
.../binary/matrix_full_other/FullPowerTest.java | 7 +-
.../caching/CachingPWriteExportTest.java | 5 +-
.../data/FullStringInitializeTest.java | 5 +-
.../integration/functions/data/ReadMMTest.java | 17 +-
.../functions/io/SeqParReadTest.java | 10 +-
.../functions/io/csv/ReadCSVTest.java | 9 +-
.../functions/io/matrixmarket/ReadMMTest.java | 9 +-
.../functions/misc/DataTypeChangeTest.java | 2 +-
.../functions/mlcontext/GNMFTest.java | 2 +-
.../parfor/ParForDependencyAnalysisTest.java | 2 +-
.../ParForReplaceThreadIDRecompileTest.java | 10 +-
.../recompile/FunctionRecompileTest.java | 9 +-
.../recompile/PredicateRecompileTest.java | 8 +-
.../functions/recompile/RandRecompileTest.java | 8 +-
.../recompile/ReblockRecompileTest.java | 6 +-
.../SparsityFunctionRecompileTest.java | 11 +-
.../recompile/SparsityRecompileTest.java | 12 +-
.../functions/transform/ScalingTest.java | 5 +-
.../functions/transform/TransformTest.java | 9 +-
.../updateinplace/UpdateInPlaceTest.java | 5 +-
111 files changed, 832 insertions(+), 613 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/api/DMLScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java b/src/main/java/org/apache/sysml/api/DMLScript.java
index 101b229..a3278c0 100644
--- a/src/main/java/org/apache/sysml/api/DMLScript.java
+++ b/src/main/java/org/apache/sysml/api/DMLScript.java
@@ -46,7 +46,7 @@ import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.xml.sax.SAXException;
-
+import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.debug.DMLDebugger;
@@ -571,13 +571,15 @@ public class DMLScript
printStartExecInfo( dmlScriptStr );
//Step 1: parse configuration files
- DMLConfig conf = DMLConfig.readAndMergeConfigurationFiles(fnameOptConfig);
- ConfigurationManager.setConfig(conf);
- LOG.debug("\nDML config: \n" + conf.getConfigInfo());
+ DMLConfig dmlconf = DMLConfig.readAndMergeConfigurationFiles(fnameOptConfig);
+ ConfigurationManager.setGlobalConfig(dmlconf);
+ CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf);
+ ConfigurationManager.setGlobalConfig(cconf);
+ LOG.debug("\nDML config: \n" + dmlconf.getConfigInfo());
//Step 2: set local/remote memory if requested (for compile in AM context)
- if( conf.getBooleanValue(DMLConfig.YARN_APPMASTER) ){
- DMLAppMasterUtils.setupConfigRemoteMaxMemory(conf);
+ if( dmlconf.getBooleanValue(DMLConfig.YARN_APPMASTER) ){
+ DMLAppMasterUtils.setupConfigRemoteMaxMemory(dmlconf);
}
//Step 3: parse dml script
@@ -621,7 +623,7 @@ public class DMLScript
}
//Step 7: generate runtime program
- Program rtprog = prog.getRuntimeProgram(conf);
+ Program rtprog = prog.getRuntimeProgram(dmlconf);
if (LOG.isDebugEnabled()) {
LOG.info("********************** Instructions *******************");
@@ -638,8 +640,8 @@ public class DMLScript
}
//launch SystemML appmaster (if requested and not already in launched AM)
- if( conf.getBooleanValue(DMLConfig.YARN_APPMASTER) ){
- if( !isActiveAM() && DMLYarnClientProxy.launchDMLYarnAppmaster(dmlScriptStr, conf, allArgs, rtprog) )
+ if( dmlconf.getBooleanValue(DMLConfig.YARN_APPMASTER) ){
+ if( !isActiveAM() && DMLYarnClientProxy.launchDMLYarnAppmaster(dmlScriptStr, dmlconf, allArgs, rtprog) )
return; //if AM launch unsuccessful, fall back to normal execute
if( isActiveAM() ) //in AM context (not failed AM launch)
DMLAppMasterUtils.setupProgramMappingRemoteMaxMemory(rtprog);
@@ -669,7 +671,7 @@ public class DMLScript
ExecutionContext ec = null;
try
{
- initHadoopExecution( conf );
+ initHadoopExecution( dmlconf );
//run execute (w/ exception handling to ensure proper shutdown)
ec = ExecutionContextFactory.createContext(rtprog);
@@ -688,7 +690,7 @@ public class DMLScript
LOG.info("END DML run " + getDateTime() );
//cleanup scratch_space and all working dirs
- cleanupHadoopExecution( conf );
+ cleanupHadoopExecution( dmlconf );
}
}
@@ -755,7 +757,7 @@ public class DMLScript
//Step 1: parse configuration files
dbprog.conf = DMLConfig.readAndMergeConfigurationFiles(fnameOptConfig);
- ConfigurationManager.setConfig(dbprog.conf);
+ ConfigurationManager.setGlobalConfig(dbprog.conf);
//Step 2: parse dml script
AParserWrapper parser = AParserWrapper.createParser(parsePyDML);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/api/MLContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/MLContext.java b/src/main/java/org/apache/sysml/api/MLContext.java
index 247cd69..f3d0bbb 100644
--- a/src/main/java/org/apache/sysml/api/MLContext.java
+++ b/src/main/java/org/apache/sysml/api/MLContext.java
@@ -36,6 +36,8 @@ import org.apache.spark.rdd.RDD;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
import org.apache.sysml.api.jmlc.JMLCUtils;
import org.apache.sysml.api.monitoring.SparkMonitoringUtil;
+import org.apache.sysml.conf.CompilerConfig;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.OptimizerUtils;
@@ -486,14 +488,16 @@ public class MLContext {
MatrixObject mo = null;
if( format.equals("csv") ) {
- MatrixCharacteristics mc = new MatrixCharacteristics(rlen, clen, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, nnz);
+ //TODO replace default block size
+ MatrixCharacteristics mc = new MatrixCharacteristics(rlen, clen, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE, nnz);
mo = new MatrixObject(ValueType.DOUBLE, null, new MatrixFormatMetaData(mc, OutputInfo.CSVOutputInfo, InputInfo.CSVInputInfo));
}
else if( format.equals("text") ) {
if(rlen == -1 || clen == -1) {
throw new DMLRuntimeException("The metadata is required in registerInput for format:" + format);
}
- MatrixCharacteristics mc = new MatrixCharacteristics(rlen, clen, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, nnz);
+ //TODO replace default block size
+ MatrixCharacteristics mc = new MatrixCharacteristics(rlen, clen, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE, nnz);
mo = new MatrixObject(ValueType.DOUBLE, null, new MatrixFormatMetaData(mc, OutputInfo.TextCellOutputInfo, InputInfo.TextCellInputInfo));
}
else if( format.equals("mm") ) {
@@ -532,7 +536,8 @@ public class MLContext {
* @throws DMLRuntimeException
*/
public void registerInput(String varName, JavaPairRDD<MatrixIndexes,MatrixBlock> rdd, long rlen, long clen) throws DMLRuntimeException {
- registerInput(varName, rdd, rlen, clen, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize);
+ //TODO replace default blocksize
+ registerInput(varName, rdd, rlen, clen, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE);
}
/**
@@ -1189,13 +1194,6 @@ public class MLContext {
// Set active MLContext.
_activeMLContext = this;
- // Setup parser parameters
- // TODO In the process of hardening mlcontext, we should also reinvestigate if we
- // could be more restrictive and require known dimensions (rm REJECT_READ_WRITE_UNKNOWNS).
- AParserWrapper.IGNORE_UNSPECIFIED_ARGS = true;
- DataExpression.REJECT_READ_WRITE_UNKNOWNS = false;
- OptimizerUtils.ALLOW_CSE_PERSISTENT_READS = false;
-
if(_monitorUtils != null) {
_monitorUtils.resetMonitoringData();
}
@@ -1254,13 +1252,7 @@ public class MLContext {
}
finally {
// Reset active MLContext.
- _activeMLContext = null;
-
- // Reset parser parameters
- AParserWrapper.IGNORE_UNSPECIFIED_ARGS = false;
- DataExpression.REJECT_READ_WRITE_UNKNOWNS = true;
- OptimizerUtils.ALLOW_CSE_PERSISTENT_READS =
- OptimizerUtils.ALLOW_COMMON_SUBEXPRESSION_ELIMINATION;
+ _activeMLContext = null;
}
}
@@ -1282,27 +1274,25 @@ public class MLContext {
* @throws ParseException
*/
private ExecutionContext executeUsingSimplifiedCompilationChain(String dmlScriptFilePath, boolean isFile, HashMap<String, String> argVals, boolean parsePyDML,
- String[] inputs, String[] outputs, LocalVariableMap inputSymbolTable, String configFilePath) throws IOException, DMLException, ParseException {
- DMLConfig config = null;
- if(configFilePath == null) {
- config = new DMLConfig();
- }
- else {
- config = new DMLConfig(configFilePath);
- }
-
+ String[] inputs, String[] outputs, LocalVariableMap inputSymbolTable, String configFilePath)
+ throws IOException, DMLException, ParseException
+ {
+ //construct dml configuration
+ DMLConfig config = (configFilePath == null) ? new DMLConfig() : new DMLConfig(configFilePath);
for(Entry<String, String> param : _additionalConfigs.entrySet()) {
config.setTextValue(param.getKey(), param.getValue());
}
- ConfigurationManager.setConfig(config);
+ //set global dml and specialized compiler configurations
+ ConfigurationManager.setGlobalConfig(config);
+ CompilerConfig cconf = new CompilerConfig();
+ cconf.set(ConfigType.IGNORE_UNSPECIFIED_ARGS, true);
+ cconf.set(ConfigType.REJECT_READ_WRITE_UNKNOWNS, false);
+ cconf.set(ConfigType.ALLOW_CSE_PERSISTENT_READS, false);
+ ConfigurationManager.setGlobalConfig(cconf);
- String dmlScriptStr = null;
- if(isFile)
- dmlScriptStr = DMLScript.readDMLScript("-f", dmlScriptFilePath);
- else
- dmlScriptStr = DMLScript.readDMLScript("-s", dmlScriptFilePath);
-
+ //read dml script string
+ String dmlScriptStr = DMLScript.readDMLScript( isFile?"-f":"-s", dmlScriptFilePath);
if(_monitorUtils != null) {
_monitorUtils.setDMLString(dmlScriptStr);
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/api/MLMatrix.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/MLMatrix.java b/src/main/java/org/apache/sysml/api/MLMatrix.java
index 84e15fa..b0c0afb 100644
--- a/src/main/java/org/apache/sysml/api/MLMatrix.java
+++ b/src/main/java/org/apache/sysml/api/MLMatrix.java
@@ -34,7 +34,7 @@ import org.apache.spark.sql.types.StructType;
import scala.Tuple2;
-import org.apache.sysml.parser.DMLTranslator;
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.parser.ParseException;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.instructions.spark.functions.GetMIMBFromRow;
@@ -91,7 +91,8 @@ public class MLMatrix extends DataFrame {
this.ml = ml;
}
- static String writeStmt = "write(output, \"tmp\", format=\"binary\", rows_in_block=" + DMLTranslator.DMLBlockSize + ", cols_in_block=" + DMLTranslator.DMLBlockSize + ");";
+ //TODO replace default blocksize
+ static String writeStmt = "write(output, \"tmp\", format=\"binary\", rows_in_block=" + OptimizerUtils.DEFAULT_BLOCKSIZE + ", cols_in_block=" + OptimizerUtils.DEFAULT_BLOCKSIZE + ");";
// ------------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/api/jmlc/Connection.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
index 43b56d0..bd764ac 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
@@ -29,19 +29,18 @@ import java.util.HashMap;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-
import org.apache.sysml.api.DMLException;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.CompilerConfig;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
-import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.hops.rewrite.ProgramRewriter;
import org.apache.sysml.hops.rewrite.RewriteRemovePersistentReadWrite;
import org.apache.sysml.parser.AParserWrapper;
import org.apache.sysml.parser.DMLProgram;
import org.apache.sysml.parser.DMLTranslator;
-import org.apache.sysml.parser.DataExpression;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.Program;
import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
@@ -59,9 +58,9 @@ import org.apache.sysml.runtime.util.DataConverter;
* * See JUnit test cases (org.apache.sysml.test.integration.functions.jmlc) for examples.
*/
public class Connection
-{
-
- private DMLConfig _conf = null;
+{
+ private DMLConfig _dmlconf = null;
+ private CompilerConfig _cconf = null;
/**
* Connection constructor, starting point for any other JMLC API calls.
@@ -69,26 +68,31 @@ public class Connection
*/
public Connection()
{
- //setup basic parameters for embedded execution
- //parser parameters
- AParserWrapper.IGNORE_UNSPECIFIED_ARGS = true;
- DataExpression.IGNORE_READ_WRITE_METADATA = true;
- DataExpression.REJECT_READ_WRITE_UNKNOWNS = false;
- //runtime parameters
DMLScript.rtplatform = RUNTIME_PLATFORM.SINGLE_NODE;
- OptimizerUtils.ALLOW_CSE_PERSISTENT_READS = false;
- OptimizerUtils.PARALLEL_CP_MATRIX_OPERATIONS = false;
- OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR = false;
- OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS = false;
- OptimizerUtils.PARALLEL_CP_WRITE_TEXTFORMATS = false;
- OptimizerUtils.PARALLEL_CP_READ_BINARYFORMATS = false;
- OptimizerUtils.PARALLEL_CP_WRITE_BINARYFORMATS = false;
- OptimizerUtils.ALLOW_DYN_RECOMPILATION = false;
+
+ //setup basic parameters for embedded execution
+ //(parser, compiler, and runtime parameters)
+ _cconf = new CompilerConfig();
+ _cconf.set(ConfigType.IGNORE_UNSPECIFIED_ARGS, true);
+ _cconf.set(ConfigType.IGNORE_READ_WRITE_METADATA, true);
+ _cconf.set(ConfigType.REJECT_READ_WRITE_UNKNOWNS, false);
+ _cconf.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, false);
+ _cconf.set(ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS, false);
+ _cconf.set(ConfigType.PARALLEL_CP_READ_BINARYFORMATS, false);
+ _cconf.set(ConfigType.PARALLEL_CP_WRITE_BINARYFORMATS, false);
+ _cconf.set(ConfigType.PARALLEL_CP_MATRIX_OPERATIONS, false);
+ _cconf.set(ConfigType.PARALLEL_LOCAL_OR_REMOTE_PARFOR, false);
+ _cconf.set(ConfigType.ALLOW_DYN_RECOMPILATION, false);
+ _cconf.set(ConfigType.ALLOW_INDIVIDUAL_SB_SPECIFIC_OPS, false);
+ _cconf.set(ConfigType.ALLOW_CSE_PERSISTENT_READS, false);
+ ConfigurationManager.setLocalConfig(_cconf);
+
+ //disable caching globally
CacheableData.disableCaching();
- //create default configuration
- _conf = new DMLConfig();
- ConfigurationManager.setConfig(_conf);
+ //create thread-local default configuration
+ _dmlconf = new DMLConfig();
+ ConfigurationManager.setLocalConfig(_dmlconf);
}
/**
@@ -143,7 +147,7 @@ public class Connection
//lop construct and runtime prog generation
dmlt.constructLops(prog);
- rtprog = prog.getRuntimeProgram(_conf);
+ rtprog = prog.getRuntimeProgram(_dmlconf);
//final cleanup runtime prog
JMLCUtils.cleanupRuntimeProgram(rtprog, outputs);
@@ -162,21 +166,9 @@ public class Connection
/**
*
*/
- public void close()
- {
- //reset parameters for embedded execution
- AParserWrapper.IGNORE_UNSPECIFIED_ARGS = false;
- DataExpression.IGNORE_READ_WRITE_METADATA = false;
- DataExpression.REJECT_READ_WRITE_UNKNOWNS = true;
- OptimizerUtils.ALLOW_CSE_PERSISTENT_READS =
- OptimizerUtils.ALLOW_COMMON_SUBEXPRESSION_ELIMINATION;
- OptimizerUtils.PARALLEL_CP_MATRIX_OPERATIONS = true;
- OptimizerUtils.PARALLEL_LOCAL_OR_REMOTE_PARFOR = true;
- OptimizerUtils.PARALLEL_CP_READ_TEXTFORMATS = true;
- OptimizerUtils.PARALLEL_CP_WRITE_TEXTFORMATS = true;
- OptimizerUtils.PARALLEL_CP_READ_BINARYFORMATS = true;
- OptimizerUtils.PARALLEL_CP_WRITE_BINARYFORMATS = true;
- OptimizerUtils.ALLOW_DYN_RECOMPILATION = true;
+ public void close() {
+ //clear thread-local dml / compiler configs
+ ConfigurationManager.clearLocalConfigs();
}
/**
@@ -250,7 +242,7 @@ public class Connection
//read input matrix
InputStream is = new ByteArrayInputStream(input.getBytes("UTF-8"));
ReaderTextCell reader = (ReaderTextCell)MatrixReaderFactory.createMatrixReader(InputInfo.TextCellInputInfo);
- MatrixBlock mb = reader.readMatrixFromInputStream(is, rows, cols, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, (long)rows*cols);
+ MatrixBlock mb = reader.readMatrixFromInputStream(is, rows, cols, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), (long)rows*cols);
//convert to double array
ret = DataConverter.convertToDoubleMatrix( mb );
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
index e29cca4..498cd24 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
@@ -28,7 +28,6 @@ import java.util.Map.Entry;
import org.apache.sysml.api.DMLException;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
import org.apache.sysml.runtime.controlprogram.Program;
@@ -168,9 +167,8 @@ public class PreparedScript
if( !_inVarnames.contains(varname) )
throw new DMLException("Unspecified input variable: "+varname);
- DMLConfig conf = ConfigurationManager.getConfig();
- String scratch_space = conf.getTextValue(DMLConfig.SCRATCH_SPACE);
- int blocksize = conf.getIntValue(DMLConfig.DEFAULT_BLOCK_SIZE);
+ String scratch_space = ConfigurationManager.getScratchSpace();
+ int blocksize = ConfigurationManager.getBlocksize();
//create new matrix object
MatrixCharacteristics mc = new MatrixCharacteristics(matrix.getNumRows(), matrix.getNumColumns(), blocksize, blocksize);
@@ -233,8 +231,7 @@ public class PreparedScript
if( !_inVarnames.contains(varname) )
throw new DMLException("Unspecified input variable: "+varname);
- DMLConfig conf = ConfigurationManager.getConfig();
- String scratch_space = conf.getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratch_space = ConfigurationManager.getScratchSpace();
//create new frame object
FrameObject fo = new FrameObject(scratch_space+"/"+varname);
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/conf/CompilerConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/conf/CompilerConfig.java b/src/main/java/org/apache/sysml/conf/CompilerConfig.java
new file mode 100644
index 0000000..d0c7f7c
--- /dev/null
+++ b/src/main/java/org/apache/sysml/conf/CompilerConfig.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.conf;
+
+import java.util.HashMap;
+
+import org.apache.sysml.hops.OptimizerUtils;
+
+/**
+ * Basic wrapper for all compiler configurations that are configured
+ * dynamically on a per script invocation basis. This allows us to
+ * provide thread-local compiler configurations to prevent side-effects
+ * between multiple scripts running in the same JVM process.
+ *
+ */
+public class CompilerConfig
+{
+ public enum ConfigType {
+ //Configured compiler optimization level (see OptimizerUtils for defails)
+ OPT_LEVEL,
+ //Configured or automatically determined binary matrix blocksize
+ BLOCK_SIZE,
+ //Enables parallel read/write of text (textcell, csv, mm) and binary formats
+ PARALLEL_CP_READ_TEXTFORMATS,
+ PARALLEL_CP_WRITE_TEXTFORMATS,
+ PARALLEL_CP_READ_BINARYFORMATS,
+ PARALLEL_CP_WRITE_BINARYFORMATS,
+ //Enables multi-threaded operations for mm, mmchain, and tsmm, rand, wdivmm,
+ //wsloss, wumm, wcemm, uagg, tak, and groupedaggregate.
+ PARALLEL_CP_MATRIX_OPERATIONS,
+ //Enables multi-threaded local or distributed remote parfor operators. Otherwise
+ //parfor is restricted to parfor local with par=1.
+ PARALLEL_LOCAL_OR_REMOTE_PARFOR,
+ //Enables dynamic re-compilation of lops/instructions. If enabled, we recompile
+ //each program block that contains at least one hop that requires re-compilation
+ //(e.g., unknown statistics during compilation, or program blocks in functions).
+ ALLOW_DYN_RECOMPILATION,
+ ALLOW_PARALLEL_DYN_RECOMPILATION,
+ //Enables to put operations with data-dependent output size into individual
+ //statement blocks / program blocks. Since recompilation is done on the granularity
+ //of program blocks this enables recompilation of subsequent operations according
+ //to the actual output size. This rewrite might limit the opportunity for piggybacking
+ //and therefore should only be applied if dyanmic recompilation is enabled as well.
+ ALLOW_INDIVIDUAL_SB_SPECIFIC_OPS,
+ //Enables common subexpression elimination in dags for persistent reads based on
+ //filenames and other relevant read meta data. Disabled for jmlc to allow binding of
+ //in-memory objects without specifying read properties.
+ ALLOW_CSE_PERSISTENT_READS,
+
+ //Global parser configuration (dml/pydml) to skip errors on unspecified args
+ // (modified by mlcontext / jmlc)
+ IGNORE_UNSPECIFIED_ARGS,
+ //Data expression configuration (modified by mlcontext, jmlc apis); no read of meta
+ //data on mlcontext (local) /jmlc (global); ignore unknowns on jmlc
+ IGNORE_READ_WRITE_METADATA, // global skip meta data reads
+ REJECT_READ_WRITE_UNKNOWNS, // ignore missing meta data
+ }
+
+ //default flags (exposed for testing purposes only)
+ public static boolean FLAG_DYN_RECOMPILE = true;
+ public static boolean FLAG_PARREAD_TEXT = true;
+
+ private HashMap<ConfigType, Boolean> _bmap = null;
+ private HashMap<ConfigType, Integer> _imap = null;
+
+ public CompilerConfig() {
+ _bmap = new HashMap<ConfigType, Boolean>();
+ _bmap.put(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, FLAG_PARREAD_TEXT);
+ _bmap.put(ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS, true);
+ _bmap.put(ConfigType.PARALLEL_CP_READ_BINARYFORMATS, true);
+ _bmap.put(ConfigType.PARALLEL_CP_WRITE_BINARYFORMATS, true);
+ _bmap.put(ConfigType.PARALLEL_CP_MATRIX_OPERATIONS, true);
+ _bmap.put(ConfigType.PARALLEL_LOCAL_OR_REMOTE_PARFOR, true);
+ _bmap.put(ConfigType.ALLOW_DYN_RECOMPILATION, FLAG_DYN_RECOMPILE);
+ _bmap.put(ConfigType.ALLOW_PARALLEL_DYN_RECOMPILATION, FLAG_DYN_RECOMPILE);
+ _bmap.put(ConfigType.ALLOW_INDIVIDUAL_SB_SPECIFIC_OPS, FLAG_DYN_RECOMPILE);
+ _bmap.put(ConfigType.ALLOW_CSE_PERSISTENT_READS, true);
+ _bmap.put(ConfigType.IGNORE_UNSPECIFIED_ARGS, false);
+ _bmap.put(ConfigType.IGNORE_READ_WRITE_METADATA, false);
+ _bmap.put(ConfigType.REJECT_READ_WRITE_UNKNOWNS, true);
+
+ _imap = new HashMap<CompilerConfig.ConfigType, Integer>();
+ _imap.put(ConfigType.BLOCK_SIZE, OptimizerUtils.DEFAULT_BLOCKSIZE);
+ _imap.put(ConfigType.OPT_LEVEL, OptimizerUtils.DEFAULT_OPTLEVEL.ordinal());
+ }
+
+ @SuppressWarnings("unchecked")
+ public CompilerConfig( CompilerConfig conf ) {
+ _bmap = (HashMap<ConfigType, Boolean>) conf._bmap.clone();
+ _imap = (HashMap<ConfigType, Integer>) conf._imap.clone();
+ }
+
+ /**
+ *
+ * @param key
+ * @param value
+ */
+ public void set( ConfigType key, boolean value ) {
+ _bmap.put(key, value);
+ }
+
+ /**
+ *
+ * @param key
+ * @param value
+ */
+ public void set( ConfigType key, int value ) {
+ _imap.put(key, value);
+ }
+
+ /**
+ *
+ * @param key
+ * @return
+ */
+ public boolean getBool( ConfigType key ) {
+ if( _bmap.containsKey(key) )
+ return _bmap.get(key);
+ return false;
+ }
+
+ /**
+ *
+ * @param key
+ * @return
+ */
+ public int getInt( ConfigType key ) {
+ if( _imap.containsKey(key) )
+ return _imap.get(key);
+ return -1;
+ }
+
+
+ /**
+ *
+ */
+ public CompilerConfig clone() {
+ return new CompilerConfig(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/conf/ConfigurationManager.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/conf/ConfigurationManager.java b/src/main/java/org/apache/sysml/conf/ConfigurationManager.java
index d6bf881..7227bbc 100644
--- a/src/main/java/org/apache/sysml/conf/ConfigurationManager.java
+++ b/src/main/java/org/apache/sysml/conf/ConfigurationManager.java
@@ -20,6 +20,7 @@
package org.apache.sysml.conf;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
@@ -31,51 +32,194 @@ import org.apache.hadoop.mapred.JobConf;
*/
public class ConfigurationManager
{
+ /** Global cached job conf for read-only operations */
+ private static JobConf _rJob = null;
- private static DMLConfig _conf = null; //read systemml configuration
- private static JobConf _rJob = null; //cached job conf for read-only operations
+ /** Global DML configuration (read or defaults) */
+ private static DMLConfig _dmlconf = null;
- static{
+ /** Local DML configuration for thread-local config updates */
+ private static ThreadLocalDMLConfig _ldmlconf = new ThreadLocalDMLConfig(null);
+
+ /** Global compiler configuration (defaults) */
+ private static CompilerConfig _cconf = null;
+
+ /** Local compiler configuration for thead-local config updates */
+ private static ThreadLocalCompilerConfig _lcconf = new ThreadLocalCompilerConfig(null);
+
+ //global static initialization
+ static {
_rJob = new JobConf();
+ _cconf = new CompilerConfig();
+ }
+
+
+ /**
+ * Returns a cached JobConf object, intended for global use by all operations
+ * with read-only access to job conf. This prevents to read the hadoop conf files
+ * over and over again from classpath. However,
+ *
+ * @return
+ */
+ public static JobConf getCachedJobConf() {
+ return _rJob;
+ }
+
+ /**
+ *
+ * @param job
+ */
+ public static void setCachedJobConf(JobConf job) {
+ _rJob = job;
}
+ /**
+ * Sets a global configuration as a basis for any thread-local configurations.
+ * NOTE: This global configuration should never be accessed directly but only
+ * through its thread-local derivatives.
+ *
+ * @param conf
+ */
+ public synchronized static void setGlobalConfig( DMLConfig conf ) {
+ _dmlconf = conf;
+
+ //reinitialize thread-local dml configs
+ _ldmlconf = new ThreadLocalDMLConfig(_dmlconf);
+ }
/**
+ * Sets the current thread-local dml configuration to the given config.
*
* @param conf
*/
- public synchronized static void setConfig( DMLConfig conf )
- {
- _conf = conf;
+ public static void setLocalConfig( DMLConfig conf ) {
+ _ldmlconf.set(conf);
}
/**
+ * Gets the current thread-local dml configuration.
*
* @return
*/
- public synchronized static DMLConfig getConfig()
- {
- return _conf;
+ public static DMLConfig getDMLConfig() {
+ return _ldmlconf.get();
}
- /**
- * Returns a cached JobConf object, intended for global use by all operations
- * with read-only access to job conf. This prevents to read the hadoop conf files
- * over and over again from classpath. However,
- *
- * @return
- */
- public static JobConf getCachedJobConf()
- {
- return _rJob;
+ /**
+ *
+ * @param conf
+ */
+ public synchronized static void setGlobalConfig( CompilerConfig conf ) {
+ _cconf = conf;
+
+ //reinitialize thread-local compiler configs
+ _lcconf = new ThreadLocalCompilerConfig(_cconf);
}
/**
+ * Sets the current thread-local compiler configuration to the given config.
*
- * @param job
+ * @param conf
*/
- public static void setCachedJobConf(JobConf job)
- {
- _rJob = job;
+ public static void setLocalConfig( CompilerConfig conf ) {
+ _lcconf.set(conf);
}
+
+ /**
+ * Removes the thread-local dml and compiler configurations, leading to
+ * a reinitialization on the next get unless set in between.
+ */
+ public static void clearLocalConfigs() {
+ _ldmlconf.remove();
+ _lcconf.remove();
+ }
+
+ /**
+ * Gets the current thread-local compiler configuration.
+ *
+ * @return
+ */
+ public static CompilerConfig getCompilerConfig() {
+ return _lcconf.get();
+ }
+
+ /**
+ * Get a boolean compiler config in a robust manner,
+ * returning false if config not existing.
+ *
+ * @param key
+ * @return
+ */
+ public static boolean getCompilerConfigFlag(ConfigType key) {
+ CompilerConfig cconf = getCompilerConfig();
+ return (cconf!=null) ? cconf.getBool(key) : false;
+ }
+
+ /////////////////////////////////////
+ // shorthand methods for common local configurations
+
+ public static String getScratchSpace() {
+ return getDMLConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
+ }
+
+ public static int getBlocksize() {
+ return getCompilerConfig().getInt(ConfigType.BLOCK_SIZE);
+ }
+
+ public static int getNumReducers() {
+ return getDMLConfig().getIntValue(DMLConfig.NUM_REDUCERS);
+ }
+
+ public static boolean isDynamicRecompilation() {
+ return getCompilerConfigFlag(ConfigType.ALLOW_DYN_RECOMPILATION);
+ }
+
+ public static boolean isParallelMatrixOperations() {
+ return getCompilerConfigFlag(ConfigType.PARALLEL_CP_MATRIX_OPERATIONS);
+ }
+
+ public static boolean isParallelParFor() {
+ return getCompilerConfigFlag(ConfigType.PARALLEL_LOCAL_OR_REMOTE_PARFOR);
+ }
+
+
+ ///////////////////////////////////////
+ // Thread-local classes
+
+ /**
+ *
+ */
+ private static class ThreadLocalDMLConfig extends ThreadLocal<DMLConfig> {
+ private DMLConfig _source = null;
+
+ private ThreadLocalDMLConfig( DMLConfig source ) {
+ _source = source;
+ }
+
+ @Override
+ protected DMLConfig initialValue() {
+ //currently initialize by reference to avoid unnecessary deep copy via clone.
+ if( _source != null )
+ return _source;
+ return null;
+ }
+ }
+
+ /**
+ *
+ */
+ private static class ThreadLocalCompilerConfig extends ThreadLocal<CompilerConfig> {
+ private CompilerConfig _source = null;
+
+ private ThreadLocalCompilerConfig( CompilerConfig source ) {
+ _source = source;
+ }
+
+ @Override
+ protected CompilerConfig initialValue() {
+ if( _source != null )
+ return _source.clone();
+ return null;
+ }
+ };
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/conf/DMLConfig.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/conf/DMLConfig.java b/src/main/java/org/apache/sysml/conf/DMLConfig.java
index df682c8..d65bba1 100644
--- a/src/main/java/org/apache/sysml/conf/DMLConfig.java
+++ b/src/main/java/org/apache/sysml/conf/DMLConfig.java
@@ -43,6 +43,7 @@ import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
+import org.apache.sysml.hops.OptimizerUtils;
import org.apache.sysml.parser.ParseException;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.util.LocalFileUtils;
@@ -81,22 +82,22 @@ public class DMLConfig
//configuration default values
private static HashMap<String, String> _defaultVals = null;
- private String config_file_name = null;
- private Element xml_root = null;
+ private String _fileName = null;
+ private Element _xmlRoot = null;
static
{
_defaultVals = new HashMap<String, String>();
- _defaultVals.put(LOCAL_TMP_DIR, "/tmp/systemml" );
- _defaultVals.put(SCRATCH_SPACE, "scratch_space" );
- _defaultVals.put(OPTIMIZATION_LEVEL, "2" );
- _defaultVals.put(NUM_REDUCERS, "10" );
- _defaultVals.put(JVM_REUSE, "false" );
- _defaultVals.put(DEFAULT_BLOCK_SIZE, "1000" );
- _defaultVals.put(YARN_APPMASTER, "false" );
- _defaultVals.put(YARN_APPMASTERMEM, "2048" );
- _defaultVals.put(YARN_MAPREDUCEMEM, "-1" );
- _defaultVals.put(YARN_APPQUEUE, "default" );
+ _defaultVals.put(LOCAL_TMP_DIR, "/tmp/systemml" );
+ _defaultVals.put(SCRATCH_SPACE, "scratch_space" );
+ _defaultVals.put(OPTIMIZATION_LEVEL, String.valueOf(OptimizerUtils.DEFAULT_OPTLEVEL.ordinal()) );
+ _defaultVals.put(NUM_REDUCERS, "10" );
+ _defaultVals.put(JVM_REUSE, "false" );
+ _defaultVals.put(DEFAULT_BLOCK_SIZE, String.valueOf(OptimizerUtils.DEFAULT_BLOCKSIZE) );
+ _defaultVals.put(YARN_APPMASTER, "false" );
+ _defaultVals.put(YARN_APPMASTERMEM, "2048" );
+ _defaultVals.put(YARN_MAPREDUCEMEM, "-1" );
+ _defaultVals.put(YARN_APPQUEUE, "default" );
_defaultVals.put(CP_PARALLEL_MATRIXMULT, "true" );
_defaultVals.put(CP_PARALLEL_TEXTIO, "true" );
}
@@ -128,7 +129,7 @@ public class DMLConfig
public DMLConfig(String fileName, boolean silent)
throws ParseException, FileNotFoundException
{
- config_file_name = fileName;
+ _fileName = fileName;
try {
parseConfig();
} catch (FileNotFoundException fnfe) {
@@ -144,15 +145,9 @@ public class DMLConfig
LOCAL_MR_MODE_STAGING_DIR = getTextValue(LOCAL_TMP_DIR) + "/hadoop/mapred/staging";
}
-
- public String getConfig_file_name()
- {
- return config_file_name;
- }
-
public DMLConfig( Element root )
{
- xml_root = root;
+ _xmlRoot = root;
}
public void merge(DMLConfig otherConfig)
@@ -163,7 +158,7 @@ public class DMLConfig
try {
// for each element in otherConfig, either overwrite existing value OR add to defaultConfig
- NodeList otherConfigNodeList = otherConfig.xml_root.getChildNodes();
+ NodeList otherConfigNodeList = otherConfig._xmlRoot.getChildNodes();
if (otherConfigNodeList != null && otherConfigNodeList.getLength() > 0){
for (int i=0; i<otherConfigNodeList.getLength(); i++){
org.w3c.dom.Node optionalConfigNode = otherConfigNodeList.item(i);
@@ -174,18 +169,18 @@ public class DMLConfig
String paramName = optionalConfigNode.getNodeName();
String paramValue = ((Element)optionalConfigNode).getFirstChild().getNodeValue();
- if (this.xml_root.getElementsByTagName(paramName) != null)
+ if (_xmlRoot.getElementsByTagName(paramName) != null)
LOG.info("Updating " + paramName + " with value " + paramValue);
else
LOG.info("Defining new attribute" + paramName + " with value " + paramValue);
- DMLConfig.setTextValue(this.xml_root, paramName, paramValue);
+ DMLConfig.setTextValue(_xmlRoot, paramName, paramValue);
}
}
} // end if (otherConfigNodeList != null && otherConfigNodeList.getLength() > 0){
} catch (Exception e){
LOG.error("Failed in merge default config file with optional config file",e);
- throw new ParseException("ERROR: error merging config file" + otherConfig.config_file_name + " with " + config_file_name);
+ throw new ParseException("ERROR: error merging config file" + otherConfig._fileName + " with " + _fileName);
}
}
@@ -201,23 +196,23 @@ public class DMLConfig
factory.setIgnoringComments(true); //ignore XML comments
DocumentBuilder builder = factory.newDocumentBuilder();
Document domTree = null;
- if (config_file_name.startsWith("hdfs:") ||
- config_file_name.startsWith("gpfs:") ) // config file from DFS
+ if (_fileName.startsWith("hdfs:") ||
+ _fileName.startsWith("gpfs:") ) // config file from DFS
{
- if( !LocalFileUtils.validateExternalFilename(config_file_name, true) )
+ if( !LocalFileUtils.validateExternalFilename(_fileName, true) )
throw new IOException("Invalid (non-trustworthy) hdfs config filename.");
FileSystem DFS = FileSystem.get(ConfigurationManager.getCachedJobConf());
- Path configFilePath = new Path(config_file_name);
+ Path configFilePath = new Path(_fileName);
domTree = builder.parse(DFS.open(configFilePath));
}
else // config from local file system
{
- if( !LocalFileUtils.validateExternalFilename(config_file_name, false) )
+ if( !LocalFileUtils.validateExternalFilename(_fileName, false) )
throw new IOException("Invalid (non-trustworthy) local config filename.");
- domTree = builder.parse(config_file_name);
+ domTree = builder.parse(_fileName);
}
- xml_root = domTree.getDocumentElement();
+ _xmlRoot = domTree.getDocumentElement();
}
/**
@@ -229,7 +224,7 @@ public class DMLConfig
public String getTextValue(String tagName)
{
//get the actual value
- String retVal = (xml_root!=null)?getTextValue(xml_root,tagName):null;
+ String retVal = (_xmlRoot!=null)?getTextValue(_xmlRoot,tagName):null;
if (retVal == null)
{
@@ -291,8 +286,8 @@ public class DMLConfig
* @param paramValue
*/
public void setTextValue(String paramName, String paramValue) throws DMLRuntimeException {
- if(this.xml_root != null)
- DMLConfig.setTextValue(this.xml_root, paramName, paramValue);
+ if(_xmlRoot != null)
+ DMLConfig.setTextValue(_xmlRoot, paramName, paramValue);
else {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setIgnoringComments(true); //ignore XML comments
@@ -301,7 +296,7 @@ public class DMLConfig
builder = factory.newDocumentBuilder();
String configString = "<root><" + paramName + ">"+paramValue+"</" + paramName + "></root>";
Document domTree = builder.parse(new ByteArrayInputStream(configString.getBytes("UTF-8")));
- this.xml_root = domTree.getDocumentElement();
+ _xmlRoot = domTree.getDocumentElement();
} catch (Exception e) {
throw new DMLRuntimeException("Unable to set config value", e);
}
@@ -319,11 +314,11 @@ public class DMLConfig
HashMap<String, String> ret = new HashMap<String, String>();
//check for non-existing config xml tree
- if( xml_root == null )
+ if( _xmlRoot == null )
return ret;
//get all mapred.* and mapreduce.* tag / value pairs
- NodeList list = xml_root.getElementsByTagName("*");
+ NodeList list = _xmlRoot.getElementsByTagName("*");
for( int i=0; list!=null && i<list.getLength(); i++ ) {
if( list.item(i) instanceof Element &&
( ((Element)list.item(i)).getNodeName().startsWith(PREFIX_MAPRED)
@@ -353,7 +348,7 @@ public class DMLConfig
transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
//transformer.setOutputProperty(OutputKeys.INDENT, "yes");
StreamResult result = new StreamResult(new StringWriter());
- DOMSource source = new DOMSource(xml_root);
+ DOMSource source = new DOMSource(_xmlRoot);
transformer.transform(source, result);
ret = result.getWriter().toString();
}
@@ -487,14 +482,14 @@ public class DMLConfig
public void updateYarnMemorySettings(String amMem, String mrMem)
{
//app master memory
- NodeList list1 = xml_root.getElementsByTagName(YARN_APPMASTERMEM);
+ NodeList list1 = _xmlRoot.getElementsByTagName(YARN_APPMASTERMEM);
if (list1 != null && list1.getLength() > 0) {
Element elem = (Element) list1.item(0);
elem.getFirstChild().setNodeValue(String.valueOf(amMem));
}
//mapreduce memory
- NodeList list2 = xml_root.getElementsByTagName(YARN_MAPREDUCEMEM);
+ NodeList list2 = _xmlRoot.getElementsByTagName(YARN_MAPREDUCEMEM);
if (list2 != null && list2.getLength() > 0) {
Element elem = (Element) list2.item(0);
elem.getFirstChild().setNodeValue(String.valueOf(mrMem));
@@ -509,7 +504,7 @@ public class DMLConfig
public void makeQualifiedScratchSpacePath()
throws IOException
{
- NodeList list2 = xml_root.getElementsByTagName(SCRATCH_SPACE);
+ NodeList list2 = _xmlRoot.getElementsByTagName(SCRATCH_SPACE);
if (list2 != null && list2.getLength() > 0) {
Element elem = (Element) list2.item(0);
@@ -526,9 +521,18 @@ public class DMLConfig
* @param key
* @return
*/
- public static String getDefaultTextValue( String key )
- {
+ public static String getDefaultTextValue( String key ) {
return _defaultVals.get( key );
}
+ /**
+ *
+ */
+ public DMLConfig clone() {
+ DMLConfig conf = new DMLConfig();
+ conf._fileName = _fileName;
+ conf._xmlRoot = (Element) _xmlRoot.cloneNode(true);
+
+ return conf;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
index f8e1541..02d32ff 100644
--- a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java
@@ -21,6 +21,7 @@ package org.apache.sysml.hops;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.lops.Aggregate;
import org.apache.sysml.lops.Binary;
@@ -449,7 +450,7 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE ) {
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE ) {
setRequiresRecompile();
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
index 4c4d7c8..0152d54 100644
--- a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
@@ -19,6 +19,7 @@
package org.apache.sysml.hops;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
import org.apache.sysml.hops.Hop.MultiThreadedHop;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
@@ -441,7 +442,7 @@ public class AggUnaryOp extends Hop implements MultiThreadedHop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE ) {
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE ) {
setRequiresRecompile();
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/BinaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/BinaryOp.java b/src/main/java/org/apache/sysml/hops/BinaryOp.java
index 7fd0e97..e434197 100644
--- a/src/main/java/org/apache/sysml/hops/BinaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/BinaryOp.java
@@ -19,6 +19,7 @@
package org.apache.sysml.hops;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.lops.Aggregate;
import org.apache.sysml.lops.AppendGAlignedSP;
@@ -766,7 +767,7 @@ public class BinaryOp extends Hop
if( dimsKnown() && _nnz<0 ) //never after inference
nnz = -1;
- if((op==OpOp2.CBIND || op==OpOp2.RBIND) && !OptimizerUtils.ALLOW_DYN_RECOMPILATION && !(getDataType()==DataType.SCALAR) ) {
+ if((op==OpOp2.CBIND || op==OpOp2.RBIND) && !ConfigurationManager.isDynamicRecompilation() && !(getDataType()==DataType.SCALAR) ) {
ret = OptimizerUtils.DEFAULT_SIZE;
}
else
@@ -997,7 +998,7 @@ public class BinaryOp extends Hop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE ) {
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE ) {
setRequiresRecompile();
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/DataGenOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/DataGenOp.java b/src/main/java/org/apache/sysml/hops/DataGenOp.java
index c584d9c..4c81f77 100644
--- a/src/main/java/org/apache/sysml/hops/DataGenOp.java
+++ b/src/main/java/org/apache/sysml/hops/DataGenOp.java
@@ -25,14 +25,12 @@ import java.util.Map.Entry;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.hops.Hop.MultiThreadedHop;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.DataGen;
import org.apache.sysml.lops.LopsException;
import org.apache.sysml.lops.LopProperties.ExecType;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.DataIdentifier;
import org.apache.sysml.parser.DataExpression;
import org.apache.sysml.parser.Expression.DataType;
@@ -109,7 +107,7 @@ public class DataGenOp extends Hop implements MultiThreadedHop
_sparsity = Double.valueOf(((LiteralOp)inputParameters.get(DataExpression.RAND_SPARSITY)).getName());
//generate base dir
- String scratch = ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE);
+ String scratch = ConfigurationManager.getScratchSpace();
_baseDir = scratch + Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID() + Lop.FILE_SEPARATOR +
Lop.FILE_SEPARATOR + ProgramConverter.CP_ROOT_THREAD_ID + Lop.FILE_SEPARATOR;
@@ -165,8 +163,8 @@ public class DataGenOp extends Hop implements MultiThreadedHop
rnd.getOutputParameters().setDimensions(
getDim1(), getDim2(),
//robust handling for blocksize (important for -exec singlenode; otherwise incorrect results)
- (getRowsInBlock()>0)?getRowsInBlock():DMLTranslator.DMLBlockSize,
- (getColsInBlock()>0)?getColsInBlock():DMLTranslator.DMLBlockSize,
+ (getRowsInBlock()>0)?getRowsInBlock():ConfigurationManager.getBlocksize(),
+ (getColsInBlock()>0)?getColsInBlock():ConfigurationManager.getBlocksize(),
//actual rand nnz might differ (in cp/mr they are corrected after execution)
(_op==DataGenMethod.RAND && et==ExecType.SPARK && getNnz()!=0) ? -1 : getNnz(),
getUpdateInPlace());
@@ -225,7 +223,8 @@ public class DataGenOp extends Hop implements MultiThreadedHop
protected double computeIntermediateMemEstimate( long dim1, long dim2, long nnz )
{
if ( _op == DataGenMethod.RAND && dimsKnown() ) {
- long numBlocks = (long) (Math.ceil((double)dim1/DMLTranslator.DMLBlockSize) * Math.ceil((double)dim2/DMLTranslator.DMLBlockSize));
+ long numBlocks = (long) (Math.ceil((double)dim1/ConfigurationManager.getBlocksize())
+ * Math.ceil((double)dim2/ConfigurationManager.getBlocksize()));
return 32 + numBlocks*8.0; // 32 bytes of overhead for an array of long & numBlocks long values.
}
else
@@ -296,7 +295,7 @@ public class DataGenOp extends Hop implements MultiThreadedHop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
setRequiresRecompile();
//always force string initialization into CP (not supported in MR)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/DataOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/DataOp.java b/src/main/java/org/apache/sysml/hops/DataOp.java
index ca447ec..bdaa3b9 100644
--- a/src/main/java/org/apache/sysml/hops/DataOp.java
+++ b/src/main/java/org/apache/sysml/hops/DataOp.java
@@ -19,6 +19,8 @@
package org.apache.sysml.hops;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.lops.Data;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.LopsException;
@@ -461,14 +463,14 @@ public class DataOp extends Hop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE ) {
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE ) {
setRequiresRecompile();
}
}
else //READ
{
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && letype==REMOTE
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && letype==REMOTE
&& (_recompileRead || _requiresCheckpoint) )
{
setRequiresRecompile();
@@ -541,7 +543,8 @@ public class DataOp extends Hop
//with multiple piggybacked csvreblock of the same input w/ unknown input sizes
DataOp that2 = (DataOp)that;
- boolean ret = ( OptimizerUtils.ALLOW_CSE_PERSISTENT_READS
+ boolean ret = ( OptimizerUtils.ALLOW_COMMON_SUBEXPRESSION_ELIMINATION
+ && ConfigurationManager.getCompilerConfigFlag(ConfigType.ALLOW_CSE_PERSISTENT_READS)
&&_dataop == that2._dataop
&& _dataop == DataOpTypes.PERSISTENTREAD
&& _fileName.equals(that2._fileName)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/Hop.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java
index a175817..adf1828 100644
--- a/src/main/java/org/apache/sysml/hops/Hop.java
+++ b/src/main/java/org/apache/sysml/hops/Hop.java
@@ -27,7 +27,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.lops.CSVReBlock;
import org.apache.sysml.lops.Checkpoint;
import org.apache.sysml.lops.Data;
@@ -414,7 +413,7 @@ public abstract class Hop
{
Lop offset = null;
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && hop.dimsKnown() )
+ if( ConfigurationManager.isDynamicRecompilation() && hop.dimsKnown() )
{
// If dynamic recompilation is enabled and dims are known, we can replace the ncol with
// a literal in order to increase the piggybacking potential. This is safe because append
@@ -1798,7 +1797,7 @@ public abstract class Hop
public String constructBaseDir()
{
StringBuilder sb = new StringBuilder();
- sb.append( ConfigurationManager.getConfig().getTextValue(DMLConfig.SCRATCH_SPACE) );
+ sb.append( ConfigurationManager.getScratchSpace() );
sb.append( Lop.FILE_SEPARATOR );
sb.append( Lop.PROCESS_PREFIX );
sb.append( DMLScript.getUUID() );
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/IndexingOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/IndexingOp.java b/src/main/java/org/apache/sysml/hops/IndexingOp.java
index fe32370..4b9901c 100644
--- a/src/main/java/org/apache/sysml/hops/IndexingOp.java
+++ b/src/main/java/org/apache/sysml/hops/IndexingOp.java
@@ -19,6 +19,7 @@
package org.apache.sysml.hops;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.AggBinaryOp.SparkAggType;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.lops.Aggregate;
@@ -363,7 +364,7 @@ public class IndexingOp extends Hop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
setRequiresRecompile();
return _etype;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java b/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
index dc4d912..07091e7 100644
--- a/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
+++ b/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
@@ -19,6 +19,7 @@
package org.apache.sysml.hops;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.lops.Binary;
import org.apache.sysml.lops.Group;
import org.apache.sysml.lops.LeftIndex;
@@ -29,7 +30,6 @@ import org.apache.sysml.lops.UnaryCP;
import org.apache.sysml.lops.ZeroOut;
import org.apache.sysml.lops.LopProperties.ExecType;
import org.apache.sysml.lops.UnaryCP.OperationTypes;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.parser.Expression.DataType;
import org.apache.sysml.parser.Expression.ValueType;
import org.apache.sysml.runtime.matrix.MatrixCharacteristics;
@@ -133,8 +133,8 @@ public class LeftIndexingOp extends Hop
OperationTypes.CAST_AS_MATRIX,
DataType.MATRIX, ValueType.DOUBLE);
rightInput.getOutputParameters().setDimensions( (long)1, (long)1,
- (long)DMLTranslator.DMLBlockSize,
- (long)DMLTranslator.DMLBlockSize,
+ (long)ConfigurationManager.getBlocksize(),
+ (long)ConfigurationManager.getBlocksize(),
(long)-1);
}
else
@@ -194,7 +194,7 @@ public class LeftIndexingOp extends Hop
Lop rightInput = right.constructLops();
if (isRightHandSideScalar()) {
rightInput = new UnaryCP(rightInput, OperationTypes.CAST_AS_MATRIX, DataType.MATRIX, ValueType.DOUBLE);
- long bsize = (long)DMLTranslator.DMLBlockSize;
+ long bsize = ConfigurationManager.getBlocksize();
rightInput.getOutputParameters().setDimensions( 1, 1, bsize, bsize, -1);
}
@@ -391,7 +391,7 @@ public class LeftIndexingOp extends Hop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
setRequiresRecompile();
return _etype;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
index 66ab82b..b4ab7e8 100644
--- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
@@ -26,6 +26,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.conf.CompilerConfig;
+import org.apache.sysml.conf.CompilerConfig.ConfigType;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.Hop.DataOpTypes;
@@ -62,12 +64,20 @@ public class OptimizerUtils
*/
public static double MEM_UTIL_FACTOR = 0.7d;
+ /** Default blocksize if unspecified or for testing purposes */
+ public static final int DEFAULT_BLOCKSIZE = 1000;
+
+ /** Default optimization level if unspecified */
+ public static final OptimizationLevel DEFAULT_OPTLEVEL =
+ OptimizationLevel.O2_LOCAL_MEMORY_DEFAULT;
+
/**
- * Default memory size, which is used the actual estimate can not be computed
- * -- for example, when input/output dimensions are unknown. In case of ROBUST,
- * the default is set to a large value so that operations are scheduled on MR.
+ * Default memory size, which is used if the actual estimate can not be computed
+ * e.g., when input/output dimensions are unknown. The default is set to a large
+ * value so that operations are scheduled on MR while avoiding overflows as well.
*/
- public static double DEFAULT_SIZE;
+ public static double DEFAULT_SIZE = getDefaultSize();
+
public static final long DOUBLE_SIZE = 8;
public static final long INT_SIZE = 4;
@@ -80,26 +90,6 @@ public class OptimizerUtils
public static final long MAX_NUMCELLS_CP_DENSE = Integer.MAX_VALUE;
public static final long MAX_NNZ_CP_SPARSE = (MatrixBlock.DEFAULT_SPARSEBLOCK ==
SparseBlock.Type.MCSR) ? Long.MAX_VALUE : Integer.MAX_VALUE;
-
- /**
- * Enables/disables dynamic re-compilation of lops/instructions.
- * If enabled, we recompile each program block that contains at least
- * one hop that requires re-compilation (e.g., unknown statistics
- * during compilation, or program blocks in functions).
- */
- public static boolean ALLOW_DYN_RECOMPILATION = true;
- public static boolean ALLOW_PARALLEL_DYN_RECOMPILATION = ALLOW_DYN_RECOMPILATION && true;
-
- /**
- * Enables/disables to put operations with data-dependent output
- * size into individual statement blocks / program blocks.
- * Since recompilation is done on the granularity of program blocks
- * this enables recompilation of subsequent operations according
- * to the actual output size. This rewrite might limit the opportunity
- * for piggybacking and therefore should only be applied if
- * dyanmic recompilation is enabled as well.
- */
- public static boolean ALLOW_INDIVIDUAL_SB_SPECIFIC_OPS = ALLOW_DYN_RECOMPILATION && true;
/**
* Enables common subexpression elimination in dags. There is however, a potential tradeoff
@@ -109,13 +99,6 @@ public class OptimizerUtils
public static boolean ALLOW_COMMON_SUBEXPRESSION_ELIMINATION = true;
/**
- * Enables common subexpression elimination in dags for persistent reads based on filenames
- * and other relevant read meta data. Disabled for jmlc to allow binding of in-memory objects
- * without specifying read properties.
- */
- public static boolean ALLOW_CSE_PERSISTENT_READS = ALLOW_COMMON_SUBEXPRESSION_ELIMINATION && true;
-
- /**
* Enables constant folding in dags. Constant folding computes simple expressions of binary
* operations and literals and replaces the hop sub-DAG with a new literal operator.
*/
@@ -188,17 +171,6 @@ public class OptimizerUtils
public static boolean ALLOW_SPLIT_HOP_DAGS = true;
- /**
- * Enables parallel read/write of all text formats (textcell, csv, mm)
- * and binary formats (binary block).
- *
- */
- public static boolean PARALLEL_CP_READ_TEXTFORMATS = true;
- public static boolean PARALLEL_CP_WRITE_TEXTFORMATS = true;
- public static boolean PARALLEL_CP_READ_BINARYFORMATS = true;
- public static boolean PARALLEL_CP_WRITE_BINARYFORMATS = true;
-
-
/**
* Specifies a multiplier computing the degree of parallelism of parallel
@@ -210,19 +182,6 @@ public class OptimizerUtils
public static final double PARALLEL_CP_WRITE_PARALLELISM_MULTIPLIER = 1.0;
/**
- * Enables multi-threaded operations for mm, mmchain, and tsmm,
- * rand, wdivmm, wsloss, wumm, wcemm, uagg, tak, and groupedaggregate.
- *
- */
- public static boolean PARALLEL_CP_MATRIX_OPERATIONS = true;
-
- /**
- * Enables multi-threaded local or distributed remote parfor operators.
- * Otherwise parfor is restricted to parfor local with par=1.
- */
- public static boolean PARALLEL_LOCAL_OR_REMOTE_PARFOR = true;
-
- /**
* Enables the use of CombineSequenceFileInputFormat with splitsize = 2x hdfs blocksize,
* if sort buffer size large enough and parallelism not hurt. This solves to issues:
* (1) it combines small files (depending on producers), and (2) it reduces task
@@ -236,8 +195,6 @@ public class OptimizerUtils
// Optimizer levels //
//////////////////////
- private static OptimizationLevel _optLevel = OptimizationLevel.O2_LOCAL_MEMORY_DEFAULT;
-
/**
* Optimization Types for Compilation
*
@@ -284,15 +241,16 @@ public class OptimizerUtils
};
public static OptimizationLevel getOptLevel() {
- return _optLevel;
+ int optlevel = ConfigurationManager.getCompilerConfig().getInt(ConfigType.OPT_LEVEL);
+ return OptimizationLevel.values()[optlevel];
}
public static boolean isMemoryBasedOptLevel() {
- return (_optLevel != OptimizationLevel.O0_LOCAL_STATIC);
+ return (getOptLevel() != OptimizationLevel.O0_LOCAL_STATIC);
}
public static boolean isOptLevel( OptimizationLevel level ){
- return (_optLevel == level);
+ return (getOptLevel() == level);
}
/**
@@ -300,9 +258,17 @@ public class OptimizerUtils
* @param optlevel
* @throws DMLRuntimeException
*/
- public static void setOptimizationLevel( int optlevel )
+ public static CompilerConfig constructCompilerConfig( DMLConfig dmlconf )
throws DMLRuntimeException
{
+ //create default compiler configuration
+ CompilerConfig cconf = new CompilerConfig();
+
+ //each script sets its own block size, opt level etc
+ cconf.set(ConfigType.BLOCK_SIZE, dmlconf.getIntValue( DMLConfig.DEFAULT_BLOCK_SIZE ));
+
+ //handle optimization level
+ int optlevel = dmlconf.getIntValue(DMLConfig.OPTIMIZATION_LEVEL);
if( optlevel < 0 || optlevel > 5 )
throw new DMLRuntimeException("Error: invalid optimization level '"+optlevel+"' (valid values: 0-5).");
@@ -317,7 +283,7 @@ public class OptimizerUtils
{
// opt level 0: static dimensionality
case 0:
- _optLevel = OptimizationLevel.O0_LOCAL_STATIC;
+ cconf.set(ConfigType.OPT_LEVEL, OptimizationLevel.O0_LOCAL_STATIC.ordinal());
ALLOW_CONSTANT_FOLDING = false;
ALLOW_COMMON_SUBEXPRESSION_ELIMINATION = false;
ALLOW_ALGEBRAIC_SIMPLIFICATION = false;
@@ -328,7 +294,7 @@ public class OptimizerUtils
break;
// opt level 1: memory-based (no advanced rewrites)
case 1:
- _optLevel = OptimizationLevel.O1_LOCAL_MEMORY_MIN;
+ cconf.set(ConfigType.OPT_LEVEL, OptimizationLevel.O1_LOCAL_MEMORY_MIN.ordinal());
ALLOW_CONSTANT_FOLDING = false;
ALLOW_COMMON_SUBEXPRESSION_ELIMINATION = false;
ALLOW_ALGEBRAIC_SIMPLIFICATION = false;
@@ -339,68 +305,71 @@ public class OptimizerUtils
break;
// opt level 2: memory-based (all advanced rewrites)
case 2:
- _optLevel = OptimizationLevel.O2_LOCAL_MEMORY_DEFAULT;
+ cconf.set(ConfigType.OPT_LEVEL, OptimizationLevel.O2_LOCAL_MEMORY_DEFAULT.ordinal());
break;
// opt level 3: resource optimization, time- and memory-based (2 w/ resource optimizat)
case 3:
- _optLevel = OptimizationLevel.O3_LOCAL_RESOURCE_TIME_MEMORY;
+ cconf.set(ConfigType.OPT_LEVEL, OptimizationLevel.O3_LOCAL_RESOURCE_TIME_MEMORY.ordinal());
break;
// opt level 3: global, time- and memory-based (all advanced rewrites)
case 4:
- _optLevel = OptimizationLevel.O4_GLOBAL_TIME_MEMORY;
+ cconf.set(ConfigType.OPT_LEVEL, OptimizationLevel.O4_GLOBAL_TIME_MEMORY.ordinal());
break;
// opt level 4: debug mode (no interfering rewrites)
case 5:
- _optLevel = OptimizationLevel.O5_DEBUG_MODE;
+ cconf.set(ConfigType.OPT_LEVEL, OptimizationLevel.O5_DEBUG_MODE.ordinal());
ALLOW_CONSTANT_FOLDING = false;
ALLOW_COMMON_SUBEXPRESSION_ELIMINATION = false;
ALLOW_ALGEBRAIC_SIMPLIFICATION = false;
ALLOW_INTER_PROCEDURAL_ANALYSIS = false;
ALLOW_BRANCH_REMOVAL = false;
- ALLOW_DYN_RECOMPILATION = false;
ALLOW_SIZE_EXPRESSION_EVALUATION = false;
ALLOW_WORSTCASE_SIZE_EXPRESSION_EVALUATION = false;
ALLOW_RAND_JOB_RECOMPILE = false;
ALLOW_SUM_PRODUCT_REWRITES = false;
ALLOW_SPLIT_HOP_DAGS = false;
+ cconf.set(ConfigType.ALLOW_DYN_RECOMPILATION, false);
+ cconf.set(ConfigType.ALLOW_INDIVIDUAL_SB_SPECIFIC_OPS, false);
break;
}
- setDefaultSize();
//handle parallel text io (incl awareness of thread contention in <jdk8)
- if (!ConfigurationManager.getConfig().getBooleanValue(DMLConfig.CP_PARALLEL_TEXTIO)) {
- PARALLEL_CP_READ_TEXTFORMATS = false;
- PARALLEL_CP_WRITE_TEXTFORMATS = false;
- PARALLEL_CP_READ_BINARYFORMATS = false;
- PARALLEL_CP_WRITE_BINARYFORMATS = false;
+ if (!dmlconf.getBooleanValue(DMLConfig.CP_PARALLEL_TEXTIO)) {
+ cconf.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, false);
+ cconf.set(ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS, false);
+ cconf.set(ConfigType.PARALLEL_CP_READ_BINARYFORMATS, false);
+ cconf.set(ConfigType.PARALLEL_CP_WRITE_BINARYFORMATS, false);
}
else if( InfrastructureAnalyzer.isJavaVersionLessThanJDK8()
&& InfrastructureAnalyzer.getLocalParallelism() > 1 )
{
LOG.warn("Auto-disable multi-threaded text read for 'text' and 'csv' due to thread contention on JRE < 1.8"
- + " (java.version="+ System.getProperty("java.version")+").");
-
- //disable parallel text read
- PARALLEL_CP_READ_TEXTFORMATS = false;
+ + " (java.version="+ System.getProperty("java.version")+").");
+ cconf.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, false);
}
//handle parallel matrix mult / rand configuration
- if (!ConfigurationManager.getConfig().getBooleanValue(DMLConfig.CP_PARALLEL_MATRIXMULT)) {
- PARALLEL_CP_MATRIX_OPERATIONS = false;
+ if (!dmlconf.getBooleanValue(DMLConfig.CP_PARALLEL_MATRIXMULT)) {
+ cconf.set(ConfigType.PARALLEL_CP_MATRIX_OPERATIONS, false);
}
+
+ return cconf;
}
/**
*
*/
- public static void setDefaultSize()
- {
+ public static long getDefaultSize() {
//we need to set default_size larger than any execution context
//memory budget, however, it should not produce overflows on sum
- DEFAULT_SIZE = Math.max( InfrastructureAnalyzer.getLocalMaxMemory(),
- Math.max(InfrastructureAnalyzer.getRemoteMaxMemoryMap(),
- InfrastructureAnalyzer.getRemoteMaxMemoryReduce()));
+ return Math.max( InfrastructureAnalyzer.getLocalMaxMemory(),
+ Math.max(InfrastructureAnalyzer.getRemoteMaxMemoryMap(),
+ InfrastructureAnalyzer.getRemoteMaxMemoryReduce()));
+ }
+
+ public static void resetDefaultSize() {
+ DEFAULT_SIZE = getDefaultSize();
}
/**
@@ -544,7 +513,7 @@ public class OptimizerUtils
*/
public static int getNumReducers( boolean configOnly )
{
- int ret = ConfigurationManager.getConfig().getIntValue(DMLConfig.NUM_REDUCERS);
+ int ret = ConfigurationManager.getNumReducers();
if( !configOnly ) {
ret = Math.min(ret,InfrastructureAnalyzer.getRemoteParallelReduceTasks());
@@ -599,7 +568,7 @@ public class OptimizerUtils
*/
public static int getParallelTextReadParallelism()
{
- if( !PARALLEL_CP_READ_TEXTFORMATS )
+ if( !ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_TEXTFORMATS) )
return 1; // sequential execution
//compute degree of parallelism for parallel text read
@@ -614,7 +583,7 @@ public class OptimizerUtils
*/
public static int getParallelBinaryReadParallelism()
{
- if( !PARALLEL_CP_READ_BINARYFORMATS )
+ if( !ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_BINARYFORMATS) )
return 1; // sequential execution
//compute degree of parallelism for parallel text read
@@ -633,7 +602,7 @@ public class OptimizerUtils
*/
public static int getParallelTextWriteParallelism()
{
- if( !PARALLEL_CP_WRITE_TEXTFORMATS )
+ if( !ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_WRITE_TEXTFORMATS) )
return 1; // sequential execution
//compute degree of parallelism for parallel text read
@@ -648,7 +617,7 @@ public class OptimizerUtils
*/
public static int getParallelBinaryWriteParallelism()
{
- if( !PARALLEL_CP_WRITE_BINARYFORMATS )
+ if( !ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_WRITE_BINARYFORMATS) )
return 1; // sequential execution
//compute degree of parallelism for parallel text read
@@ -954,7 +923,7 @@ public class OptimizerUtils
}
//apply global multi-threading constraint
- if( !PARALLEL_CP_MATRIX_OPERATIONS ) {
+ if( !ConfigurationManager.isParallelMatrixOperations() ) {
ret = 1;
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
index 70915fd..0da75a0 100644
--- a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
+++ b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java
@@ -22,6 +22,7 @@ package org.apache.sysml.hops;
import java.util.HashMap;
import java.util.Map.Entry;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop.MultiThreadedHop;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.lops.Aggregate;
@@ -995,7 +996,7 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
setRequiresRecompile();
return _etype;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/QuaternaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/QuaternaryOp.java b/src/main/java/org/apache/sysml/hops/QuaternaryOp.java
index b011b8e..484a84a 100644
--- a/src/main/java/org/apache/sysml/hops/QuaternaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/QuaternaryOp.java
@@ -19,6 +19,7 @@
package org.apache.sysml.hops;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop.MultiThreadedHop;
import org.apache.sysml.lops.Aggregate;
import org.apache.sysml.lops.DataPartition;
@@ -1613,7 +1614,7 @@ public class QuaternaryOp extends Hop implements MultiThreadedHop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
setRequiresRecompile();
return _etype;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/ReorgOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/ReorgOp.java b/src/main/java/org/apache/sysml/hops/ReorgOp.java
index cfd2f65..a88a098 100644
--- a/src/main/java/org/apache/sysml/hops/ReorgOp.java
+++ b/src/main/java/org/apache/sysml/hops/ReorgOp.java
@@ -21,6 +21,7 @@ package org.apache.sysml.hops;
import java.util.ArrayList;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.lops.Aggregate;
import org.apache.sysml.lops.Group;
@@ -486,7 +487,7 @@ public class ReorgOp extends Hop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
setRequiresRecompile();
return _etype;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/TernaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/TernaryOp.java b/src/main/java/org/apache/sysml/hops/TernaryOp.java
index 562189d..e353273 100644
--- a/src/main/java/org/apache/sysml/hops/TernaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/TernaryOp.java
@@ -19,6 +19,7 @@
package org.apache.sysml.hops;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.lops.Aggregate;
import org.apache.sysml.lops.CentralMoment;
@@ -784,7 +785,7 @@ public class TernaryOp extends Hop
// Necessary condition for recompilation is unknown dimensions.
// When execType=CP, it is marked for recompilation only when additional
// dimension inputs are provided (and those values are unknown at initial compile time).
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) ) {
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) ) {
if ( _etype==REMOTE || (_etype == ExecType.CP && _dimInputsPresent))
setRequiresRecompile();
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/UnaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/UnaryOp.java b/src/main/java/org/apache/sysml/hops/UnaryOp.java
index 4437cdb..fa0e401 100644
--- a/src/main/java/org/apache/sysml/hops/UnaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/UnaryOp.java
@@ -21,6 +21,7 @@ package org.apache.sysml.hops;
import java.util.ArrayList;
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.lops.Aggregate;
import org.apache.sysml.lops.Aggregate.OperationTypes;
import org.apache.sysml.lops.CombineUnary;
@@ -670,7 +671,7 @@ public class UnaryOp extends Hop
}
//mark for recompile (forever)
- if( OptimizerUtils.ALLOW_DYN_RECOMPILATION && !dimsKnown(true) && _etype==REMOTE )
+ if( ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype==REMOTE )
setRequiresRecompile();
//ensure cp exec type for single-node operations
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
index 30cd6d7..00be756 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java
@@ -28,14 +28,13 @@ import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.HopsException;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.LopsException;
import org.apache.sysml.parser.DMLProgram;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.DMLUnsupportedOperationException;
import org.apache.sysml.runtime.controlprogram.ExternalFunctionProgramBlock;
@@ -373,7 +372,7 @@ public abstract class CostEstimator
String varname = iinst.output.getName();
long rlen = iinst.getRows();
long clen = iinst.getCols();
- VarStats vs = new VarStats(rlen, clen, DMLTranslator.DMLBlockSize, DMLTranslator.DMLBlockSize, rlen*clen, true);
+ VarStats vs = new VarStats(rlen, clen, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), rlen*clen, true);
stats.put(varname, vs);
}
else if( inst instanceof FunctionCallCPInstruction )
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java
index b68285f..a4128ac 100644
--- a/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java
+++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java
@@ -23,14 +23,12 @@ import java.util.ArrayList;
import java.util.HashSet;
import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.lops.DataGen;
import org.apache.sysml.lops.Lop;
import org.apache.sysml.lops.MapMult;
import org.apache.sysml.lops.LopProperties.ExecType;
import org.apache.sysml.lops.MMTSJ.MMTSJType;
import org.apache.sysml.lops.compile.JobType;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.DMLUnsupportedOperationException;
import org.apache.sysml.runtime.controlprogram.caching.CacheableData;
@@ -167,7 +165,7 @@ public class CostEstimatorStaticRuntime extends CostEstimator
boolean localJob = InfrastructureAnalyzer.isLocalMode();
int maxPMap = InfrastructureAnalyzer.getRemoteParallelMapTasks();
int maxPRed = Math.min( InfrastructureAnalyzer.getRemoteParallelReduceTasks(),
- ConfigurationManager.getConfig().getIntValue(DMLConfig.NUM_REDUCERS) );
+ ConfigurationManager.getNumReducers() );
double blocksize = ((double)InfrastructureAnalyzer.getHDFSBlockSize())/(1024*1024);
//correction max number of mappers/reducers on yarn clusters
@@ -633,8 +631,8 @@ public class CostEstimatorStaticRuntime extends CostEstimator
default: {
for( int i=0; i<mapOutIx.length; i++ )
{
- int lret = (int) Math.ceil((double)vs[mapOutIx[i]]._rlen/DMLTranslator.DMLBlockSize)
- *(int) Math.ceil((double)vs[mapOutIx[i]]._clen/DMLTranslator.DMLBlockSize);
+ int lret = (int) Math.ceil((double)vs[mapOutIx[i]]._rlen/ConfigurationManager.getBlocksize())
+ *(int) Math.ceil((double)vs[mapOutIx[i]]._clen/ConfigurationManager.getBlocksize());
ret = Math.max(lret, ret);
}
break;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/257793d8/src/main/java/org/apache/sysml/hops/globalopt/GDFEnumOptimizer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/globalopt/GDFEnumOptimizer.java b/src/main/java/org/apache/sysml/hops/globalopt/GDFEnumOptimizer.java
index 678ce92..f3a9188 100644
--- a/src/main/java/org/apache/sysml/hops/globalopt/GDFEnumOptimizer.java
+++ b/src/main/java/org/apache/sysml/hops/globalopt/GDFEnumOptimizer.java
@@ -24,7 +24,7 @@ import java.util.HashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
+import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.DataOp;
import org.apache.sysml.hops.Hop;
import org.apache.sysml.hops.Hop.DataOpTypes;
@@ -44,7 +44,6 @@ import org.apache.sysml.hops.rewrite.HopRewriteUtils;
import org.apache.sysml.hops.recompile.Recompiler;
import org.apache.sysml.lops.LopsException;
import org.apache.sysml.lops.LopProperties.ExecType;
-import org.apache.sysml.parser.DMLTranslator;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.DMLUnsupportedOperationException;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
@@ -579,8 +578,8 @@ public class GDFEnumOptimizer extends GlobalOptimizer
if( hop!=null )
{
hop.setForcedExecType(null);
- hop.setRowsInBlock(DMLTranslator.DMLBlockSize);
- hop.setColsInBlock(DMLTranslator.DMLBlockSize);
+ hop.setRowsInBlock(ConfigurationManager.getBlocksize());
+ hop.setColsInBlock(ConfigurationManager.getBlocksize());
if( !HopRewriteUtils.alwaysRequiresReblock(hop) ) {
hop.setRequiresReblock(false);
}