You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/12/01 06:10:22 UTC
systemml git commit: [SYSTEMML-2033] Fix MLContext handling of
compiler configurations
Repository: systemml
Updated Branches:
refs/heads/master 54cf8f0b8 -> f53fcc8af
[SYSTEMML-2033] Fix MLContext handling of compiler configurations
This patch fixes the handling of compiler configurations passed and
modified via dml configuration properties. So far MLContext never set
the optimization level, which ignored any custom configuration.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/f53fcc8a
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/f53fcc8a
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/f53fcc8a
Branch: refs/heads/master
Commit: f53fcc8af4c4080a02c85a72645946ca508dd642
Parents: 54cf8f0
Author: Matthias Boehm <mb...@gmail.com>
Authored: Thu Nov 30 22:11:04 2017 -0800
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Thu Nov 30 22:11:13 2017 -0800
----------------------------------------------------------------------
.../sysml/api/mlcontext/ScriptExecutor.java | 17 ++++-
.../org/apache/sysml/hops/OptimizerUtils.java | 44 ++++++++----
.../mlcontext/MLContextOptLevelTest.java | 70 ++++++++++++++++++++
.../integration/mlcontext/ZPackageSuite.java | 13 ++--
4 files changed, 123 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/f53fcc8a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
index aa2364d..b00b642 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
@@ -31,6 +31,7 @@ import org.apache.sysml.api.ScriptExecutorUtils;
import org.apache.sysml.api.jmlc.JMLCUtils;
import org.apache.sysml.api.mlcontext.MLContext.ExecutionType;
import org.apache.sysml.api.mlcontext.MLContext.ExplainLevel;
+import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.hops.HopsException;
@@ -249,9 +250,19 @@ public class ScriptExecutor {
oldGPU = DMLScript.USE_ACCELERATOR;
DMLScript.USE_ACCELERATOR = gpu;
DMLScript.STATISTICS_COUNT = statisticsMaxHeavyHitters;
-
- // Sets the GPUs to use for this process (a range, all GPUs, comma separated list or a specific GPU)
- GPUContextPool.AVAILABLE_GPUS = ConfigurationManager.getDMLConfig().getTextValue(DMLConfig.AVAILABLE_GPUS);
+
+ // set the global compiler configuration
+ try {
+ OptimizerUtils.resetStaticCompilerFlags();
+ CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(
+ ConfigurationManager.getCompilerConfig(), config);
+ ConfigurationManager.setGlobalConfig(cconf);
+ } catch(DMLRuntimeException ex) {
+ throw new RuntimeException(ex);
+ }
+
+ // set the GPUs to use for this process (a range, all GPUs, comma separated list or a specific GPU)
+ GPUContextPool.AVAILABLE_GPUS = config.getTextValue(DMLConfig.AVAILABLE_GPUS);
}
/**
http://git-wip-us.apache.org/repos/asf/systemml/blob/f53fcc8a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
index 4684a97..9cc2442 100644
--- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
@@ -263,12 +263,13 @@ public class OptimizerUtils
return (getOptLevel() == level);
}
- public static CompilerConfig constructCompilerConfig( DMLConfig dmlconf )
+ public static CompilerConfig constructCompilerConfig( DMLConfig dmlconf ) throws DMLRuntimeException {
+ return constructCompilerConfig(new CompilerConfig(), dmlconf);
+ }
+
+ public static CompilerConfig constructCompilerConfig( CompilerConfig cconf, DMLConfig dmlconf )
throws DMLRuntimeException
{
- //create default compiler configuration
- CompilerConfig cconf = new CompilerConfig();
-
//each script sets its own block size, opt level etc
cconf.set(ConfigType.BLOCK_SIZE, dmlconf.getIntValue( DMLConfig.DEFAULT_BLOCK_SIZE ));
@@ -319,13 +320,13 @@ public class OptimizerUtils
case 3:
cconf.set(ConfigType.OPT_LEVEL, OptimizationLevel.O3_LOCAL_RESOURCE_TIME_MEMORY.ordinal());
break;
-
+
// opt level 3: global, time- and memory-based (all advanced rewrites)
case 4:
cconf.set(ConfigType.OPT_LEVEL, OptimizationLevel.O4_GLOBAL_TIME_MEMORY.ordinal());
break;
// opt level 4: debug mode (no interfering rewrites)
- case 5:
+ case 5:
cconf.set(ConfigType.OPT_LEVEL, OptimizationLevel.O5_DEBUG_MODE.ordinal());
ALLOW_CONSTANT_FOLDING = false;
ALLOW_COMMON_SUBEXPRESSION_ELIMINATION = false;
@@ -348,12 +349,12 @@ public class OptimizerUtils
cconf.set(ConfigType.OPT_LEVEL, OptimizationLevel.O2_LOCAL_MEMORY_DEFAULT.ordinal());
ALLOW_AUTO_VECTORIZATION = false;
break;
- case 7:
+ case 7:
cconf.set(ConfigType.OPT_LEVEL, OptimizationLevel.O2_LOCAL_MEMORY_DEFAULT.ordinal());
ALLOW_OPERATOR_FUSION = false;
ALLOW_AUTO_VECTORIZATION = false;
ALLOW_SUM_PRODUCT_REWRITES = false;
- break;
+ break;
}
//handle parallel text io (incl awareness of thread contention in <jdk8)
@@ -363,21 +364,40 @@ public class OptimizerUtils
cconf.set(ConfigType.PARALLEL_CP_READ_BINARYFORMATS, false);
cconf.set(ConfigType.PARALLEL_CP_WRITE_BINARYFORMATS, false);
}
- else if( InfrastructureAnalyzer.isJavaVersionLessThanJDK8()
- && InfrastructureAnalyzer.getLocalParallelism() > 1 )
+ else if( InfrastructureAnalyzer.isJavaVersionLessThanJDK8()
+ && InfrastructureAnalyzer.getLocalParallelism() > 1 )
{
LOG.warn("Auto-disable multi-threaded text read for 'text' and 'csv' due to thread contention on JRE < 1.8"
- + " (java.version="+ System.getProperty("java.version")+").");
+ + " (java.version="+ System.getProperty("java.version")+").");
cconf.set(ConfigType.PARALLEL_CP_READ_TEXTFORMATS, false);
}
//handle parallel matrix mult / rand configuration
if (!dmlconf.getBooleanValue(DMLConfig.CP_PARALLEL_OPS)) {
cconf.set(ConfigType.PARALLEL_CP_MATRIX_OPERATIONS, false);
- }
+ }
return cconf;
}
+
+ public static void resetStaticCompilerFlags() {
+ //TODO this is a workaround for MLContext to avoid a major refactoring before the release; this method
+ //should be removed as soon all modified static variables are properly handled in the compiler config
+ ALLOW_ALGEBRAIC_SIMPLIFICATION = true;
+ ALLOW_AUTO_VECTORIZATION = true;
+ ALLOW_BRANCH_REMOVAL = true;
+ ALLOW_CONSTANT_FOLDING = true;
+ ALLOW_COMMON_SUBEXPRESSION_ELIMINATION = true;
+ ALLOW_INTER_PROCEDURAL_ANALYSIS = true;
+ ALLOW_LOOP_UPDATE_IN_PLACE = true;
+ ALLOW_OPERATOR_FUSION = true;
+ ALLOW_RAND_JOB_RECOMPILE = true;
+ ALLOW_SIZE_EXPRESSION_EVALUATION = true;
+ ALLOW_SPLIT_HOP_DAGS = true;
+ ALLOW_SUM_PRODUCT_REWRITES = true;
+ ALLOW_WORSTCASE_SIZE_EXPRESSION_EVALUATION = true;
+ IPA_NUM_REPETITIONS = 3;
+ }
public static long getDefaultSize() {
//we need to set default_size larger than any execution context
http://git-wip-us.apache.org/repos/asf/systemml/blob/f53fcc8a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextOptLevelTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextOptLevelTest.java b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextOptLevelTest.java
new file mode 100644
index 0000000..87f58a7
--- /dev/null
+++ b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextOptLevelTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.mlcontext;
+
+import static org.apache.sysml.api.mlcontext.ScriptFactory.dml;
+
+import org.apache.sysml.api.mlcontext.MLContext.ExplainLevel;
+import org.apache.sysml.api.mlcontext.MLResults;
+import org.apache.sysml.api.mlcontext.Script;
+import org.apache.sysml.conf.DMLConfig;
+import org.apache.sysml.test.utils.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class MLContextOptLevelTest extends MLContextTestBase
+{
+ private final static int rows = 200;
+ private final static int cols = 100;
+
+ @Test
+ public void testOptLevel1() {
+ runMLContextOptLevelTest(1);
+ }
+
+ @Test
+ public void testOptLevel2() {
+ runMLContextOptLevelTest(2);
+ }
+
+ private void runMLContextOptLevelTest(int optLevel)
+ {
+ try
+ {
+ String s = "R = sum(matrix(0,"+rows+","+cols+") + 7);";
+ ml.setExplain(true);
+ ml.setExplainLevel(ExplainLevel.RUNTIME);
+ ml.setStatistics(true);
+ ml.setConfigProperty(DMLConfig.OPTIMIZATION_LEVEL, String.valueOf(optLevel));
+
+ Script script = dml(s).out("R");
+ MLResults results = ml.execute(script);
+
+ //check result correctness
+ TestUtils.compareScalars(results.getDouble("R"), rows*cols*7, 0.000001);
+
+ //check correct opt level
+ Assert.assertTrue(heavyHittersContainsString("+") == (optLevel==1));
+ }
+ catch(Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/systemml/blob/f53fcc8a/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/ZPackageSuite.java
----------------------------------------------------------------------
diff --git a/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/ZPackageSuite.java
index d5a6a21..d5e910e 100644
--- a/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/ZPackageSuite.java
+++ b/src/test_suites/java/org/apache/sysml/test/integration/mlcontext/ZPackageSuite.java
@@ -27,12 +27,13 @@ import org.junit.runners.Suite;
* they should not be run in parallel. */
@RunWith(Suite.class)
@Suite.SuiteClasses({
- MLContextFrameTest.class,
- MLContextMultipleScriptsTest.class,
- MLContextOutputBlocksizeTest.class,
- MLContextParforDatasetTest.class,
- MLContextScratchCleanupTest.class,
- MLContextTest.class
+ MLContextFrameTest.class,
+ MLContextMultipleScriptsTest.class,
+ MLContextOptLevelTest.class,
+ MLContextOutputBlocksizeTest.class,
+ MLContextParforDatasetTest.class,
+ MLContextScratchCleanupTest.class,
+ MLContextTest.class
})