You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by de...@apache.org on 2017/07/17 22:32:42 UTC
systemml git commit: [SYSTEMML-1596] Set runtime platform via
MLContext
Repository: systemml
Updated Branches:
refs/heads/master 2625cd903 -> 62b64b32d
[SYSTEMML-1596] Set runtime platform via MLContext
Update MLContext and ScriptExecutor to allow for convenient setting
of the runtime platform. This is useful when debugging and developing.
Closes #497.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/62b64b32
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/62b64b32
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/62b64b32
Branch: refs/heads/master
Commit: 62b64b32d200817fa320a91b5d5a35751a747856
Parents: 2625cd9
Author: Deron Eriksson <de...@apache.org>
Authored: Mon Jul 17 15:31:29 2017 -0700
Committer: Deron Eriksson <de...@apache.org>
Committed: Mon Jul 17 15:31:29 2017 -0700
----------------------------------------------------------------------
.../apache/sysml/api/mlcontext/MLContext.java | 63 +++++++++++++++++++-
.../sysml/api/mlcontext/ScriptExecutor.java | 24 +++++++-
2 files changed, 83 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/62b64b32/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
index a9ab393..b35faa6 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
@@ -28,7 +28,6 @@ import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SparkSession;
import org.apache.sysml.api.DMLScript;
-import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
import org.apache.sysml.api.jmlc.JMLCUtils;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
@@ -114,6 +113,12 @@ public class MLContext {
private ExplainLevel explainLevel = null;
/**
+ * The runtime platform on which to execute. By default, MLContext runs on
+ * {@code ExecutionType.DRIVER_AND_SPARK}.
+ */
+ private ExecutionType executionType = ExecutionType.DRIVER_AND_SPARK;
+
+ /**
* Whether or not all values should be maintained in the symbol table after
* execution.
*/
@@ -160,6 +165,38 @@ public class MLContext {
};
/**
+ * The different types of execution environments supported by SystemML. The
+ * default execution type is {@code DRIVER_AND_SPARK}. {@code DRIVER} refers
+ * to all operations occurring in the local driver JVM. {@code SPARK} refers
+ * to all operations occurring on Spark. {@code HADOOP} refers to all
+ * operations occurring on Hadoop. {@code DRIVER_AND_SPARK} refers to
+ * operations occurring in the local driver JVM and on Spark when
+ * appropriate. {@code DRIVER_AND_HADOOP} refers to operations occurring in
+ * the local driver JVM and on Hadoop when appropriate.
+ *
+ */
+ public enum ExecutionType {
+ DRIVER, SPARK, HADOOP, DRIVER_AND_SPARK, DRIVER_AND_HADOOP;
+
+ public DMLScript.RUNTIME_PLATFORM getRuntimePlatform() {
+ switch (this) {
+ case DRIVER:
+ return DMLScript.RUNTIME_PLATFORM.SINGLE_NODE;
+ case SPARK:
+ return DMLScript.RUNTIME_PLATFORM.SPARK;
+ case HADOOP:
+ return DMLScript.RUNTIME_PLATFORM.HADOOP;
+ case DRIVER_AND_SPARK:
+ return DMLScript.RUNTIME_PLATFORM.HYBRID_SPARK;
+ case DRIVER_AND_HADOOP:
+ return DMLScript.RUNTIME_PLATFORM.HYBRID;
+ default:
+ return DMLScript.RUNTIME_PLATFORM.HYBRID_SPARK;
+ }
+ }
+ }
+
+ /**
* Retrieve the currently active MLContext. This is used internally by
* SystemML via MLContextProxy.
*
@@ -235,8 +272,7 @@ public class MLContext {
}
this.spark = spark;
- // by default, run in hybrid Spark mode for optimal performance
- DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
+ DMLScript.rtplatform = executionType.getRuntimePlatform();
activeMLContext = this;
MLContextProxy.setActive(true);
@@ -279,6 +315,7 @@ public class MLContext {
*/
public MLResults execute(Script script) {
ScriptExecutor scriptExecutor = new ScriptExecutor();
+ scriptExecutor.setExecutionType(executionType);
scriptExecutor.setExplain(explain);
scriptExecutor.setExplainLevel(explainLevel);
scriptExecutor.setGPU(gpu);
@@ -697,4 +734,24 @@ public class MLContext {
this.initBeforeExecution = initBeforeExecution;
}
+ /**
+ * Obtain the current execution environment.
+ *
+ * @return the execution environment
+ */
+ public ExecutionType getExecutionType() {
+ return executionType;
+ }
+
+ /**
+ * Set the execution environment.
+ *
+ * @param executionType
+ * the execution environment
+ */
+ public void setExecutionType(ExecutionType executionType) {
+ DMLScript.rtplatform = executionType.getRuntimePlatform();
+ this.executionType = executionType;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/62b64b32/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
index 1a6a8ed..6d19166 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
@@ -25,9 +25,10 @@ import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.sysml.api.DMLScript;
-import org.apache.sysml.api.ScriptExecutorUtils;
import org.apache.sysml.api.DMLScript.DMLOptions;
+import org.apache.sysml.api.ScriptExecutorUtils;
import org.apache.sysml.api.jmlc.JMLCUtils;
+import org.apache.sysml.api.mlcontext.MLContext.ExecutionType;
import org.apache.sysml.api.mlcontext.MLContext.ExplainLevel;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
@@ -125,6 +126,7 @@ public class ScriptExecutor {
protected boolean statistics = false;
protected boolean oldStatistics = false;
protected ExplainLevel explainLevel;
+ protected ExecutionType executionType;
protected int statisticsMaxHeavyHitters = 10;
protected boolean maintainSymbolTable = false;
@@ -693,4 +695,24 @@ public class ScriptExecutor {
public DMLConfig getConfig() {
return config;
}
+
+ /**
+ * Obtain the current execution environment.
+ *
+ * @return the execution environment
+ */
+ public ExecutionType getExecutionType() {
+ return executionType;
+ }
+
+ /**
+ * Set the execution environment.
+ *
+ * @param executionType
+ * the execution environment
+ */
+ public void setExecutionType(ExecutionType executionType) {
+ DMLScript.rtplatform = executionType.getRuntimePlatform();
+ this.executionType = executionType;
+ }
}