You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by de...@apache.org on 2017/06/01 18:47:17 UTC
incubator-systemml git commit: [SYSTEMML-1562] Remove history from
MLContext
Repository: incubator-systemml
Updated Branches:
refs/heads/master f3b1aebc2 -> 5c3be1e43
[SYSTEMML-1562] Remove history from MLContext
Remove Script history from MLContext for multiple reasons:
1) Reduce number of object references
2) In environments such as Spark Shell, a user typically already
has references to the Script objects
3) History implementation can lead to confusing behavior since the
LinkedHashMap values can point to the same Script object, which
can change over time.
Closes #523.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/5c3be1e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/5c3be1e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/5c3be1e4
Branch: refs/heads/master
Commit: 5c3be1e4398d2cbaa54d86c66fd2052297e02edc
Parents: f3b1aeb
Author: Deron Eriksson <de...@us.ibm.com>
Authored: Thu Jun 1 11:44:37 2017 -0700
Committer: Deron Eriksson <de...@us.ibm.com>
Committed: Thu Jun 1 11:44:37 2017 -0700
----------------------------------------------------------------------
.../apache/sysml/api/mlcontext/MLContext.java | 99 +++++++++-----------
.../sysml/api/mlcontext/MLContextUtil.java | 46 ---------
2 files changed, 43 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5c3be1e4/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
index 8ea9fcc..f4cce99 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
@@ -21,9 +21,6 @@ package org.apache.sysml.api.mlcontext;
import java.util.ArrayList;
import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
import java.util.Set;
import org.apache.log4j.Logger;
@@ -68,9 +65,9 @@ public class MLContext {
private SparkSession spark = null;
/**
- * Reference to the currently executing script.
+ * Reference to the current script.
*/
- private Script executingScript = null;
+ private Script executionScript = null;
/**
* The currently active MLContext.
@@ -122,8 +119,11 @@ public class MLContext {
*/
private boolean maintainSymbolTable = false;
- private List<String> scriptHistoryStrings = new ArrayList<String>();
- private Map<String, Script> scripts = new LinkedHashMap<String, Script>();
+ /**
+ * Whether or not the default ScriptExecutor should be initialized before
+ * execution. See {@link ScriptExecutor#init(boolean)}.
+ */
+ private boolean initBeforeExecution = true;
/**
* The different explain levels supported by SystemML.
@@ -281,7 +281,10 @@ public class MLContext {
scriptExecutor.setForceGPU(forceGPU);
scriptExecutor.setStatistics(statistics);
scriptExecutor.setStatisticsMaxHeavyHitters(statisticsMaxHeavyHitters);
- scriptExecutor.setInit(scriptHistoryStrings.isEmpty());
+ scriptExecutor.setInit(initBeforeExecution);
+ if (initBeforeExecution) {
+ initBeforeExecution = false;
+ }
scriptExecutor.setMaintainSymbolTable(maintainSymbolTable);
return execute(script, scriptExecutor);
}
@@ -299,7 +302,7 @@ public class MLContext {
*/
public MLResults execute(Script script, ScriptExecutor scriptExecutor) {
try {
- executingScript = script;
+ executionScript = script;
Long time = new Long((new Date()).getTime());
if ((script.getName() == null) || (script.getName().equals(""))) {
@@ -308,10 +311,6 @@ public class MLContext {
MLResults results = scriptExecutor.execute(script);
- String history = MLContextUtil.createHistoryForScript(script, time);
- scriptHistoryStrings.add(history);
- scripts.put(script.getName(), script);
-
return results;
} catch (RuntimeException e) {
throw new MLContextException("Exception when executing script", e);
@@ -513,8 +512,8 @@ public class MLContext {
}
private boolean isRegisteredAsInput(String parameterName) {
- if (executingScript != null) {
- Set<String> inputVariableNames = executingScript.getInputVariables();
+ if (executionScript != null) {
+ Set<String> inputVariableNames = executionScript.getInputVariables();
if (inputVariableNames != null) {
return inputVariableNames.contains(parameterName);
}
@@ -523,8 +522,8 @@ public class MLContext {
}
private MatrixObject getMatrixObject(String parameterName) {
- if (executingScript != null) {
- LocalVariableMap symbolTable = executingScript.getSymbolTable();
+ if (executionScript != null) {
+ LocalVariableMap symbolTable = executionScript.getSymbolTable();
if (symbolTable != null) {
Data data = symbolTable.get(parameterName);
if (data instanceof MatrixObject) {
@@ -540,10 +539,10 @@ public class MLContext {
}
public ArrayList<Instruction> performCleanupAfterRecompilation(ArrayList<Instruction> instructions) {
- if (executingScript == null || executingScript.getOutputVariables() == null)
+ if (executionScript == null || executionScript.getOutputVariables() == null)
return instructions;
- Set<String> outputVariableNames = executingScript.getOutputVariables();
+ Set<String> outputVariableNames = executionScript.getOutputVariables();
return JMLCUtils.cleanupRuntimeInstructions(instructions, outputVariableNames.toArray(new String[0]));
}
}
@@ -594,39 +593,6 @@ public class MLContext {
}
/**
- * Obtain a map of the scripts that have executed.
- *
- * @return a map of the scripts that have executed
- */
- public Map<String, Script> getScripts() {
- return scripts;
- }
-
- /**
- * Obtain a script that has executed by name.
- *
- * @param name
- * the name of the script
- * @return the script corresponding to the name
- */
- public Script getScriptByName(String name) {
- Script script = scripts.get(name);
- if (script == null) {
- throw new MLContextException("Script with name '" + name + "' not found.");
- }
- return script;
- }
-
- /**
- * Display the history of scripts that have executed.
- *
- * @return the history of scripts that have executed
- */
- public String history() {
- return MLContextUtil.displayScriptHistory(scriptHistoryStrings);
- }
-
- /**
* Closes the mlcontext, which includes the cleanup of static and local
* state as well as scratch space and buffer pool cleanup. Note that the
* spark context is not explicitly closed to allow external reuse.
@@ -646,10 +612,7 @@ public class MLContext {
// clear local status, but do not stop sc as it
// may be used or stopped externally
- for (Script script : scripts.values())
- script.clearAll();
- scripts.clear();
- scriptHistoryStrings.clear();
+ executionScript.clearAll();
resetConfig();
spark = null;
}
@@ -703,4 +666,28 @@ public class MLContext {
public int getStatisticsMaxHeavyHitters() {
return statisticsMaxHeavyHitters;
}
+
+ /**
+ * Whether or not the default ScriptExecutor should be initialized before
+ * execution.
+ *
+ * @return {@code true} if ScriptExecutor should be initialized before
+ * execution, {@code false} otherwise
+ */
+ public boolean isInitBeforeExecution() {
+ return initBeforeExecution;
+ }
+
+ /**
+ * Whether or not the default ScriptExecutor should be initialized before
+ * execution.
+ *
+ * @param initBeforeExecution
+ * {@code true} if ScriptExecutor should be initialized before
+ * execution, {@code false} otherwise
+ */
+ public void setInitBeforeExecution(boolean initBeforeExecution) {
+ this.initBeforeExecution = initBeforeExecution;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5c3be1e4/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java
index 43365dd..bc55e41 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java
@@ -22,12 +22,8 @@ package org.apache.sysml.api.mlcontext;
import java.io.File;
import java.io.FileNotFoundException;
import java.net.URL;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.Date;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Scanner;
@@ -988,48 +984,6 @@ public final class MLContextUtil {
}
/**
- * Generate a String history entry for a script.
- *
- * @param script
- * the script
- * @param when
- * when the script was executed
- * @return a script history entry as a String
- */
- public static String createHistoryForScript(Script script, long when) {
- DateFormat dateFormat = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss.SSS");
- StringBuilder sb = new StringBuilder();
- sb.append("Script Name: " + script.getName() + "\n");
- sb.append("When: " + dateFormat.format(new Date(when)) + "\n");
- sb.append(script.displayInputs());
- sb.append(script.displayOutputs());
- sb.append(script.displaySymbolTable());
- return sb.toString();
- }
-
- /**
- * Generate a String listing of the script execution history.
- *
- * @param scriptHistory
- * the list of script history entries
- * @return the listing of the script execution history as a String
- */
- public static String displayScriptHistory(List<String> scriptHistory) {
- StringBuilder sb = new StringBuilder();
- sb.append("MLContext Script History:\n");
- if (scriptHistory.isEmpty()) {
- sb.append("None");
- }
- int i = 1;
- for (String history : scriptHistory) {
- sb.append("--------------------------------------------\n");
- sb.append("#" + (i++) + ":\n");
- sb.append(history);
- }
- return sb.toString();
- }
-
- /**
* Obtain the Spark Context
*
* @param mlContext