You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by de...@apache.org on 2017/02/03 00:36:01 UTC
incubator-systemml git commit: [SYSTEMML-1192] Optionally delete
rmvar instructions via MLContext
Repository: incubator-systemml
Updated Branches:
refs/heads/master e2cb2328b -> 9b77782fa
[SYSTEMML-1192] Optionally delete rmvar instructions via MLContext
Allow option to remove all rmvar instructions so that variables are maintained
in the symbol table, which can be useful when troubleshooting in an interactive
environment such as the Spark Shell. This behavior is off by default and can be
turned on via ml.setMaintainSymbolTable(true).
Closes #352.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/9b77782f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/9b77782f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/9b77782f
Branch: refs/heads/master
Commit: 9b77782fac46a2cdb6e7d91c0d27fc9f175d2b9c
Parents: e2cb232
Author: Deron Eriksson <de...@us.ibm.com>
Authored: Thu Feb 2 16:23:56 2017 -0800
Committer: Deron Eriksson <de...@us.ibm.com>
Committed: Thu Feb 2 16:23:56 2017 -0800
----------------------------------------------------------------------
.../apache/sysml/api/mlcontext/MLContext.java | 31 ++++++++
.../sysml/api/mlcontext/MLContextUtil.java | 76 ++++++++++++++++++++
.../sysml/api/mlcontext/ScriptExecutor.java | 39 ++++++++--
3 files changed, 142 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9b77782f/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
index 401653c..17cb92b 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
@@ -116,6 +116,12 @@ public class MLContext {
*/
private ProjectInfo projectInfo = null;
+ /**
+ * Whether or not all values should be maintained in the symbol table
+ * after execution.
+ */
+ private boolean maintainSymbolTable = false;
+
private List<String> scriptHistoryStrings = new ArrayList<String>();
private Map<String, Script> scripts = new LinkedHashMap<String, Script>();
@@ -273,6 +279,7 @@ public class MLContext {
scriptExecutor.setStatistics(statistics);
scriptExecutor.setStatisticsMaxHeavyHitters(statisticsMaxHeavyHitters);
scriptExecutor.setInit(scriptHistoryStrings.isEmpty());
+ scriptExecutor.setMaintainSymbolTable(maintainSymbolTable);
return execute(script, scriptExecutor);
}
@@ -359,6 +366,30 @@ public class MLContext {
this.explain = explain;
}
+
+ /**
+ * Obtain whether or not all values should be maintained in the symbol table
+ * after execution.
+ *
+ * @return {@code true} if all values should be maintained in the symbol
+ * table, {@code false} otherwise
+ */
+ public boolean isMaintainSymbolTable() {
+ return maintainSymbolTable;
+ }
+
+ /**
+ * Set whether or not all values should be maintained in the symbol table
+ * after execution.
+ *
+ * @param maintainSymbolTable
+ * {@code true} if all values should be maintained in the symbol
+ * table, {@code false} otherwise
+ */
+ public void setMaintainSymbolTable(boolean maintainSymbolTable) {
+ this.maintainSymbolTable = maintainSymbolTable;
+ }
+
/**
* Set the level of program explanation that should be displayed if explain
* is set to true.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9b77782f/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java
index 74e17ee..b4e7f01 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java
@@ -23,6 +23,7 @@ import java.io.FileNotFoundException;
import java.net.URL;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@@ -52,14 +53,22 @@ import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
import org.apache.sysml.parser.ParseException;
import org.apache.sysml.parser.Statement;
+import org.apache.sysml.runtime.controlprogram.ForProgramBlock;
+import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock;
+import org.apache.sysml.runtime.controlprogram.IfProgramBlock;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.ProgramBlock;
+import org.apache.sysml.runtime.controlprogram.WhileProgramBlock;
import org.apache.sysml.runtime.controlprogram.caching.FrameObject;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysml.runtime.instructions.Instruction;
import org.apache.sysml.runtime.instructions.cp.BooleanObject;
import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.instructions.cp.DoubleObject;
import org.apache.sysml.runtime.instructions.cp.IntObject;
import org.apache.sysml.runtime.instructions.cp.StringObject;
+import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction;
import org.apache.sysml.runtime.matrix.data.FrameBlock;
import org.apache.sysml.runtime.matrix.data.MatrixBlock;
import org.apache.sysml.runtime.matrix.data.MatrixIndexes;
@@ -989,4 +998,71 @@ public final class MLContextUtil {
return (symbolTable != null && symbolTable.keySet().contains(variableName)
&& symbolTable.get(variableName) instanceof MatrixObject);
}
+
+ /**
+ * Delete the 'remove variable' instructions from a runtime program.
+ *
+ * @param progam
+ * runtime program
+ */
+ public static void deleteRemoveVariableInstructions(Program progam) {
+ Map<String, FunctionProgramBlock> fpbs = progam.getFunctionProgramBlocks();
+ if (fpbs != null && !fpbs.isEmpty()) {
+ for (Entry<String, FunctionProgramBlock> e : fpbs.entrySet()) {
+ FunctionProgramBlock fpb = e.getValue();
+ for (ProgramBlock pb : fpb.getChildBlocks()) {
+ deleteRemoveVariableInstructions(pb);
+ }
+ }
+ }
+
+ for (ProgramBlock pb : progam.getProgramBlocks()) {
+ deleteRemoveVariableInstructions(pb);
+ }
+ }
+
+ /**
+ * Recursively traverse program block to delete 'remove variable'
+ * instructions.
+ *
+ * @param pb
+ * Program block
+ */
+ private static void deleteRemoveVariableInstructions(ProgramBlock pb) {
+ if (pb instanceof WhileProgramBlock) {
+ WhileProgramBlock wpb = (WhileProgramBlock) pb;
+ for (ProgramBlock pbc : wpb.getChildBlocks())
+ deleteRemoveVariableInstructions(pbc);
+ } else if (pb instanceof IfProgramBlock) {
+ IfProgramBlock ipb = (IfProgramBlock) pb;
+ for (ProgramBlock pbc : ipb.getChildBlocksIfBody())
+ deleteRemoveVariableInstructions(pbc);
+ for (ProgramBlock pbc : ipb.getChildBlocksElseBody())
+ deleteRemoveVariableInstructions(pbc);
+ } else if (pb instanceof ForProgramBlock) {
+ ForProgramBlock fpb = (ForProgramBlock) pb;
+ for (ProgramBlock pbc : fpb.getChildBlocks())
+ deleteRemoveVariableInstructions(pbc);
+ } else {
+ ArrayList<Instruction> instructions = pb.getInstructions();
+ deleteRemoveVariableInstructions(instructions);
+ }
+ }
+
+ /**
+ * Delete 'remove variable' instructions.
+ *
+ * @param instructions
+ * list of instructions
+ */
+ private static void deleteRemoveVariableInstructions(ArrayList<Instruction> instructions) {
+ for (int i = 0; i < instructions.size(); i++) {
+ Instruction linst = instructions.get(i);
+ if (linst instanceof VariableCPInstruction && ((VariableCPInstruction) linst).isRemoveVariable()) {
+ VariableCPInstruction varinst = (VariableCPInstruction) linst;
+ instructions.remove(varinst);
+ i--;
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9b77782f/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
index 7f80267..734475e 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
@@ -119,6 +119,7 @@ public class ScriptExecutor {
protected boolean statistics = false;
protected ExplainLevel explainLevel;
protected int statisticsMaxHeavyHitters = 10;
+ protected boolean maintainSymbolTable = false;
/**
* ScriptExecutor constructor.
@@ -373,12 +374,19 @@ public class ScriptExecutor {
}
/**
- * Remove rmvar instructions so as to maintain registered outputs after the
- * program terminates.
+ * If {@code maintainSymbolTable} is true, delete all 'remove variable'
+ * instructions so as to maintain the values in the symbol table, which are
+ * useful when working interactively in an environment such as the Spark
+ * Shell. Otherwise, only delete 'remove variable' instructions for
+ * registered outputs.
*/
protected void cleanupRuntimeProgram() {
- JMLCUtils.cleanupRuntimeProgram(runtimeProgram, (script.getOutputVariables() == null) ? new String[0] : script
- .getOutputVariables().toArray(new String[0]));
+ if (maintainSymbolTable) {
+ MLContextUtil.deleteRemoveVariableInstructions(runtimeProgram);
+ } else {
+ JMLCUtils.cleanupRuntimeProgram(runtimeProgram, (script.getOutputVariables() == null) ? new String[0]
+ : script.getOutputVariables().toArray(new String[0]));
+ }
}
/**
@@ -648,6 +656,29 @@ public class ScriptExecutor {
}
/**
+ * Obtain whether or not all values should be maintained in the symbol table
+ * after execution.
+ *
+ * @return {@code true} if all values should be maintained in the symbol
+ * table, {@code false} otherwise
+ */
+ public boolean isMaintainSymbolTable() {
+ return maintainSymbolTable;
+ }
+
+ /**
+ * Set whether or not all values should be maintained in the symbol table
+ * after execution.
+ *
+ * @param maintainSymbolTable
+ * {@code true} if all values should be maintained in the symbol
+ * table, {@code false} otherwise
+ */
+ public void setMaintainSymbolTable(boolean maintainSymbolTable) {
+ this.maintainSymbolTable = maintainSymbolTable;
+ }
+
+ /**
* Whether or not to initialize the scratch_space, bufferpool, etc. Note that any
* redundant initialize (e.g., multiple scripts from one MLContext) clears existing
* files from the scratch space and buffer pool.