You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2017/07/01 01:07:16 UTC
systemml git commit: [MINOR] Fix mlcontext missing explain header and
statistics reset
Repository: systemml
Updated Branches:
refs/heads/master 9e6715daf -> 31952e47d
[MINOR] Fix mlcontext missing explain header and statistics reset
This patch fixes two minor issues of the new mlcontext. First, it adds
the missing explain header of available memory budgets and degree of
parallelism by calling a new primitive that prepares the explain output
(called for consistency from dmlscript and mlcontext). This also
includes a slight modification of runtime explain which now avoids an
unnecessary pass over the program. Second, it adds reset calls for the
number of executed jobs and cache statistics which would otherwise
accumulate over multiple runs of dml scripts in the same process.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/31952e47
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/31952e47
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/31952e47
Branch: refs/heads/master
Commit: 31952e47d130ada3aab5dfb8dcc058b098ece90c
Parents: 9e6715d
Author: Matthias Boehm <mb...@gmail.com>
Authored: Fri Jun 30 17:27:39 2017 -0700
Committer: Matthias Boehm <mb...@gmail.com>
Committed: Fri Jun 30 18:05:56 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/sysml/api/DMLScript.java | 8 ++---
.../sysml/api/mlcontext/ScriptExecutor.java | 12 +++++--
.../java/org/apache/sysml/utils/Explain.java | 34 ++++++++++++++++----
.../mlcontext/MLContextParforDatasetTest.java | 5 +++
4 files changed, 44 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/31952e47/src/main/java/org/apache/sysml/api/DMLScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java b/src/main/java/org/apache/sysml/api/DMLScript.java
index 6a9077a..6dd7e89 100644
--- a/src/main/java/org/apache/sysml/api/DMLScript.java
+++ b/src/main/java/org/apache/sysml/api/DMLScript.java
@@ -741,12 +741,8 @@ public class DMLScript
Statistics.resetNoOfCompiledJobs( counts.numJobs );
//explain plan of program (hops or runtime)
- if( EXPLAIN != ExplainType.NONE ) {
- LOG.info("EXPLAIN ("+EXPLAIN.toString()+"):\n"
- + Explain.explainMemoryBudget(counts)+"\n"
- + Explain.explainDegreeOfParallelism(counts)
- + Explain.explain(prog, rtprog, EXPLAIN));
- }
+ if( EXPLAIN != ExplainType.NONE )
+ LOG.info(Explain.display(prog, rtprog, EXPLAIN, counts));
Statistics.stopCompileTimer();
http://git-wip-us.apache.org/repos/asf/systemml/blob/31952e47/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
index 0035350..1a6a8ed 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
@@ -47,6 +47,7 @@ import org.apache.sysml.parser.ParserWrapper;
import org.apache.sysml.runtime.DMLRuntimeException;
import org.apache.sysml.runtime.controlprogram.LocalVariableMap;
import org.apache.sysml.runtime.controlprogram.Program;
+import org.apache.sysml.runtime.controlprogram.caching.CacheStatistics;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
import org.apache.sysml.utils.Explain;
@@ -179,8 +180,10 @@ public class ScriptExecutor {
return;
try {
- ExplainType explainType = (explainLevel != null) ? explainLevel.getExplainType() : ExplainType.RUNTIME;
- System.out.println(Explain.explain(dmlProgram, runtimeProgram, explainType));
+ ExplainType explainType = (explainLevel != null) ?
+ explainLevel.getExplainType() : ExplainType.RUNTIME;
+ System.out.println(Explain.display(
+ dmlProgram, runtimeProgram, explainType, null));
} catch (Exception e) {
throw new MLContextException("Exception occurred while explaining dml program", e);
}
@@ -333,7 +336,10 @@ public class ScriptExecutor {
// Set global variable indicating the script type
DMLScript.SCRIPT_TYPE = script.getScriptType();
setGlobalFlags();
- if (statistics) {
+ //reset all relevant summary statistics
+ Statistics.resetNoOfExecutedJobs();
+ if( statistics ) {
+ CacheStatistics.reset();
Statistics.reset();
}
}
http://git-wip-us.apache.org/repos/asf/systemml/blob/31952e47/src/main/java/org/apache/sysml/utils/Explain.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/utils/Explain.java b/src/main/java/org/apache/sysml/utils/Explain.java
index 6451396..bd9d6aa 100644
--- a/src/main/java/org/apache/sysml/utils/Explain.java
+++ b/src/main/java/org/apache/sysml/utils/Explain.java
@@ -105,6 +105,18 @@ public class Explain
//////////////
// public explain interface
+ public static String display(DMLProgram prog, Program rtprog, ExplainType type, ExplainCounts counts)
+ throws HopsException, DMLRuntimeException, LanguageException {
+ if( counts == null )
+ counts = countDistributedOperations(rtprog);
+
+ //explain plan of program (hops or runtime)
+ return "# EXPLAIN ("+type.name()+"):\n"
+ + Explain.explainMemoryBudget(counts)+"\n"
+ + Explain.explainDegreeOfParallelism(counts)
+ + Explain.explain(prog, rtprog, type, counts);
+ }
+
public static String explainMemoryBudget() {
return explainMemoryBudget(new ExplainCounts());
}
@@ -187,6 +199,11 @@ public class Explain
}
public static String explain(DMLProgram prog, Program rtprog, ExplainType type)
+ throws HopsException, DMLRuntimeException, LanguageException {
+ return explain(prog, rtprog, type);
+ }
+
+ public static String explain(DMLProgram prog, Program rtprog, ExplainType type, ExplainCounts counts)
throws HopsException, DMLRuntimeException, LanguageException
{
//dispatch to individual explain utils
@@ -198,7 +215,7 @@ public class Explain
//explain runtime program
case RUNTIME:
case RECOMPILE_RUNTIME:
- return explain(rtprog);
+ return explain(rtprog, counts);
case NONE:
//do nothing
}
@@ -250,13 +267,19 @@ public class Explain
return sb.toString();
}
- public static String explain( Program rtprog )
+ public static String explain( Program rtprog ) throws HopsException {
+ return explain(rtprog, null);
+ }
+
+ public static String explain( Program rtprog, ExplainCounts counts )
throws HopsException
{
//counts number of instructions
boolean sparkExec = OptimizerUtils.isSparkExecutionMode();
- ExplainCounts counts = new ExplainCounts();
- countCompiledInstructions(rtprog, counts, !sparkExec, true, sparkExec);
+ if( counts == null ) {
+ counts = new ExplainCounts();
+ countCompiledInstructions(rtprog, counts, !sparkExec, true, sparkExec);
+ }
StringBuilder sb = new StringBuilder();
@@ -426,8 +449,7 @@ public class Explain
* @param rtprog runtime program
* @return counts
*/
- public static ExplainCounts countDistributedOperations( Program rtprog )
- {
+ public static ExplainCounts countDistributedOperations( Program rtprog ) {
ExplainCounts counts = new ExplainCounts();
if( OptimizerUtils.isSparkExecutionMode() )
Explain.countCompiledInstructions(rtprog, counts, false, true, true);
http://git-wip-us.apache.org/repos/asf/systemml/blob/31952e47/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextParforDatasetTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextParforDatasetTest.java b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextParforDatasetTest.java
index 36e7990..68b1373 100644
--- a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextParforDatasetTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextParforDatasetTest.java
@@ -32,6 +32,7 @@ import org.apache.sysml.api.mlcontext.MLResults;
import org.apache.sysml.api.mlcontext.MatrixFormat;
import org.apache.sysml.api.mlcontext.MatrixMetadata;
import org.apache.sysml.api.mlcontext.Script;
+import org.apache.sysml.api.mlcontext.MLContext.ExplainLevel;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext;
import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
@@ -152,6 +153,10 @@ public class MLContextParforDatasetTest extends AutomatedTestBase
+ "r = sum(v);";
String s = multiInputs ? s2 : s1;
+ ml.setExplain(true);
+ ml.setExplainLevel(ExplainLevel.RUNTIME);
+ ml.setStatistics(true);
+
Script script = dml(s).in("X", df, mm).out("r");
MLResults results = ml.execute(script);