You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by du...@apache.org on 2016/09/02 22:08:04 UTC
incubator-systemml git commit: [SYSTEMML-886] Add setStatistics and
setExplain to the PythonAPI
Repository: incubator-systemml
Updated Branches:
refs/heads/master 701b9e319 -> b7657dbc3
[SYSTEMML-886] Add setStatistics and setExplain to the PythonAPI
Closes #226.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/b7657dbc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/b7657dbc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/b7657dbc
Branch: refs/heads/master
Commit: b7657dbc32187d1a983f3adbbd618082ec8cec76
Parents: 701b9e3
Author: MechCoder <mk...@nyu.edu>
Authored: Fri Sep 2 15:06:14 2016 -0700
Committer: Mike Dusenberry <mw...@us.ibm.com>
Committed: Fri Sep 2 15:06:14 2016 -0700
----------------------------------------------------------------------
.../apache/sysml/api/mlcontext/MLContext.java | 67 +++++++++++++-------
src/main/python/systemml/mlcontext.py | 38 ++++++++++-
2 files changed, 81 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b7657dbc/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
index be37575..88c937b 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -146,7 +146,7 @@ public class MLContext {
/**
* Retrieve the currently active MLContext. This is used internally by
* SystemML via MLContextProxy.
- *
+ *
* @return the active MLContext
*/
public static MLContext getActiveMLContext() {
@@ -156,7 +156,7 @@ public class MLContext {
/**
* Create an MLContext based on a SparkContext for interaction with SystemML
* on Spark.
- *
+ *
* @param sparkContext
* SparkContext
*/
@@ -167,7 +167,7 @@ public class MLContext {
/**
* Create an MLContext based on a JavaSparkContext for interaction with
* SystemML on Spark.
- *
+ *
* @param javaSparkContext
* JavaSparkContext
*/
@@ -178,7 +178,7 @@ public class MLContext {
/**
* Create an MLContext based on a SparkContext for interaction with SystemML
* on Spark, optionally monitor performance.
- *
+ *
* @param sc
* SparkContext object.
* @param monitorPerformance
@@ -193,7 +193,7 @@ public class MLContext {
* Initialize MLContext. Verify Spark version supported, set default
* execution mode, set MLContextProxy, set default config, set compiler
* config, and configure monitoring if needed.
- *
+ *
* @param sc
* SparkContext object.
* @param monitorPerformance
@@ -242,7 +242,7 @@ public class MLContext {
/**
* Set configuration property, such as
* {@code setConfigProperty("localtmpdir", "/tmp/systemml")}.
- *
+ *
* @param propertyName
* property name
* @param propertyValue
@@ -259,7 +259,7 @@ public class MLContext {
/**
* Execute a DML or PYDML Script.
- *
+ *
* @param script
* The DML or PYDML Script object to execute.
*/
@@ -275,7 +275,7 @@ public class MLContext {
* Execute a DML or PYDML Script object using a ScriptExecutor. The
* ScriptExecutor class can be extended to allow the modification of the
* default execution pathway.
- *
+ *
* @param script
* the DML or PYDML Script object
* @param scriptExecutor
@@ -304,7 +304,7 @@ public class MLContext {
/**
* Set SystemML configuration based on a configuration file.
- *
+ *
* @param configFilePath
* path to the configuration file
*/
@@ -314,7 +314,7 @@ public class MLContext {
/**
* Obtain the SparkMonitoringUtil if it is available.
- *
+ *
* @return the SparkMonitoringUtil if it is available.
*/
public SparkMonitoringUtil getSparkMonitoringUtil() {
@@ -323,7 +323,7 @@ public class MLContext {
/**
* Obtain the SparkContext associated with this MLContext.
- *
+ *
* @return the SparkContext associated with this MLContext.
*/
public SparkContext getSparkContext() {
@@ -333,7 +333,7 @@ public class MLContext {
/**
* Whether or not an explanation of the DML/PYDML program should be output
* to standard output.
- *
+ *
* @return {@code true} if explanation should be output, {@code false}
* otherwise
*/
@@ -344,7 +344,7 @@ public class MLContext {
/**
* Whether or not an explanation of the DML/PYDML program should be output
* to standard output.
- *
+ *
* @param explain
* {@code true} if explanation should be output, {@code false}
* otherwise
@@ -356,7 +356,7 @@ public class MLContext {
/**
* Set the level of program explanation that should be displayed if explain
* is set to true.
- *
+ *
* @param explainLevel
* the level of program explanation
*/
@@ -365,6 +365,27 @@ public class MLContext {
}
/**
+ * Set the level of program explanation that should be displayed if explain
+ * is set to true.
+ *
+ * @param explainLevel
+ * string denoting program explanation
+ */
+ public void setExplainLevel(String explainLevel) {
+ if (explainLevel != null) {
+ for (ExplainLevel exp : ExplainLevel.values()) {
+ String expString = exp.toString();
+ if (expString.equalsIgnoreCase(explainLevel)) {
+ setExplainLevel(exp);
+ return;
+ }
+ }
+ }
+ throw new MLContextException("Failed to parse explain level: "+ explainLevel + " " +
+ "(valid types: hops, runtime, recompile_hops, recompile_runtime).");
+ }
+
+ /**
* Used internally by MLContextProxy.
*
*/
@@ -377,7 +398,7 @@ public class MLContext {
DataExpression exp = (DataExpression) source;
// Do not check metadata file for registered reads
exp.setCheckMetadata(false);
-
+
//Value retured from getVarParam is of type stringidentifier at runtime, but at compile type its Expression
//Could not find better way to compare this condition.
Expression datatypeExp = ((DataExpression)source).getVarParam("data_type");
@@ -402,7 +423,7 @@ public class MLContext {
blp, bcp, elp, ecp));
exp.addVarParam(DataExpression.VALUETYPEPARAM, new StringIdentifier("double", source.getFilename(),
blp, bcp, elp, ecp));
-
+
if (mo.getMetaData() instanceof MatrixFormatMetaData) {
MatrixFormatMetaData metaData = (MatrixFormatMetaData) mo.getMetaData();
if (metaData.getOutputInfo() == OutputInfo.CSVOutputInfo) {
@@ -492,7 +513,7 @@ public class MLContext {
/**
* Whether or not statistics of the DML/PYDML program execution should be
* output to standard output.
- *
+ *
* @return {@code true} if statistics should be output, {@code false}
* otherwise
*/
@@ -503,7 +524,7 @@ public class MLContext {
/**
* Whether or not statistics of the DML/PYDML program execution should be
* output to standard output.
- *
+ *
* @param statistics
* {@code true} if statistics should be output, {@code false}
* otherwise
@@ -515,7 +536,7 @@ public class MLContext {
/**
* Obtain a map of the scripts that have executed.
- *
+ *
* @return a map of the scripts that have executed
*/
public Map<String, Script> getScripts() {
@@ -524,7 +545,7 @@ public class MLContext {
/**
* Obtain a script that has executed by name.
- *
+ *
* @param name
* the name of the script
* @return the script corresponding to the name
@@ -539,7 +560,7 @@ public class MLContext {
/**
* Display the history of scripts that have executed.
- *
+ *
* @return the history of scripts that have executed
*/
public String history() {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b7657dbc/src/main/python/systemml/mlcontext.py
----------------------------------------------------------------------
diff --git a/src/main/python/systemml/mlcontext.py b/src/main/python/systemml/mlcontext.py
index c578a8e..da8e025 100644
--- a/src/main/python/systemml/mlcontext.py
+++ b/src/main/python/systemml/mlcontext.py
@@ -19,6 +19,8 @@
#
#-------------------------------------------------------------
+__all__ = ['MLResults', 'MLContext', 'Script', 'dml', 'pydml']
+
import os
try:
@@ -26,6 +28,7 @@ try:
from py4j.java_gateway import JavaObject
except ImportError:
raise ImportError('Unable to import JavaObject from py4j.java_gateway. Hint: Make sure you are running with pyspark')
+
from pyspark import SparkContext
import pyspark.mllib.common
from pyspark.sql import DataFrame, SQLContext
@@ -298,5 +301,38 @@ class MLContext(object):
script_java.out(val)
return MLResults(self._ml.execute(script_java), self._sc)
+ def setStatistics(self, statistics):
+ """
+ Whether or not to output statistics (such as execution time, elapsed time)
+ about script executions.
-__all__ = ['MLResults', 'MLContext', 'Script', 'dml', 'pydml']
+ Parameters
+ ----------
+ statistics: boolean
+ """
+ self._ml.setStatistics(bool(statistics))
+ return self
+
+ def setExplain(self, explain):
+ """
+ Explanation about the program. Mainly intended for developers.
+
+ Parameters
+ ----------
+ explain: boolean
+ """
+ self._ml.setExplain(bool(explain))
+ return self
+
+ def setExplainLevel(self, explainLevel):
+ """
+ Set explain level.
+
+ Parameters
+ ----------
+ explainLevel: string
+ Can be one of "hops", "runtime", "recompile_hops", "recompile_runtime"
+ or in the above in upper case.
+ """
+ self._ml.setExplainLevel(explainLevel)
+ return self