You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by de...@apache.org on 2016/12/09 21:46:57 UTC
incubator-systemml git commit: [SYSTEMML-292] Fix PyDML boolean
output case
Repository: incubator-systemml
Updated Branches:
refs/heads/master 984e0cb17 -> f8677e804
[SYSTEMML-292] Fix PyDML boolean output case
Create global ScriptType variable in DMLScript.
Set global ScriptType in DMLScript, MLContext APIs, and JMLC API.
Add correct case boolean string output method to BooleanObject.
Update ScalarBuiltinCPInstruction, ScalarScalarArithmeticCPInstruction, and
ScalarScalarBuiltinCPInstruction for boolean display.
Add tests to display boolean in DML and PYDML.
Closes #312.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/f8677e80
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/f8677e80
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/f8677e80
Branch: refs/heads/master
Commit: f8677e80498589af0b9c5fb16e2f12f2aa712059
Parents: 984e0cb
Author: Deron Eriksson <de...@us.ibm.com>
Authored: Fri Dec 9 13:42:57 2016 -0800
Committer: Deron Eriksson <de...@us.ibm.com>
Committed: Fri Dec 9 13:42:57 2016 -0800
----------------------------------------------------------------------
src/main/java/org/apache/sysml/api/DMLScript.java | 9 +++++++++
src/main/java/org/apache/sysml/api/MLContext.java | 4 ++++
.../org/apache/sysml/api/jmlc/Connection.java | 3 +++
.../sysml/api/mlcontext/ScriptExecutor.java | 2 ++
.../runtime/instructions/cp/BooleanObject.java | 13 ++++++++++++-
.../cp/ScalarBuiltinCPInstruction.java | 7 ++++++-
.../cp/ScalarScalarArithmeticCPInstruction.java | 14 ++++++++++++--
.../cp/ScalarScalarBuiltinCPInstruction.java | 2 +-
.../test/integration/mlcontext/MLContextTest.java | 18 ++++++++++++++++++
9 files changed, 67 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8677e80/src/main/java/org/apache/sysml/api/DMLScript.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java b/src/main/java/org/apache/sysml/api/DMLScript.java
index 22d8354..3df1320 100644
--- a/src/main/java/org/apache/sysml/api/DMLScript.java
+++ b/src/main/java/org/apache/sysml/api/DMLScript.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
+import org.apache.sysml.api.mlcontext.ScriptType;
import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
@@ -105,6 +106,12 @@ public class DMLScript
public static boolean USE_LOCAL_SPARK_CONFIG = false; //set default local spark configuration - used for local testing
public static String DML_FILE_PATH_ANTLR_PARSER = null;
public static ExplainType EXPLAIN = ExplainType.NONE; //default explain
+ /**
+ * Global variable indicating the script type (DML or PYDML). Can be used
+ * for DML/PYDML-specific tasks, such as outputting booleans in the correct
+ * case (TRUE/FALSE for DML and True/False for PYDML).
+ */
+ public static ScriptType SCRIPT_TYPE = ScriptType.DML;
public static boolean USE_ACCELERATOR = false;
public static boolean FORCE_ACCELERATOR = false;
@@ -584,6 +591,8 @@ public class DMLScript
private static void execute(String dmlScriptStr, String fnameOptConfig, Map<String,String> argVals, String[] allArgs, boolean parsePyDML)
throws ParseException, IOException, DMLRuntimeException, LanguageException, HopsException, LopsException
{
+ SCRIPT_TYPE = parsePyDML ? ScriptType.PYDML : ScriptType.DML;
+
//print basic time and environment info
printStartExecInfo( dmlScriptStr );
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8677e80/src/main/java/org/apache/sysml/api/MLContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/MLContext.java b/src/main/java/org/apache/sysml/api/MLContext.java
index aa64c16..a4f25b8 100644
--- a/src/main/java/org/apache/sysml/api/MLContext.java
+++ b/src/main/java/org/apache/sysml/api/MLContext.java
@@ -39,6 +39,7 @@ import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.SQLContext;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
import org.apache.sysml.api.jmlc.JMLCUtils;
+import org.apache.sysml.api.mlcontext.ScriptType;
import org.apache.sysml.api.monitoring.SparkMonitoringUtil;
import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.conf.CompilerConfig.ConfigType;
@@ -1491,6 +1492,9 @@ public class MLContext {
*/
private synchronized MLOutput compileAndExecuteScript(String dmlScriptFilePath, String [] args, boolean isFile, boolean isNamedArgument, boolean isPyDML, String configFilePath) throws IOException, DMLException {
try {
+
+ DMLScript.SCRIPT_TYPE = isPyDML ? ScriptType.PYDML : ScriptType.DML;
+
if(getActiveMLContext() != null) {
throw new DMLRuntimeException("SystemML (and hence by definition MLContext) doesnot support parallel execute() calls from same or different MLContexts. "
+ "As a temporary fix, please do explicit synchronization, i.e. synchronized(MLContext.class) { ml.execute(...) } ");
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8677e80/src/main/java/org/apache/sysml/api/jmlc/Connection.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
index 6e4951c..a3d7ae7 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.sysml.api.DMLException;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.api.mlcontext.ScriptType;
import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.conf.CompilerConfig.ConfigType;
import org.apache.sysml.conf.ConfigurationManager;
@@ -157,6 +158,8 @@ public class Connection implements Closeable
public PreparedScript prepareScript( String script, Map<String, String> args, String[] inputs, String[] outputs, boolean parsePyDML)
throws DMLException
{
+ DMLScript.SCRIPT_TYPE = parsePyDML ? ScriptType.PYDML : ScriptType.DML;
+
//prepare arguments
//simplified compilation chain
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8677e80/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
index 162c0a0..791557a 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java
@@ -302,6 +302,8 @@ public class ScriptExecutor {
checkScriptHasTypeAndString();
script.setScriptExecutor(this);
setScriptStringInSparkMonitor();
+ // Set global variable indicating the script type
+ DMLScript.SCRIPT_TYPE = script.getScriptType();
// main steps in script execution
parseScript();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8677e80/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanObject.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanObject.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanObject.java
index fa985a9..a55f51a 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanObject.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanObject.java
@@ -19,6 +19,9 @@
package org.apache.sysml.runtime.instructions.cp;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.mlcontext.ScriptType;
import org.apache.sysml.parser.Expression.ValueType;
@@ -57,7 +60,15 @@ public class BooleanObject extends ScalarObject
public String getStringValue(){
return Boolean.toString(_value).toUpperCase();
}
-
+
+ public String getLanguageSpecificBooleanStringValue() {
+ if (DMLScript.SCRIPT_TYPE == ScriptType.DML) {
+ return Boolean.toString(_value).toUpperCase();
+ } else {
+ return StringUtils.capitalize(Boolean.toString(_value));
+ }
+ }
+
@Override
public Object getValue(){
return _value;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8677e80/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinCPInstruction.java
index a9a9994..5738f4a 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarBuiltinCPInstruction.java
@@ -50,7 +50,12 @@ public class ScalarBuiltinCPInstruction extends BuiltinUnaryCPInstruction
//core execution
if ( opcode.equalsIgnoreCase("print") ) {
- String outString = so.getStringValue();
+ String outString = null;
+ if (so instanceof BooleanObject) {
+ outString = ((BooleanObject) so).getLanguageSpecificBooleanStringValue();
+ } else {
+ outString = so.getStringValue();
+ }
// print to stdout only when suppress flag in DMLScript is not set.
// The flag will be set, for example, when SystemML is invoked in fenced mode from Jaql.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8677e80/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarArithmeticCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarArithmeticCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarArithmeticCPInstruction.java
index 11bd779..1e7a44c 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarArithmeticCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarArithmeticCPInstruction.java
@@ -54,9 +54,19 @@ public class ScalarScalarArithmeticCPInstruction extends ArithmeticBinaryCPInstr
if( input1.getValueType() == ValueType.STRING
|| input2.getValueType() == ValueType.STRING )
{
+ String val1 = null;
+ if (so1 instanceof BooleanObject) {
+ val1 = ((BooleanObject) so1).getLanguageSpecificBooleanStringValue();
+ } else {
+ val1 = so1.getStringValue();
+ }
+ String val2 = null;
+ if (so2 instanceof BooleanObject) {
+ val2 = ((BooleanObject) so2).getLanguageSpecificBooleanStringValue();
+ } else {
+ val2 = so2.getStringValue();
+ }
//pre-check (for robustness regarding too long strings)
- String val1 = so1.getStringValue();
- String val2 = so2.getStringValue();
// This line was commented out because of the addition of
// the built-in function toString.
// The toString function adds its own memory estimation
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8677e80/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarBuiltinCPInstruction.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarBuiltinCPInstruction.java
index bc28d05..c831741 100644
--- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarBuiltinCPInstruction.java
+++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ScalarScalarBuiltinCPInstruction.java
@@ -63,7 +63,7 @@ public class ScalarScalarBuiltinCPInstruction extends BuiltinBinaryCPInstruction
System.out.println(buffer + so1.getDoubleValue());
break;
case BOOLEAN:
- System.out.println(buffer + so1.getBooleanValue());
+ System.out.println(buffer + ((BooleanObject) so1).getLanguageSpecificBooleanStringValue());
break;
case STRING:
System.out.println(buffer + so1.getStringValue());
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f8677e80/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java
index dd343d4..4446d56 100644
--- a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java
+++ b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java
@@ -2246,6 +2246,24 @@ public class MLContextTest extends AutomatedTestBase {
ml.execute(script);
}
+ @Test
+ public void testDisplayBooleanDML() {
+ System.out.println("MLContextTest - display boolean DML");
+ String s = "print(b);";
+ Script script = dml(s).in("b", true);
+ setExpectedStdOut("TRUE");
+ ml.execute(script);
+ }
+
+ @Test
+ public void testDisplayBooleanPYDML() {
+ System.out.println("MLContextTest - display boolean PYDML");
+ String s = "print(b)";
+ Script script = pydml(s).in("b", true);
+ setExpectedStdOut("True");
+ ml.execute(script);
+ }
+
// NOTE: Uncomment these tests once they work
// @SuppressWarnings({ "rawtypes", "unchecked" })