You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/01/10 23:31:21 UTC
[2/4] incubator-systemml git commit: [SYSTEMML-157][SYSTEMML-300] Fix
python parser (cast function handling)
[SYSTEMML-157][SYSTEMML-300] Fix python parser (cast function handling)
This change fixes a general issue of our python parser that led to not
recognizing cast functions (after mapping to dml names) with '.' in
their names as builtin functions. Due to this problem, any as.scalar,
as.matrix, as.integer, as.logical, as.double in assignment statements
were incorrectly dispatched to functional calls. The fix is a simple
whitelist for now.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/07e3d145
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/07e3d145
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/07e3d145
Branch: refs/heads/master
Commit: 07e3d145583d15bf85bcab93cb2a41cdd8940e02
Parents: 2a6294d
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sat Jan 9 23:09:59 2016 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sat Jan 9 23:09:59 2016 -0800
----------------------------------------------------------------------
.../parser/python/PydmlSyntacticValidator.java | 27 +++++++++++++-------
1 file changed, 18 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/07e3d145/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
index 00f4e4b..8efdbe0 100644
--- a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
+++ b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
@@ -21,6 +21,7 @@ package org.apache.sysml.parser.python;
import java.io.File;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -30,7 +31,6 @@ import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.apache.commons.lang.StringUtils;
-
import org.apache.sysml.parser.AssignmentStatement;
import org.apache.sysml.parser.BinaryExpression;
import org.apache.sysml.parser.BooleanExpression;
@@ -1455,34 +1455,43 @@ public class PydmlSyntacticValidator implements PydmlListener
return;
}
- if(!functionName.contains(".") || functionName.startsWith(DMLProgram.DEFAULT_NAMESPACE)) {
+ //Note: In contrast to the dml parser, namespace and function names are separated by '.' not '::'.
+ //Hence, we have to include a whitelist of function names to handle builtins like 'as.scalar'.
+ String[] whitelist = new String[]{"as.matrix","as.scalar","as.double","as.integer","as.logical"};
+ boolean isWhitelisted = Arrays.asList(whitelist).contains(functionName);
+
+ if( !functionName.contains(".") || isWhitelisted
+ || functionName.startsWith(DMLProgram.DEFAULT_NAMESPACE) )
+ {
// In global namespace, so it can be a builtin function
if(!helper.validateBuiltinFunctions(ctx)) {
return; // it is a built-in function and validation failed, so donot proceed ahead.
}
// Double verification: verify passed function name is a (non-parameterized) built-in function.
- try {
+ try
+ {
+ // builtin functions
BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (bife != null){
- // It is a builtin function
+ if (bife != null) {
setAssignmentStatement(target, bife, ctx);
return;
}
+ // parameterized builtin functions
ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (pbife != null){
- // It is a parameterized builtin function
+ if (pbife != null) {
setAssignmentStatement(target, pbife, ctx);
return;
}
- // built-in read, rand ...
+ // built-in data expressions, e.g. read
DataExpression dbife = DataExpression.getDataExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
if (dbife != null){
setAssignmentStatement(target, dbife, ctx);
return;
}
- } catch(Exception e) {
+ }
+ catch(Exception e) {
helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
return ;
}