You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by mb...@apache.org on 2016/01/10 23:31:20 UTC
[1/4] incubator-systemml git commit: [SYSTEMML-146] Fix missing
existence check for read input files
Repository: incubator-systemml
Updated Branches:
refs/heads/master 91a1863b9 -> 572752a9e
[SYSTEMML-146] Fix missing existence check for read input files
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/2a6294d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/2a6294d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/2a6294d3
Branch: refs/heads/master
Commit: 2a6294d390dcc6e43deef77636758e12dc48d138
Parents: 91a1863
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sat Jan 9 21:40:53 2016 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sat Jan 9 21:40:53 2016 -0800
----------------------------------------------------------------------
.../org/apache/sysml/parser/DataExpression.java | 19 ++++++++++++++-----
1 file changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/2a6294d3/src/main/java/org/apache/sysml/parser/DataExpression.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/DataExpression.java b/src/main/java/org/apache/sysml/parser/DataExpression.java
index 5742871..5bd92ea 100644
--- a/src/main/java/org/apache/sysml/parser/DataExpression.java
+++ b/src/main/java/org/apache/sysml/parser/DataExpression.java
@@ -31,11 +31,12 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.wink.json4j.JSONArray;
import org.apache.wink.json4j.JSONObject;
-
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.hops.DataGenOp;
import org.apache.sysml.parser.LanguageException.LanguageErrorCodes;
+import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer;
import org.apache.sysml.runtime.util.LocalFileUtils;
+import org.apache.sysml.runtime.util.MapReduceTool;
import org.apache.sysml.runtime.util.UtilFunctions;
import org.apache.sysml.utils.JSONHelper;
@@ -486,7 +487,7 @@ public class DataExpression extends DataIdentifier
_varParams.remove(name);
}
- private String processInputFileName(HashMap<String, ConstIdentifier> currConstVars, boolean conditional)
+ private String getInputFileName(HashMap<String, ConstIdentifier> currConstVars, boolean conditional)
throws LanguageException
{
String filename = null;
@@ -615,15 +616,23 @@ public class DataExpression extends DataIdentifier
JSONObject configObject = null;
// Process expressions in input filename
- String inputFileName = processInputFileName(currConstVars, conditional);
+ String inputFileName = getInputFileName(currConstVars, conditional);
// Obtain and validate metadata filename
String mtdFileName = getMTDFileName(inputFileName);
-
// track whether should attempt to read MTD file or not
boolean shouldReadMTD = checkMetadata;
-
+
+ // Check for file existence (before metadata parsing for meaningful error messages)
+ if( shouldReadMTD && REJECT_READ_UNKNOWN_SIZE //skip check for jmlc/mlcontext apis
+ && !MapReduceTool.existsFileOnHDFS(inputFileName))
+ {
+ String fsext = InfrastructureAnalyzer.isLocalMode() ? "FS (local mode)" : "HDFS";
+ raiseValidateError("Read input file does not exist on "+fsext+": " +
+ inputFileName, conditional, LanguageErrorCodes.INVALID_PARAMETERS);
+ }
+
// track whether format type has been inferred
boolean inferredFormatType = false;
[2/4] incubator-systemml git commit: [SYSTEMML-157][SYSTEMML-300] Fix
python parser (cast function handling)
Posted by mb...@apache.org.
[SYSTEMML-157][SYSTEMML-300] Fix python parser (cast function handling)
This change fixes a general issue of our python parser that led to not
recognizing cast functions (after mapping to dml names) with '.' in
their names as builtin functions. Due to this problem, any as.scalar,
as.matrix, as.integer, as.logical, as.double in assignment statements
were incorrectly dispatched to functional calls. The fix is a simple
whitelist for now.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/07e3d145
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/07e3d145
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/07e3d145
Branch: refs/heads/master
Commit: 07e3d145583d15bf85bcab93cb2a41cdd8940e02
Parents: 2a6294d
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sat Jan 9 23:09:59 2016 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sat Jan 9 23:09:59 2016 -0800
----------------------------------------------------------------------
.../parser/python/PydmlSyntacticValidator.java | 27 +++++++++++++-------
1 file changed, 18 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/07e3d145/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
index 00f4e4b..8efdbe0 100644
--- a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
+++ b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
@@ -21,6 +21,7 @@ package org.apache.sysml.parser.python;
import java.io.File;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -30,7 +31,6 @@ import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.apache.commons.lang.StringUtils;
-
import org.apache.sysml.parser.AssignmentStatement;
import org.apache.sysml.parser.BinaryExpression;
import org.apache.sysml.parser.BooleanExpression;
@@ -1455,34 +1455,43 @@ public class PydmlSyntacticValidator implements PydmlListener
return;
}
- if(!functionName.contains(".") || functionName.startsWith(DMLProgram.DEFAULT_NAMESPACE)) {
+ //Note: In contrast to the dml parser, namespace and function names are separated by '.' not '::'.
+ //Hence, we have to include a whitelist of function names to handle builtins like 'as.scalar'.
+ String[] whitelist = new String[]{"as.matrix","as.scalar","as.double","as.integer","as.logical"};
+ boolean isWhitelisted = Arrays.asList(whitelist).contains(functionName);
+
+ if( !functionName.contains(".") || isWhitelisted
+ || functionName.startsWith(DMLProgram.DEFAULT_NAMESPACE) )
+ {
// In global namespace, so it can be a builtin function
if(!helper.validateBuiltinFunctions(ctx)) {
return; // it is a built-in function and validation failed, so donot proceed ahead.
}
// Double verification: verify passed function name is a (non-parameterized) built-in function.
- try {
+ try
+ {
+ // builtin functions
BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (bife != null){
- // It is a builtin function
+ if (bife != null) {
setAssignmentStatement(target, bife, ctx);
return;
}
+ // parameterized builtin functions
ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (pbife != null){
- // It is a parameterized builtin function
+ if (pbife != null) {
setAssignmentStatement(target, pbife, ctx);
return;
}
- // built-in read, rand ...
+ // built-in data expressions, e.g. read
DataExpression dbife = DataExpression.getDataExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
if (dbife != null){
setAssignmentStatement(target, dbife, ctx);
return;
}
- } catch(Exception e) {
+ }
+ catch(Exception e) {
helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
return ;
}
[3/4] incubator-systemml git commit: [SYSTEMML-155] Cleanup python
parser (type messages, commented code)
Posted by mb...@apache.org.
[SYSTEMML-155] Cleanup python parser (type messages, commented code)
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/888c9ca4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/888c9ca4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/888c9ca4
Branch: refs/heads/master
Commit: 888c9ca483461cd09d53bfdcf861bb75ff1c37c0
Parents: 07e3d14
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Jan 10 00:14:04 2016 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sun Jan 10 00:14:04 2016 -0800
----------------------------------------------------------------------
.../sysml/parser/python/PyDMLParserWrapper.java | 38 +----
.../python/PydmlSyntacticErrorListener.java | 5 +-
.../parser/python/PydmlSyntacticValidator.java | 138 ++++---------------
.../python/PydmlSyntacticValidatorHelper.java | 31 +----
4 files changed, 34 insertions(+), 178 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/888c9ca4/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java b/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java
index 2a1a8dd..e52bb73 100644
--- a/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java
+++ b/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java
@@ -142,17 +142,14 @@ public class PyDMLParserWrapper extends AParserWrapper
InputStream stream = new ByteArrayInputStream(dmlScript.getBytes());
in = new org.antlr.v4.runtime.ANTLRInputStream(stream);
-// else {
-// if(!(new File(fileName)).exists()) {
-// throw new ParseException("ERROR: Cannot open file:" + fileName);
-// }
-// in = new ANTLRInputStream(new FileInputStream(fileName));
-// }
- } catch (FileNotFoundException e) {
+ }
+ catch (FileNotFoundException e) {
throw new ParseException("ERROR: Cannot find file:" + fileName, e);
- } catch (IOException e) {
+ }
+ catch (IOException e) {
throw new ParseException("ERROR: Cannot open file:" + fileName, e);
- } catch (LanguageException e) {
+ }
+ catch (LanguageException e) {
throw new ParseException("ERROR: " + e.getMessage(), e);
}
@@ -267,29 +264,6 @@ public class PyDMLParserWrapper extends AParserWrapper
// Add the DMLProgram entries into current program
for(Map.Entry<String, DMLProgram> entry : stmtCtx.info.namespaces.entrySet()) {
dmlPgm.getNamespaces().put(entry.getKey(), entry.getValue());
-// // Don't add DMLProgram into the current program, just add function statements
-// // dmlPgm.getNamespaces().put(entry.getKey(), entry.getValue());
-// // Add function statements to current dml program
-// DMLProgram importedPgm = entry.getValue();
-//
-// try {
-// for(FunctionStatementBlock importedFnBlk : importedPgm.getFunctionStatementBlocks()) {
-// if(importedFnBlk.getStatements() != null && importedFnBlk.getStatements().size() == 1) {
-// String functionName = ((FunctionStatement)importedFnBlk.getStatement(0)).getName();
-// dmlPgm.addFunctionStatementBlock(entry.getKey(), functionName, importedFnBlk);
-// }
-// else {
-// LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " incorrect number of functions in the imported function block .... strange");
-// return null;
-// }
-// }
-// if(importedPgm.getStatementBlocks() != null && importedPgm.getStatementBlocks().size() > 0) {
-// LOG.warn("Only the functions can be imported from the namespace " + entry.getKey());
-// }
-// } catch (LanguageException e) {
-// LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot import functions from the file in the import statement");
-// return null;
-// }
}
}
else {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/888c9ca4/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java
index 7a82ecc..9ae63a9 100644
--- a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java
+++ b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java
@@ -29,9 +29,8 @@ import org.apache.commons.logging.LogFactory;
import org.apache.sysml.api.DMLScript;
-public class PydmlSyntacticErrorListener {
-
-
+public class PydmlSyntacticErrorListener
+{
private static final Log LOG = LogFactory.getLog(DMLScript.class.getName());
public static class CustomDmlErrorListener extends BaseErrorListener {
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/888c9ca4/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
index 8efdbe0..fc56fc2 100644
--- a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
+++ b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
@@ -380,26 +380,6 @@ public class PydmlSyntacticValidator implements PydmlListener
@Override
public void exitRelationalExpression(RelationalExpressionContext ctx) {
if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
-// String fileName = helper.getCurrentFileName();
-// int line = ctx.start.getLine();
-// int col = ctx.start.getCharPositionInLine();
-// ArrayList<ParameterExpression> paramExpression = new ArrayList<ParameterExpression>();
-// paramExpression.add(new ParameterExpression(null, ctx.left.info.expr));
-// paramExpression.add(new ParameterExpression(null, ctx.right.info.expr));
-// ParameterExpression operator = new ParameterExpression(null, new StringIdentifier(ctx.op.getText(), fileName, line, col, line, col));
-// paramExpression.add(operator);
-//
-// try {
-// BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression("ppred", paramExpression, fileName, line, col, line, col);
-// if (bife != null){
-// // It is a builtin function
-// ctx.info.expr = bife;
-// return;
-// }
-// }
-// catch(Exception e) {}
-// helper.notifyErrorListeners("Cannot parse relational expression", ctx.getStart());
-
Expression.RelationalOp rop = Expression.getRelationalOp(ctx.op.getText());
ctx.info.expr = new RelationalExpression(rop);
((RelationalExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
@@ -468,25 +448,6 @@ public class PydmlSyntacticValidator implements PydmlListener
setFileLineColumn(ctx.info.expr, ctx);
}
-// @Override
-// public void exitConstBooleanIdExpression(ConstBooleanIdExpressionContext ctx) {
-// boolean val = false;
-// if(ctx.getText().compareTo("TRUE") == 0) {
-// val = true;
-// }
-// else if(ctx.getText().compareTo("FALSE") == 0) {
-// val = false;
-// }
-// else {
-// helper.notifyErrorListeners("cannot parse the boolean value: \'" + ctx.getText() + "\'", ctx.getStart());
-// return;
-// }
-// int linePosition = ctx.start.getLine();
-// int charPosition = ctx.start.getCharPositionInLine();
-// ctx.info.expr = new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
-// setFileLineColumn(ctx.info.expr, ctx);
-// }
-
@Override
public void exitConstDoubleIdExpression(ConstDoubleIdExpressionContext ctx) {
try {
@@ -497,7 +458,7 @@ public class PydmlSyntacticValidator implements PydmlListener
setFileLineColumn(ctx.info.expr, ctx);
}
catch(Exception e) {
- helper.notifyErrorListeners("cannot parse the double value: \'" + ctx.getText() + "\'", ctx.getStart());
+ helper.notifyErrorListeners("cannot parse the float value: \'" + ctx.getText() + "\'", ctx.getStart());
return;
}
}
@@ -512,7 +473,7 @@ public class PydmlSyntacticValidator implements PydmlListener
setFileLineColumn(ctx.info.expr, ctx);
}
catch(Exception e) {
- helper.notifyErrorListeners("cannot parse the integer value: \'" + ctx.getText() + "\'", ctx.getStart());
+ helper.notifyErrorListeners("cannot parse the int value: \'" + ctx.getText() + "\'", ctx.getStart());
return;
}
}
@@ -547,13 +508,6 @@ public class PydmlSyntacticValidator implements PydmlListener
int col = ctx.start.getCharPositionInLine();
ctx.info.expr.setAllPositions(helper.getCurrentFileName(), line, col, line, col);
setFileLineColumn(ctx.info.expr, ctx);
-// if(ctx.getChild(0) instanceof DataIdentifierContext) {
-// ctx.info.expr = ctx.dataIdentifier().dataInfo.expr;
-// }
-// else {
-// String msg = "cannot evaluate data expression ... strange";
-// helper.notifyErrorListeners(msg, ctx.start);
-// }
}
@Override
@@ -661,16 +615,14 @@ public class PydmlSyntacticValidator implements PydmlListener
}
}
else {
- val = text;
// the commandline parameters can be passed without any quotes
-// helper.notifyErrorListeners("something wrong while parsing string ... strange", start);
-// return null;
+ val = text;
}
return new StringIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
}
}
catch(Exception e3) {
- helper.notifyErrorListeners("unable to cast the commandline parameter into int/double/boolean/string", start);
+ helper.notifyErrorListeners("unable to cast the commandline parameter into int/float/bool/str", start);
return null;
}
}
@@ -851,25 +803,21 @@ public class PydmlSyntacticValidator implements PydmlListener
return;
}
if(paramExpression.get(0).getExpr() instanceof DataIdentifier) {
- // && paramExpression.get(0).getName() == null
- // correct usage of identifier
- // if(paramExpression.get(1).getName() == null) {
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- HashMap<String, Expression> varParams = new HashMap<String, Expression>();
- varParams.put(DataExpression.IO_FILENAME, paramExpression.get(1).getExpr());
- for(int i = 2; i < paramExpression.size(); i++) {
- // DataExpression.FORMAT_TYPE, DataExpression.DELIM_DELIMITER, DataExpression.DELIM_HAS_HEADER_ROW, DataExpression.DELIM_SPARSE
- varParams.put(paramExpression.get(i).getName(), paramExpression.get(i).getExpr());
- }
-
- DataExpression dataExpression = new DataExpression(DataOp.WRITE, varParams, fileName, line, col, line, col);
- ctx.info.stmt = new OutputStatement((DataIdentifier) paramExpression.get(0).getExpr(), DataOp.WRITE, fileName, line, col, line, col);
- setFileLineColumn(ctx.info.stmt, ctx);
- ((OutputStatement)ctx.info.stmt).setExprParams(dataExpression);
- return;
- //}
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+ HashMap<String, Expression> varParams = new HashMap<String, Expression>();
+ varParams.put(DataExpression.IO_FILENAME, paramExpression.get(1).getExpr());
+ for(int i = 2; i < paramExpression.size(); i++) {
+ // DataExpression.FORMAT_TYPE, DataExpression.DELIM_DELIMITER, DataExpression.DELIM_HAS_HEADER_ROW, DataExpression.DELIM_SPARSE
+ varParams.put(paramExpression.get(i).getName(), paramExpression.get(i).getExpr());
+ }
+
+ DataExpression dataExpression = new DataExpression(DataOp.WRITE, varParams, fileName, line, col, line, col);
+ ctx.info.stmt = new OutputStatement((DataIdentifier) paramExpression.get(0).getExpr(), DataOp.WRITE, fileName, line, col, line, col);
+ setFileLineColumn(ctx.info.stmt, ctx);
+ ((OutputStatement)ctx.info.stmt).setExprParams(dataExpression);
+ return;
}
helper.notifyErrorListeners("incorrect usage of write function", ctx.start);
@@ -1107,7 +1055,7 @@ public class PydmlSyntacticValidator implements PydmlListener
}
else if(inDefaultNamespace(namespace) && functionName.compareTo("full") == 0) {
if(paramExpression.size() != 3) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (constant double value, number of rows, number of columns)", fnName);
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (constant float value, number of rows, number of columns)", fnName);
return null;
}
paramExpression.get(1).setName("rows");
@@ -1118,7 +1066,7 @@ public class PydmlSyntacticValidator implements PydmlListener
else if(inDefaultNamespace(namespace) && functionName.compareTo("matrix") == 0) {
// This can either be string initializer or as.matrix function
if(paramExpression.size() != 1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 1 argument (either string or double value)", fnName);
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 1 argument (either str or float value)", fnName);
return null;
}
@@ -1359,17 +1307,7 @@ public class PydmlSyntacticValidator implements PydmlListener
dmlOperator = "%*%";
}
}
-// else if(inDefaultNamespace(namespace) && functionName.compareTo("power") == 0) {
-// if(paramExpression.size() == 2) {
-// dmlOperator = "^";
-// }
-// }
-// else if(inDefaultNamespace(namespace) && functionName.compareTo("multiply") == 0) {
-// if(paramExpression.size() == 2) {
-// dmlOperator = "*";
-// }
-// }
-
+
if(dmlOperator != null) {
Expression.BinaryOp bop = Expression.getBinaryOp(dmlOperator);
Expression expr = new BinaryExpression(bop);
@@ -1421,11 +1359,6 @@ public class PydmlSyntacticValidator implements PydmlListener
paramExpression = convertedSyntax.paramExpression;
}
- // Because the user might want to define a corresponding UDF function
-// else if(inDefaultNamespace(namespace) && functionName.compareTo("seq") == 0) {
-// helper.notifyErrorListeners("No function with name \'" + functionName + "\' exists for PyDML. Use \'range\' instead.", ctx.name);
-// return;
-// }
// ===========================================================================================
FunctionCallIdentifier functCall = new FunctionCallIdentifier(paramExpression);
try {
@@ -1503,9 +1436,6 @@ public class PydmlSyntacticValidator implements PydmlListener
@Override
public void exitBuiltinFunctionExpression(BuiltinFunctionExpressionContext ctx) {
-// if(!helper.validateBuiltinFunctions(ctx)) {
-// return; // it is a built-in function and validation failed, so donot proceed ahead.
-// }
// Double verification: verify passed function name is a (non-parameterized) built-in function.
ArrayList<String> names = helper.getQualifiedNames(ctx.name.getText());
if(names == null) {
@@ -1623,9 +1553,6 @@ public class PydmlSyntacticValidator implements PydmlListener
if(!functionName.contains(".") || functionName.startsWith(DMLProgram.DEFAULT_NAMESPACE)) {
// In global namespace, so it can be a builtin function
-// if(!helper.validateBuiltinFunctions(ctx)) {
-// return; // it is a built-in function and validation failed, so donot proceed ahead.
-// }
// Double verification: verify passed function name is a (non-parameterized) built-in function.
try {
BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
@@ -1890,7 +1817,7 @@ public class PydmlSyntacticValidator implements PydmlListener
// Empty value allowed
}
else {
- helper.notifyErrorListeners("the value of user parameter for external function should be of type string", ctx.start);
+ helper.notifyErrorListeners("the value of user parameter for external function should be of type str", ctx.start);
return;
}
otherParams.put(paramName, val);
@@ -1903,22 +1830,7 @@ public class PydmlSyntacticValidator implements PydmlListener
helper.notifyErrorListeners("the parameter \'className\' needs to be passed for defExternal", ctx.start);
return;
}
-
-// if(ctx.body.size() > 0) {
-// // handle function body
-// // Create arraylist of one statement block
-// ArrayList<StatementBlock> body = new ArrayList<StatementBlock>();
-// for(StatementContext stmtCtx : ctx.body) {
-// body.add(getStatementBlock(stmtCtx.info.stmt));
-// }
-// ((ExternalFunctionStatement) functionStmt).setBody(body);
-// ((ExternalFunctionStatement) functionStmt).mergeStatementBlocks();
-// }
-// else {
-// helper.notifyErrorListeners("functions with no statements are not allowed", ctx.start);
-// return;
-// }
-
+
ctx.info.stmt = functionStmt;
setFileLineColumn(ctx.info.stmt, ctx);
ctx.info.functionName = ctx.name.getText();
@@ -2113,6 +2025,4 @@ public class PydmlSyntacticValidator implements PydmlListener
helper.notifyErrorListeners("incorrect valuetype (expected int, str, bool or float)", ctx.start);
}
}
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/888c9ca4/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java
index f7486a3..67aeb4a 100644
--- a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java
+++ b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java
@@ -30,8 +30,8 @@ import org.apache.sysml.parser.python.PydmlParser.ParameterizedExpressionContext
import org.apache.sysml.parser.python.PydmlSyntacticErrorListener.CustomDmlErrorListener;
-public class PydmlSyntacticValidatorHelper {
-
+public class PydmlSyntacticValidatorHelper
+{
private CustomDmlErrorListener _errorListener = null;
public PydmlSyntacticValidatorHelper(CustomDmlErrorListener errorListener) {
this._errorListener = errorListener;
@@ -70,33 +70,6 @@ public class PydmlSyntacticValidatorHelper {
return retVal;
}
-// public static void setInfoForArithmeticOp(org.apache.sysml.parser.Expression current,
-// org.apache.sysml.parser.Expression left,
-// org.apache.sysml.parser.Expression right, String opStr) {
-// try {
-// // PLUS, MINUS, MULT, DIV, MODULUS, INTDIV, MATMULT, POW, INVALID
-// org.apache.sysml.parser.Expression.BinaryOp bop = org.apache.sysml.parser.Expression.getBinaryOp(opStr);
-// current = new org.apache.sysml.parser.BinaryExpression(bop);
-// ((org.apache.sysml.parser.BinaryExpression)current).setLeft(left);
-// ((org.apache.sysml.parser.BinaryExpression)current).setRight(right);
-// ((org.apache.sysml.parser.BinaryExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
-// }
-// catch(Exception e) {
-// System.out.println("In setInfoForArithmeticOp>>");
-// e.printStackTrace();
-// }
-// }
-
-// public static void setInfoForBooleanOp(org.apache.sysml.parser.Expression current,
-// org.apache.sysml.parser.Expression left,
-// org.apache.sysml.parser.Expression right, String opStr) {
-// org.apache.sysml.parser.Expression.BooleanOp bop = org.apache.sysml.parser.Expression.getBooleanOp(opStr);
-// current = new org.apache.sysml.parser.BooleanExpression(bop);
-// ((org.apache.sysml.parser.BooleanExpression)current).setLeft(left);
-// ((org.apache.sysml.parser.BooleanExpression)current).setRight(right);
-// ((org.apache.sysml.parser.BooleanExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
-// }
-
public boolean validateBuiltinFunctions(FunctionCallAssignmentStatementContext ctx) {
String functionName = ctx.name.getText().replaceAll(" ", "").trim();
if(functionName.compareTo("write") == 0 || functionName.compareTo(DMLProgram.DEFAULT_NAMESPACE + ".write") == 0) {
[4/4] incubator-systemml git commit: [SYSTEMML-146] Fix old rowsums
test (removed missing unused input file)
Posted by mb...@apache.org.
[SYSTEMML-146] Fix old rowsums test (removed missing unused input file)
After having introduced the additional check for non-existing read input
files, this test failed due to a missing input 'in/helper'. The test did
not fail before because the dimensions were specified (no need to read
non-existing mtd file) and the read variable was never used. Removing
the unnecessary read from the script template fixed this issue.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/572752a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/572752a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/572752a9
Branch: refs/heads/master
Commit: 572752a9e310b5dedd712bc078a779bed5b8e5aa
Parents: 888c9ca
Author: Matthias Boehm <mb...@us.ibm.com>
Authored: Sun Jan 10 14:29:52 2016 -0800
Committer: Matthias Boehm <mb...@us.ibm.com>
Committed: Sun Jan 10 14:29:52 2016 -0800
----------------------------------------------------------------------
src/test/scripts/functions/aggregate/RowSumTest.dml | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/572752a9/src/test/scripts/functions/aggregate/RowSumTest.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/aggregate/RowSumTest.dml b/src/test/scripts/functions/aggregate/RowSumTest.dml
index 949abf1..cdedda5 100644
--- a/src/test/scripts/functions/aggregate/RowSumTest.dml
+++ b/src/test/scripts/functions/aggregate/RowSumTest.dml
@@ -21,8 +21,6 @@
# junit test class: org.apache.sysml.test.integration.functions.aggregate.RowSumTest.java
-$$readhelper$$
-
Vector = read("$$indir$$vector", rows=$$rows$$, cols=1, format="text");
VectorRowSum = rowSums(Vector);
write(VectorRowSum, "$$outdir$$vector_rowsum", format="text");