You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by du...@apache.org on 2016/01/21 02:06:18 UTC
[1/8] incubator-systemml git commit: [SYSTEMML-148] Refactored
.parser.{antlr4, python} to .parser.{dml, pydml}.
Repository: incubator-systemml
Updated Branches:
refs/heads/master 8ba0fdcca -> c04fc99fc
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/python/StatementInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/StatementInfo.java b/src/main/java/org/apache/sysml/parser/python/StatementInfo.java
deleted file mode 100644
index cf855cd..0000000
--- a/src/main/java/org/apache/sysml/parser/python/StatementInfo.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.parser.python;
-
-/**
- * This class exists solely to prevent compiler warnings.
- *
- * <p>
- * The ExpressionInfo and StatementInfo classes are shared among both parsers
- * (R-like and Python-like dialects), and Antlr-generated code assumes that
- * these classes are present in the parser's namespace.
- */
-class StatementInfo extends org.apache.sysml.parser.antlr4.StatementInfo {
-
-}
[7/8] incubator-systemml git commit: [SYSTEMML-148] Refactored
.parser.{antlr4, python} to .parser.{dml, pydml}.
Posted by du...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidator.java
deleted file mode 100644
index 8af3de9..0000000
--- a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidator.java
+++ /dev/null
@@ -1,1533 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.parser.antlr4;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.antlr.v4.runtime.ParserRuleContext;
-import org.antlr.v4.runtime.Token;
-import org.antlr.v4.runtime.tree.ErrorNode;
-import org.antlr.v4.runtime.tree.TerminalNode;
-import org.apache.sysml.parser.ConditionalPredicate;
-import org.apache.sysml.parser.DMLProgram;
-import org.apache.sysml.parser.DataIdentifier;
-import org.apache.sysml.parser.DoubleIdentifier;
-import org.apache.sysml.parser.Expression;
-import org.apache.sysml.parser.Expression.DataOp;
-import org.apache.sysml.parser.Expression.DataType;
-import org.apache.sysml.parser.Expression.ValueType;
-import org.apache.sysml.parser.antlr4.DmlParser.AddSubExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.AssignmentStatementContext;
-import org.apache.sysml.parser.antlr4.DmlParser.AtomicExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.BooleanAndExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.BooleanNotExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.BooleanOrExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.BuiltinFunctionExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.CommandlineParamExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.CommandlinePositionExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.ConstDoubleIdExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.ConstFalseExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.ConstIntIdExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.ConstStringIdExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.ConstTrueExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.DataIdExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.DataIdentifierContext;
-import org.apache.sysml.parser.antlr4.DmlParser.DmlprogramContext;
-import org.apache.sysml.parser.antlr4.DmlParser.ExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.ExternalFunctionDefExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.ForStatementContext;
-import org.apache.sysml.parser.antlr4.DmlParser.FunctionCallAssignmentStatementContext;
-import org.apache.sysml.parser.antlr4.DmlParser.FunctionCallMultiAssignmentStatementContext;
-import org.apache.sysml.parser.antlr4.DmlParser.FunctionStatementContext;
-import org.apache.sysml.parser.antlr4.DmlParser.IfStatementContext;
-import org.apache.sysml.parser.antlr4.DmlParser.IfdefAssignmentStatementContext;
-import org.apache.sysml.parser.antlr4.DmlParser.ImportStatementContext;
-import org.apache.sysml.parser.antlr4.DmlParser.IndexedExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.InternalFunctionDefExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.IterablePredicateColonExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.IterablePredicateSeqExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.MatrixDataTypeCheckContext;
-import org.apache.sysml.parser.antlr4.DmlParser.MatrixMulExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.Ml_typeContext;
-import org.apache.sysml.parser.antlr4.DmlParser.ModIntDivExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.MultDivExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.ParForStatementContext;
-import org.apache.sysml.parser.antlr4.DmlParser.ParameterizedExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.PathStatementContext;
-import org.apache.sysml.parser.antlr4.DmlParser.PowerExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.RelationalExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.SimpleDataIdentifierExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.StatementContext;
-import org.apache.sysml.parser.antlr4.DmlParser.StrictParameterizedExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.StrictParameterizedKeyValueStringContext;
-import org.apache.sysml.parser.antlr4.DmlParser.TypedArgNoAssignContext;
-import org.apache.sysml.parser.antlr4.DmlParser.UnaryExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlParser.ValueTypeContext;
-import org.apache.sysml.parser.antlr4.DmlParser.WhileStatementContext;
-import org.apache.sysml.parser.AParserWrapper;
-import org.apache.sysml.parser.AssignmentStatement;
-import org.apache.sysml.parser.BinaryExpression;
-import org.apache.sysml.parser.BooleanExpression;
-import org.apache.sysml.parser.BooleanIdentifier;
-import org.apache.sysml.parser.BuiltinFunctionExpression;
-import org.apache.sysml.parser.ConstIdentifier;
-import org.apache.sysml.parser.DataExpression;
-import org.apache.sysml.parser.ExternalFunctionStatement;
-import org.apache.sysml.parser.ForStatement;
-import org.apache.sysml.parser.FunctionCallIdentifier;
-import org.apache.sysml.parser.FunctionStatement;
-import org.apache.sysml.parser.IfStatement;
-import org.apache.sysml.parser.ImportStatement;
-import org.apache.sysml.parser.IndexedIdentifier;
-import org.apache.sysml.parser.IntIdentifier;
-import org.apache.sysml.parser.IterablePredicate;
-import org.apache.sysml.parser.LanguageException;
-import org.apache.sysml.parser.MultiAssignmentStatement;
-import org.apache.sysml.parser.OutputStatement;
-import org.apache.sysml.parser.ParForStatement;
-import org.apache.sysml.parser.ParameterExpression;
-import org.apache.sysml.parser.ParameterizedBuiltinFunctionExpression;
-import org.apache.sysml.parser.ParseException;
-import org.apache.sysml.parser.PathStatement;
-import org.apache.sysml.parser.PrintStatement;
-import org.apache.sysml.parser.RelationalExpression;
-import org.apache.sysml.parser.Statement;
-import org.apache.sysml.parser.StatementBlock;
-import org.apache.sysml.parser.StringIdentifier;
-import org.apache.sysml.parser.WhileStatement;
-
-/**
- * TODO: Refactor duplicated parser code dml/pydml (entire package).
- *
- */
-public class DmlSyntacticValidator implements DmlListener
-{
- private DmlSyntacticValidatorHelper helper = null;
-
- private String _workingDir = "."; //current working directory
- private String _currentPath = null; //current file path
- private HashMap<String,String> argVals = null;
-
- public DmlSyntacticValidator(DmlSyntacticValidatorHelper helper, String currentPath, HashMap<String,String> argVals) {
- this.helper = helper;
- this.argVals = argVals;
-
- _currentPath = currentPath;
- }
-
- // Functions we have to implement but don't really need it
- @Override
- public void enterAddSubExpression(AddSubExpressionContext ctx) { }
- @Override
- public void enterAssignmentStatement(AssignmentStatementContext ctx) {}
- @Override
- public void enterAtomicExpression(AtomicExpressionContext ctx) { }
- @Override
- public void enterBooleanAndExpression(BooleanAndExpressionContext ctx) { }
- @Override
- public void enterBooleanNotExpression(BooleanNotExpressionContext ctx) { }
- @Override
- public void enterBooleanOrExpression(BooleanOrExpressionContext ctx) { }
- @Override
- public void enterCommandlineParamExpression(CommandlineParamExpressionContext ctx) { }
- @Override
- public void enterCommandlinePositionExpression(CommandlinePositionExpressionContext ctx) { }
- @Override
- public void enterConstDoubleIdExpression(ConstDoubleIdExpressionContext ctx) { }
- @Override
- public void enterConstIntIdExpression(ConstIntIdExpressionContext ctx) { }
- @Override
- public void enterConstStringIdExpression(ConstStringIdExpressionContext ctx) { }
- @Override
- public void enterDataIdExpression(DataIdExpressionContext ctx) { }
-
- @Override
- public void enterDmlprogram(DmlprogramContext ctx) { }
- @Override
- public void enterEveryRule(ParserRuleContext arg0) {
- if(arg0 instanceof StatementContext) {
- if(((StatementContext) arg0).info == null) {
- ((StatementContext) arg0).info = new StatementInfo();
- }
- }
- if(arg0 instanceof FunctionStatementContext) {
- if(((FunctionStatementContext) arg0).info == null) {
- ((FunctionStatementContext) arg0).info = new StatementInfo();
- }
- }
- if(arg0 instanceof ExpressionContext) {
- if(((ExpressionContext) arg0).info == null) {
- ((ExpressionContext) arg0).info = new ExpressionInfo();
- }
- }
- if(arg0 instanceof DataIdentifierContext) {
- if(((DataIdentifierContext) arg0).dataInfo == null) {
- ((DataIdentifierContext) arg0).dataInfo = new ExpressionInfo();
- }
- }
- }
- @Override
- public void enterExternalFunctionDefExpression(ExternalFunctionDefExpressionContext ctx) { }
- @Override
- public void enterForStatement(ForStatementContext ctx) {}
- @Override
- public void enterFunctionCallAssignmentStatement(FunctionCallAssignmentStatementContext ctx) { }
- @Override
- public void enterFunctionCallMultiAssignmentStatement(FunctionCallMultiAssignmentStatementContext ctx) { }
- @Override
- public void enterIfStatement(IfStatementContext ctx) { }
- @Override
- public void enterImportStatement(ImportStatementContext ctx) { }
- @Override
- public void enterIndexedExpression(IndexedExpressionContext ctx) { }
- @Override
- public void enterInternalFunctionDefExpression(InternalFunctionDefExpressionContext ctx) { }
- public void enterMatrixMulExpression(MatrixMulExpressionContext ctx) { }
- @Override
- public void enterMl_type(Ml_typeContext ctx) { }
- @Override
- public void enterModIntDivExpression(ModIntDivExpressionContext ctx) { }
- @Override
- public void enterMultDivExpression(MultDivExpressionContext ctx) { }
- @Override
- public void enterParameterizedExpression(ParameterizedExpressionContext ctx) { }
- @Override
- public void enterParForStatement(ParForStatementContext ctx) { }
- @Override
- public void enterPathStatement(PathStatementContext ctx) { }
- @Override
- public void enterPowerExpression(PowerExpressionContext ctx) { }
- @Override
- public void enterRelationalExpression(RelationalExpressionContext ctx) { }
- @Override
- public void enterSimpleDataIdentifierExpression(SimpleDataIdentifierExpressionContext ctx) { }
- @Override
- public void enterStrictParameterizedExpression(StrictParameterizedExpressionContext ctx) { }
- @Override
- public void enterTypedArgNoAssign(TypedArgNoAssignContext ctx) { }
- @Override
- public void enterUnaryExpression(UnaryExpressionContext ctx) { }
- @Override
- public void enterValueType(ValueTypeContext ctx) { }
- @Override
- public void enterWhileStatement(WhileStatementContext ctx) { }
-
- @Override
- public void visitErrorNode(ErrorNode arg0) { }
- @Override
- public void visitTerminal(TerminalNode arg0) { }
- @Override
- public void exitEveryRule(ParserRuleContext arg0) {}
- // --------------------------------------------------------------------
- private void setFileLineColumn(Expression expr, ParserRuleContext ctx) {
- // expr.setFilename(helper.getCurrentFileName());
- String txt = ctx.getText();
- expr.setFilename(_currentPath);
- expr.setBeginLine(ctx.start.getLine());
- expr.setBeginColumn(ctx.start.getCharPositionInLine());
- expr.setEndLine(ctx.stop.getLine());
- expr.setEndColumn(ctx.stop.getCharPositionInLine());
- if(expr.getBeginColumn() == expr.getEndColumn() && expr.getBeginLine() == expr.getEndLine() && txt.length() > 1) {
- expr.setEndColumn(expr.getBeginColumn() + txt.length() - 1);
- }
- }
-
- private void setFileLineColumn(Statement stmt, ParserRuleContext ctx) {
- String txt = ctx.getText();
- stmt.setFilename(helper.getCurrentFileName());
- stmt.setBeginLine(ctx.start.getLine());
- stmt.setBeginColumn(ctx.start.getCharPositionInLine());
- stmt.setEndLine(ctx.stop.getLine());
- stmt.setEndColumn(ctx.stop.getCharPositionInLine());
- if(stmt.getBeginColumn() == stmt.getEndColumn() && stmt.getBeginLine() == stmt.getEndLine() && txt.length() > 1) {
- stmt.setEndColumn(stmt.getBeginColumn() + txt.length() - 1);
- }
- }
-
- // For now do no type checking, let validation handle it.
- // This way parser doesn't have to open metadata file
- @Override
- public void exitAddSubExpression(AddSubExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
- Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
- @Override
- public void exitModIntDivExpression(ModIntDivExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
- Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- @Override
- public void exitUnaryExpression(UnaryExpressionContext ctx) {
- if(ctx.left.info.expr != null) {
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- if(ctx.left.info.expr instanceof IntIdentifier) {
- if(ctx.op.getText().compareTo("-") == 0) {
- ((IntIdentifier) ctx.left.info.expr).multiplyByMinusOne();
- }
- ctx.info.expr = ctx.left.info.expr;
- }
- else if(ctx.left.info.expr instanceof DoubleIdentifier) {
- if(ctx.op.getText().compareTo("-") == 0) {
- ((DoubleIdentifier) ctx.left.info.expr).multiplyByMinusOne();
- }
- ctx.info.expr = ctx.left.info.expr;
- }
- else {
- Expression right = new IntIdentifier(1, fileName, line, col, line, col);
- if(ctx.op.getText().compareTo("-") == 0) {
- right = new IntIdentifier(-1, fileName, line, col, line, col);
- }
-
- Expression.BinaryOp bop = Expression.getBinaryOp("*");
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(right);
- }
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- @Override
- public void exitMultDivExpression(MultDivExpressionContext ctx) {
- Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- @Override
- public void exitPowerExpression(PowerExpressionContext ctx) {
- Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
- @Override
- public void exitMatrixMulExpression(MatrixMulExpressionContext ctx) {
- Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitRelationalExpression(RelationalExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
-// String fileName = helper.getCurrentFileName();
-// int line = ctx.start.getLine();
-// int col = ctx.start.getCharPositionInLine();
-// ArrayList<ParameterExpression> paramExpression = new ArrayList<ParameterExpression>();
-// paramExpression.add(new ParameterExpression(null, ctx.left.info.expr));
-// paramExpression.add(new ParameterExpression(null, ctx.right.info.expr));
-// ParameterExpression operator = new ParameterExpression(null, new StringIdentifier(ctx.op.getText(), fileName, line, col, line, col));
-// paramExpression.add(operator);
-//
-// try {
-// BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression("ppred", paramExpression, fileName, line, col, line, col);
-// if (bife != null){
-// // It is a builtin function
-// ctx.info.expr = bife;
-// return;
-// }
-// }
-// catch(Exception e) {}
-// helper.notifyErrorListeners("Cannot parse relational expression", ctx.getStart());
-
- Expression.RelationalOp rop = Expression.getRelationalOp(ctx.op.getText());
- ctx.info.expr = new RelationalExpression(rop);
- ((RelationalExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((RelationalExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitBooleanAndExpression(BooleanAndExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
- Expression.BooleanOp bop = Expression.getBooleanOp(ctx.op.getText());
- ctx.info.expr = new BooleanExpression(bop);
- ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BooleanExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- @Override
- public void exitBooleanOrExpression(BooleanOrExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
- Expression.BooleanOp bop = Expression.getBooleanOp(ctx.op.getText());
- ctx.info.expr = new BooleanExpression(bop);
- ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BooleanExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- @Override
- public void exitBooleanNotExpression(BooleanNotExpressionContext ctx) {
- if(ctx.left.info.expr != null) {
- Expression.BooleanOp bop = Expression.getBooleanOp(ctx.op.getText());
- ctx.info.expr = new BooleanExpression(bop);
- ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitAtomicExpression(AtomicExpressionContext ctx) {
- ctx.info.expr = ctx.left.info.expr;
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
-// @Override
-// public void exitConstBooleanIdExpression(ConstBooleanIdExpressionContext ctx) {
-// boolean val = false;
-// if(ctx.getText().compareTo("TRUE") == 0) {
-// val = true;
-// }
-// else if(ctx.getText().compareTo("FALSE") == 0) {
-// val = false;
-// }
-// else {
-// helper.notifyErrorListeners("cannot parse the boolean value: \'" + ctx.getText() + "\'", ctx.getStart());
-// return;
-// }
-// int linePosition = ctx.start.getLine();
-// int charPosition = ctx.start.getCharPositionInLine();
-// ctx.info.expr = new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
-// setFileLineColumn(ctx.info.expr, ctx);
-// }
-
- @Override
- public void exitConstDoubleIdExpression(ConstDoubleIdExpressionContext ctx) {
- try {
- double val = Double.parseDouble(ctx.getText());
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new DoubleIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- catch(Exception e) {
- helper.notifyErrorListeners("cannot parse the double value: \'" + ctx.getText() + "\'", ctx.getStart());
- return;
- }
- }
-
- @Override
- public void exitConstIntIdExpression(ConstIntIdExpressionContext ctx) {
- try {
- long val = Long.parseLong(ctx.getText());
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new IntIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- catch(Exception e) {
- helper.notifyErrorListeners("cannot parse the integer value: \'" + ctx.getText() + "\'", ctx.getStart());
- return;
- }
- }
-
- @Override
- public void exitConstStringIdExpression(ConstStringIdExpressionContext ctx) {
- String val = "";
- String text = ctx.getText();
- if( (text.startsWith("\"") && text.endsWith("\"")) ||
- (text.startsWith("\'") && text.endsWith("\'"))) {
- if(text.length() > 2) {
- val = text.substring(1, text.length()-1);
- }
- }
- else {
- helper.notifyErrorListeners("something wrong while parsing string ... strange", ctx.start);
- return;
- }
-
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new StringIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitDataIdExpression(DataIdExpressionContext ctx) {
- ctx.info.expr = ctx.dataIdentifier().dataInfo.expr;
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- ctx.info.expr.setAllPositions(helper.getCurrentFileName(), line, col, line, col);
- setFileLineColumn(ctx.info.expr, ctx);
-// if(ctx.getChild(0) instanceof DataIdentifierContext) {
-// ctx.info.expr = ctx.dataIdentifier().dataInfo.expr;
-// }
-// else {
-// String msg = "cannot evaluate data expression ... strange";
-// helper.notifyErrorListeners(msg, ctx.start);
-// }
- }
-
- @Override
- public void exitSimpleDataIdentifierExpression(SimpleDataIdentifierExpressionContext ctx) {
- // This is either a function, or variable with namespace
- // By default, it assigns to a data type
- ctx.dataInfo.expr = new DataIdentifier(ctx.getText());
- setFileLineColumn(ctx.dataInfo.expr, ctx);
- }
-
- @Override
- public void exitIndexedExpression(IndexedExpressionContext ctx) {
- ctx.dataInfo.expr = new IndexedIdentifier(ctx.name.getText(), false, false);
- setFileLineColumn(ctx.dataInfo.expr, ctx);
- try {
- ArrayList< ArrayList<Expression> > exprList = new ArrayList< ArrayList<Expression> >();
-
- ArrayList<Expression> rowIndices = new ArrayList<Expression>();
- ArrayList<Expression> colIndices = new ArrayList<Expression>();
-
- boolean isRowLower = (ctx.rowLower != null && !ctx.rowLower.isEmpty() && (ctx.rowLower.info.expr != null));
- boolean isRowUpper = (ctx.rowUpper != null && !ctx.rowUpper.isEmpty() && (ctx.rowUpper.info.expr != null));
- boolean isColLower = (ctx.colLower != null && !ctx.colLower.isEmpty() && (ctx.colLower.info.expr != null));
- boolean isColUpper = (ctx.colUpper != null && !ctx.colUpper.isEmpty() && (ctx.colUpper.info.expr != null));
-
- if(!isRowLower && !isRowUpper) {
- // both not set
- rowIndices.add(null); rowIndices.add(null);
- }
- else if(isRowLower && isRowUpper) {
- // both set
- rowIndices.add(ctx.rowLower.info.expr);
- rowIndices.add(ctx.rowUpper.info.expr);
- }
- else if(isRowLower && !isRowUpper) {
- // only row set
- rowIndices.add(ctx.rowLower.info.expr);
- }
- else {
- helper.notifyErrorListeners("incorrect index expression for row", ctx.start);
- return;
- }
-
- if(!isColLower && !isColUpper) {
- // both not set
- colIndices.add(null); colIndices.add(null);
- }
- else if(isColLower && isColUpper) {
- colIndices.add(ctx.colLower.info.expr);
- colIndices.add(ctx.colUpper.info.expr);
- }
- else if(isColLower && !isColUpper) {
- colIndices.add(ctx.colLower.info.expr);
- }
- else {
- helper.notifyErrorListeners("incorrect index expression for column", ctx.start);
- return;
- }
-
-
-// boolean rowIndexLowerSet = false;
-// boolean colIndexLowerSet = false;
-//
-// if(ctx.rowLower != null && !ctx.rowLower.isEmpty() && (ctx.rowLower.info.expr != null)) {
-// rowIndices.add(ctx.rowLower.info.expr);
-// rowIndexLowerSet = true;
-// }
-// else {
-// rowIndices.add(null);
-// }
-// if(ctx.rowUpper != null && !ctx.rowUpper.isEmpty() && (ctx.rowUpper.info.expr != null)) {
-// rowIndices.add(ctx.rowUpper.info.expr);
-// if(!rowIndexLowerSet) {
-// helper.notifyErrorListeners("incorrect index expression for row", ctx.start);
-// return;
-// }
-// }
-// if(ctx.colLower != null && !ctx.colLower.isEmpty() && (ctx.colLower.info.expr != null)) {
-// colIndices.add(ctx.colLower.info.expr);
-// colIndexLowerSet = true;
-// }
-// else {
-// colIndices.add(null);
-// }
-// if(ctx.colUpper != null && !ctx.colUpper.isEmpty() && (ctx.colUpper.info.expr != null)) {
-// colIndices.add(ctx.colUpper.info.expr);
-// if(!colIndexLowerSet) {
-// helper.notifyErrorListeners("incorrect index expression for column", ctx.start);
-// return;
-// }
-// }
- exprList.add(rowIndices);
- exprList.add(colIndices);
- ((IndexedIdentifier) ctx.dataInfo.expr).setIndices(exprList);
- }
- catch(Exception e) {
- helper.notifyErrorListeners("cannot set the indices", ctx.start);
- return;
- }
- }
-
- private ConstIdentifier getConstIdFromString(String varValue, Token start) {
- // Both varName and varValue are correct
- int linePosition = start.getLine();
- int charPosition = start.getCharPositionInLine();
- try {
- long val = Long.parseLong(varValue);
- return new IntIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- }
- catch(Exception e) {
- try {
- double val = Double.parseDouble(varValue);
- return new DoubleIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- }
- catch(Exception e1) {
- try {
- if(varValue.compareTo("TRUE") == 0 || varValue.compareTo("FALSE") == 0) {
- boolean val = false;
- if(varValue.compareTo("TRUE") == 0) {
- val = true;
- }
- return new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- }
- else {
- String val = "";
- String text = varValue;
- if( (text.startsWith("\"") && text.endsWith("\"")) ||
- (text.startsWith("\'") && text.endsWith("\'"))) {
- if(text.length() > 2) {
- val = text.substring(1, text.length()-1);
- }
- }
- else {
- val = text;
- // the commandline parameters can be passed without any quotes
-// helper.notifyErrorListeners("something wrong while parsing string ... strange", start);
-// return null;
- }
- return new StringIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- }
- }
- catch(Exception e3) {
- helper.notifyErrorListeners("unable to cast the commandline parameter into int/double/boolean/string", start);
- return null;
- }
- }
- }
-
- }
-
- private void fillExpressionInfoCommandLineParameters(String varName, ExpressionInfo dataInfo, Token start) {
-
- if(!varName.startsWith("$")) {
- helper.notifyErrorListeners("commandline param doesnot start with $ ... strange", start);
- return;
- }
-
- String varValue = null;
- for(Map.Entry<String, String> arg : this.argVals.entrySet()) {
- if(arg.getKey().trim().compareTo(varName) == 0) {
- if(varValue != null) {
- helper.notifyErrorListeners("multiple values passed for the parameter " + varName + " via commandline", start);
- return;
- }
- else {
- varValue = arg.getValue().trim();
- }
- }
- }
-
- if(varValue == null) {
- // helper.notifyErrorListeners("the parameter " + varName + " either needs to be passed through commandline or initialized to default value", start);
- return;
- }
-
- // Command line param cannot be empty string
- // If you want to pass space, please quote it
- if(varValue.trim().compareTo("") == 0)
- return;
-
- dataInfo.expr = getConstIdFromString(varValue, start);
- }
-
- @Override
- public void exitCommandlineParamExpression(CommandlineParamExpressionContext ctx) {
- handleCommandlineArgumentExpression(ctx);
- }
-
- @Override
- public void exitCommandlinePositionExpression(CommandlinePositionExpressionContext ctx) {
- handleCommandlineArgumentExpression(ctx);
- }
-
- /**
- *
- * @param ctx
- */
- private void handleCommandlineArgumentExpression(DataIdentifierContext ctx)
- {
- String varName = ctx.getText().trim();
- fillExpressionInfoCommandLineParameters(varName, ctx.dataInfo, ctx.start);
-
- if(ctx.dataInfo.expr == null) {
- if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) {
- String msg = "The parameter " + varName + " either needs to be passed "
- + "through commandline or initialized to default value.";
- if( AParserWrapper.IGNORE_UNSPECIFIED_ARGS ) {
- ctx.dataInfo.expr = getConstIdFromString(" ", ctx.start);
- helper.raiseWarning(msg, ctx.start);
- }
- else {
- helper.notifyErrorListeners(msg, ctx.start);
- }
- }
- }
- }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitImportStatement(ImportStatementContext ctx)
- {
- //prepare import filepath
- String filePath = ctx.filePath.getText();
- String namespace = DMLProgram.DEFAULT_NAMESPACE;
- if(ctx.namespace != null && ctx.namespace.getText() != null && !ctx.namespace.getText().isEmpty()) {
- namespace = ctx.namespace.getText();
- }
- if((filePath.startsWith("\"") && filePath.endsWith("\"")) ||
- filePath.startsWith("'") && filePath.endsWith("'")) {
- filePath = filePath.substring(1, filePath.length()-1);
- }
-
- //concatenate working directory to filepath
- filePath = _workingDir + File.separator + filePath;
-
- DMLProgram prog = null;
- try {
- prog = (new DMLParserWrapper()).doParse(filePath, null, argVals);
- } catch (ParseException e) {
- helper.notifyErrorListeners("Exception found during importing a program from file " + filePath, ctx.start);
- return;
- }
- // Custom logic whether to proceed ahead or not. Better than the current exception handling mechanism
- if(prog == null) {
- helper.notifyErrorListeners("One or more errors found during importing a program from file " + filePath, ctx.start);
- return;
- }
- else {
- ctx.info.namespaces = new HashMap<String, DMLProgram>();
- ctx.info.namespaces.put(namespace, prog);
- ctx.info.stmt = new ImportStatement();
- ((ImportStatement) ctx.info.stmt).setCompletePath(filePath);
- ((ImportStatement) ctx.info.stmt).setFilePath(ctx.filePath.getText());
- ((ImportStatement) ctx.info.stmt).setNamespace(namespace);
- }
- }
-
- @Override
- public void exitAssignmentStatement(AssignmentStatementContext ctx) {
- if(ctx.targetList == null || ctx.targetList.size() != 1) {
- helper.notifyErrorListeners("incorrect parsing for assignment", ctx.start);
- return;
- }
- String targetListText = ctx.targetList.get(0).getText();
- if(targetListText.startsWith("$")) {
- helper.notifyErrorListeners("assignment of commandline parameters is not allowed. (Quickfix: try using someLocalVariable=ifdef(" + targetListText + ", default value))", ctx.start);
- return;
- }
-
- DataIdentifier target = null;
- if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
- target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
- Expression source = ctx.source.info.expr;
-
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- try {
- ctx.info.stmt = new AssignmentStatement(target, source, line, col, line, col);
- setFileLineColumn(ctx.info.stmt, ctx);
- } catch (LanguageException e) {
- // TODO: extract more meaningful info from this exception.
- helper.notifyErrorListeners("invalid assignment", ctx.targetList.get(0).start);
- return;
- }
- }
- else {
- helper.notifyErrorListeners("incorrect lvalue ... strange", ctx.targetList.get(0).start);
- return;
- }
-
- }
-
-
- private void setAssignmentStatement(DataIdentifier target, Expression expression, StatementContext ctx) {
- try {
- ctx.info.stmt = new AssignmentStatement(target, expression, ctx.start.getLine(), ctx.start.getCharPositionInLine(), ctx.start.getLine(), ctx.start.getCharPositionInLine());
- setFileLineColumn(ctx.info.stmt, ctx);
- } catch (LanguageException e) {
- // TODO: extract more meaningful info from this exception.
- helper.notifyErrorListeners("invalid function call", ctx.start);
- return;
- }
- }
-
- private void setPrintStatement(FunctionCallAssignmentStatementContext ctx, String functionName) {
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners(functionName + "() has only one parameter", ctx.start);
- return;
- }
- Expression expr = paramExpression.get(0).getExpr();
- if(expr == null) {
- helper.notifyErrorListeners("cannot process " + functionName + "() function", ctx.start);
- return;
- }
- try {
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- ctx.info.stmt = new PrintStatement(functionName, expr, line, col, line, col);
- setFileLineColumn(ctx.info.stmt, ctx);
- } catch (LanguageException e) {
- helper.notifyErrorListeners("cannot process " + functionName + "() function", ctx.start);
- return;
- }
- }
-
- private void setOutputStatement(FunctionCallAssignmentStatementContext ctx) {
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
- if(paramExpression.size() < 2){
- helper.notifyErrorListeners("incorrect usage of write function (atleast 2 arguments required)", ctx.start);
- return;
- }
- if(paramExpression.get(0).getExpr() instanceof DataIdentifier) {
- // && paramExpression.get(0).getName() == null
- // correct usage of identifier
- // if(paramExpression.get(1).getName() == null) {
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- HashMap<String, Expression> varParams = new HashMap<String, Expression>();
- varParams.put(DataExpression.IO_FILENAME, paramExpression.get(1).getExpr());
- for(int i = 2; i < paramExpression.size(); i++) {
- // DataExpression.FORMAT_TYPE, DataExpression.DELIM_DELIMITER, DataExpression.DELIM_HAS_HEADER_ROW, DataExpression.DELIM_SPARSE
- varParams.put(paramExpression.get(i).getName(), paramExpression.get(i).getExpr());
- }
-
- DataExpression dataExpression = new DataExpression(DataOp.WRITE, varParams, fileName, line, col, line, col);
- ctx.info.stmt = new OutputStatement((DataIdentifier) paramExpression.get(0).getExpr(), DataOp.WRITE, fileName, line, col, line, col);
- setFileLineColumn(ctx.info.stmt, ctx);
- ((OutputStatement)ctx.info.stmt).setExprParams(dataExpression);
- return;
- //}
- }
-
- helper.notifyErrorListeners("incorrect usage of write function", ctx.start);
- return;
-
- }
-
- @Override
- public void exitFunctionCallAssignmentStatement(FunctionCallAssignmentStatementContext ctx) {
- String fullyQualifiedFunctionName = ctx.name.getText();
- String [] fnNames = fullyQualifiedFunctionName.split("::");
- String functionName = "";
- String namespace = "";
- if(fnNames.length == 1) {
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- functionName = fnNames[0].trim();
- }
- else if(fnNames.length == 2) {
- namespace = fnNames[0].trim();
- functionName = fnNames[1].trim();
- }
- else {
- helper.notifyErrorListeners("incorrect function name", ctx.name);
- return;
- }
-
- if((functionName.compareTo("print") == 0 || functionName.compareTo("stop") == 0 ) && namespace.compareTo(DMLProgram.DEFAULT_NAMESPACE) == 0) {
- setPrintStatement(ctx, functionName);
- return;
- }
- else if(functionName.compareTo("write") == 0
- && namespace.compareTo(DMLProgram.DEFAULT_NAMESPACE) == 0) {
- setOutputStatement(ctx);
- return;
- }
-
- boolean ignoreLValue = false;
- if(ctx.targetList == null || ctx.targetList.size() == 0 || ctx.targetList.get(0).isEmpty()) {
- helper.notifyErrorListeners("function call needs to have lvalue (Quickfix: change it to \'tmpVar = " + functionName + "(...)\')", ctx.name);
- return;
- }
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
-// if(functionName.compareTo("read") == 0 && paramExpression.size() > 0 && paramExpression.get(0).getName() == null) {
-// paramExpression.get(0).setName(DataExpression.IO_FILENAME);
-// }
-
- FunctionCallIdentifier functCall = new FunctionCallIdentifier(paramExpression);
- try {
- functCall.setFunctionName(functionName);
- functCall.setFunctionNamespace(namespace);
- } catch (ParseException e1) {
- helper.notifyErrorListeners("unable to process function " + functionName, ctx.start);
- return;
- }
-
- DataIdentifier target = null;
- if(!ignoreLValue) {
- if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
- target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
- }
- else {
- helper.notifyErrorListeners("incorrect lvalue ... strange", ctx.targetList.get(0).start);
- //target = new DataIdentifier(); // so as not to avoid null pointer
- return;
- }
- }
-
- if(!functionName.contains("::") || functionName.startsWith(DMLProgram.DEFAULT_NAMESPACE)) {
- // In global namespace, so it can be a builtin function
- if(!helper.validateBuiltinFunctions(ctx)) {
- return; // it is a built-in function and validation failed, so donot proceed ahead.
- }
- // Double verification: verify passed function name is a (non-parameterized) built-in function.
- try {
- BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (bife != null){
- // It is a builtin function
- setAssignmentStatement(target, bife, ctx);
- return;
- }
-
- ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (pbife != null){
- // It is a parameterized builtin function
- setAssignmentStatement(target, pbife, ctx);
- return;
- }
-
- // built-in read, rand ...
- DataExpression dbife = DataExpression.getDataExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (dbife != null){
- setAssignmentStatement(target, dbife, ctx);
- return;
- }
- } catch(Exception e) {
- helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
- return ;
- }
- }
-
- setAssignmentStatement(target, functCall, ctx);
- }
-
- @Override
- public void exitBuiltinFunctionExpression(BuiltinFunctionExpressionContext ctx) {
-// if(!helper.validateBuiltinFunctions(ctx)) {
-// return; // it is a built-in function and validation failed, so donot proceed ahead.
-// }
- // Double verification: verify passed function name is a (non-parameterized) built-in function.
- String functionName = ctx.name.getText();
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
-// if(functionName.compareTo("read") == 0 && paramExpression.size() > 0 && paramExpression.get(0).getName() == null) {
-// paramExpression.get(0).setName(DataExpression.IO_FILENAME);
-// }
-
- try {
- BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, paramExpression, fileName, line, col, line, col);
- if (bife != null){
- // It is a builtin function
- ctx.info.expr = bife;
- setFileLineColumn(ctx.info.expr, ctx);
- return;
- }
-
- ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, paramExpression, fileName, line, col, line, col);
- if (pbife != null){
- // It is a parameterized builtin function
- ctx.info.expr = pbife;
- setFileLineColumn(ctx.info.expr, ctx);
- return;
- }
-
- // built-in read, rand ...
- DataExpression dbife = DataExpression.getDataExpression(functionName, paramExpression, fileName, line, col, line, col);
- if (dbife != null){
- ctx.info.expr = dbife;
- setFileLineColumn(ctx.info.expr, ctx);
- return;
- }
-
- } catch(Exception e) {
- helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
- return ;
- }
- helper.notifyErrorListeners("only builtin functions allowed as part of expression", ctx.start);
- }
-
- private void setMultiAssignmentStatement(ArrayList<DataIdentifier> target, Expression expression, StatementContext ctx) {
- ctx.info.stmt = new MultiAssignmentStatement(target, expression);
- ctx.info.stmt.setAllPositions(helper.getCurrentFileName(), ctx.start.getLine(), ctx.start.getCharPositionInLine(), ctx.start.getLine(), ctx.start.getCharPositionInLine());
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
- @Override
- public void exitFunctionCallMultiAssignmentStatement(
- FunctionCallMultiAssignmentStatementContext ctx) {
- String fullyQualifiedFunctionName = ctx.name.getText();
- String [] fnNames = fullyQualifiedFunctionName.split("::");
- String functionName = "";
- String namespace = "";
- if(fnNames.length == 1) {
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- functionName = fnNames[0].trim();
- }
- else if(fnNames.length == 2) {
- namespace = fnNames[0].trim();
- functionName = fnNames[1].trim();
- }
- else {
- helper.notifyErrorListeners("incorrect function name", ctx.name);
- return;
- }
-
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
-// if(functionName.compareTo("read") == 0 && paramExpression.size() > 0 && paramExpression.get(0).getName() == null) {
-// paramExpression.get(0).setName(DataExpression.IO_FILENAME);
-// }
-
- FunctionCallIdentifier functCall = new FunctionCallIdentifier(paramExpression);
- try {
- functCall.setFunctionName(functionName);
- functCall.setFunctionNamespace(namespace);
- } catch (ParseException e1) {
- helper.notifyErrorListeners("unable to process function " + functionName, ctx.start);
- return;
- }
-
- ArrayList<DataIdentifier> targetList = new ArrayList<DataIdentifier>();
- for(DataIdentifierContext dataCtx : ctx.targetList) {
- if(dataCtx.dataInfo.expr instanceof DataIdentifier) {
- targetList.add((DataIdentifier) dataCtx.dataInfo.expr);
- }
- else {
- helper.notifyErrorListeners("incorrect lvalue ... strange", dataCtx.start);
- //target = new DataIdentifier(); // so as not to avoid null pointer
- return;
- }
- }
-
- if(!functionName.contains("::") || functionName.startsWith(DMLProgram.DEFAULT_NAMESPACE)) {
- // In global namespace, so it can be a builtin function
-// if(!helper.validateBuiltinFunctions(ctx)) {
-// return; // it is a built-in function and validation failed, so donot proceed ahead.
-// }
- // Double verification: verify passed function name is a (non-parameterized) built-in function.
- try {
- BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (bife != null){
- // It is a builtin function
- setMultiAssignmentStatement(targetList, bife, ctx);
- return;
- }
-
- ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (pbife != null){
- // It is a parameterized builtin function
- setMultiAssignmentStatement(targetList, pbife, ctx);
- return;
- }
-
- // built-in read, rand ...
- DataExpression dbife = DataExpression.getDataExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (dbife != null){
- setMultiAssignmentStatement(targetList, dbife, ctx);
- return;
- }
- } catch(Exception e) {
- helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
- return;
- }
- }
-
- setMultiAssignmentStatement(targetList, functCall, ctx);
- }
-
- private StatementBlock getStatementBlock(Statement current) {
- return DMLParserWrapper.getStatementBlock(current);
- }
-
- @Override
- public void exitIfStatement(IfStatementContext ctx) {
- IfStatement ifStmt = new IfStatement();
- ConditionalPredicate predicate = new ConditionalPredicate(ctx.predicate.info.expr);
- ifStmt.setConditionalPredicate(predicate);
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- ifStmt.setAllPositions(fileName, line, col, line, col);
-
- if(ctx.ifBody.size() > 0) {
- for(StatementContext stmtCtx : ctx.ifBody) {
- ifStmt.addStatementBlockIfBody(getStatementBlock(stmtCtx.info.stmt));
- }
- ifStmt.mergeStatementBlocksIfBody();
- }
-
- if(ctx.elseBody.size() > 0) {
- for(StatementContext stmtCtx : ctx.elseBody) {
- ifStmt.addStatementBlockElseBody(getStatementBlock(stmtCtx.info.stmt));
- }
- ifStmt.mergeStatementBlocksElseBody();
- }
-
- ctx.info.stmt = ifStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
- @Override
- public void exitWhileStatement(WhileStatementContext ctx) {
- WhileStatement whileStmt = new WhileStatement();
- ConditionalPredicate predicate = new ConditionalPredicate(ctx.predicate.info.expr);
- whileStmt.setPredicate(predicate);
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- whileStmt.setAllPositions(fileName, line, col, line, col);
-
- if(ctx.body.size() > 0) {
- for(StatementContext stmtCtx : ctx.body) {
- whileStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
- }
- whileStmt.mergeStatementBlocks();
- }
-
- ctx.info.stmt = whileStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
- @Override
- public void exitForStatement(ForStatementContext ctx) {
- ForStatement forStmt = new ForStatement();
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText());
- HashMap<String, String> parForParamValues = null;
- Expression incrementExpr = new IntIdentifier(1, fileName, line, col, line, col);
- if(ctx.iterPred.info.increment != null) {
- incrementExpr = ctx.iterPred.info.increment;
- }
- IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, fileName, line, col, line, col);
- forStmt.setPredicate(predicate);
-
- if(ctx.body.size() > 0) {
- for(StatementContext stmtCtx : ctx.body) {
- forStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
- }
- forStmt.mergeStatementBlocks();
- }
- ctx.info.stmt = forStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
- @Override
- public void exitParForStatement(ParForStatementContext ctx) {
- ParForStatement parForStmt = new ParForStatement();
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText());
- HashMap<String, String> parForParamValues = new HashMap<String, String>();
- if(ctx.parForParams != null && ctx.parForParams.size() > 0) {
- for(StrictParameterizedExpressionContext parForParamCtx : ctx.parForParams) {
- parForParamValues.put(parForParamCtx.paramName.getText(), parForParamCtx.paramVal.getText());
- }
- }
-
- Expression incrementExpr = new IntIdentifier(1, fileName, line, col, line, col);
-
- if( ctx.iterPred.info.increment != null ) {
- incrementExpr = ctx.iterPred.info.increment;
- }
- IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, fileName, line, col, line, col);
- parForStmt.setPredicate(predicate);
- if(ctx.body.size() > 0) {
- for(StatementContext stmtCtx : ctx.body) {
- parForStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
- }
- parForStmt.mergeStatementBlocks();
- }
- ctx.info.stmt = parForStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
-
- @Override
- public void exitDmlprogram(DmlprogramContext ctx) { }
-
-
- // ----------------------------------------------------------------------
- @Override
- public void exitValueType(ValueTypeContext ctx) { }
- @Override
- public void exitMl_type(Ml_typeContext ctx) { }
-
- private ArrayList<DataIdentifier> getFunctionParameters(List<TypedArgNoAssignContext> ctx) {
- ArrayList<DataIdentifier> retVal = new ArrayList<DataIdentifier>();
- for(TypedArgNoAssignContext paramCtx : ctx) {
- DataIdentifier dataId = new DataIdentifier(paramCtx.paramName.getText());
- String dataType = null;
- String valueType = null;
-
- if(paramCtx.paramType == null || paramCtx.paramType.dataType() == null
- || paramCtx.paramType.dataType().getText() == null || paramCtx.paramType.dataType().getText().isEmpty()) {
- dataType = "scalar";
- }
- else {
- dataType = paramCtx.paramType.dataType().getText();
- }
-
- if(dataType.compareTo("matrix") == 0 || dataType.compareTo("Matrix") == 0) {
- // matrix
- dataId.setDataType(DataType.MATRIX);
- }
- else if(dataType.compareTo("scalar") == 0 || dataType.compareTo("Scalar") == 0) {
- // scalar
- dataId.setDataType(DataType.SCALAR);
- }
- else {
- helper.notifyErrorListeners("invalid datatype " + dataType, paramCtx.start);
- return null;
- }
-
- valueType = paramCtx.paramType.valueType().getText();
- if(valueType.compareTo("int") == 0 || valueType.compareTo("integer") == 0
- || valueType.compareTo("Int") == 0 || valueType.compareTo("Integer") == 0) {
- dataId.setValueType(ValueType.INT);
- }
- else if(valueType.compareTo("string") == 0 || valueType.compareTo("String") == 0) {
- dataId.setValueType(ValueType.STRING);
- }
- else if(valueType.compareTo("boolean") == 0 || valueType.compareTo("Boolean") == 0) {
- dataId.setValueType(ValueType.BOOLEAN);
- }
- else if(valueType.compareTo("double") == 0 || valueType.compareTo("Double") == 0) {
- dataId.setValueType(ValueType.DOUBLE);
- }
- else if(valueType.compareTo("bool") == 0) {
- helper.notifyErrorListeners("invalid valuetype " + valueType + " (Quickfix: use \'boolean\' instead)", paramCtx.start);
- return null;
- }
- else {
- helper.notifyErrorListeners("invalid valuetype " + valueType, paramCtx.start);
- return null;
- }
- retVal.add(dataId);
- }
- return retVal;
- }
-
- @Override
- public void exitInternalFunctionDefExpression(InternalFunctionDefExpressionContext ctx) {
- FunctionStatement functionStmt = new FunctionStatement();
-
- ArrayList<DataIdentifier> functionInputs = getFunctionParameters(ctx.inputParams);
- functionStmt.setInputParams(functionInputs);
-
- // set function outputs
- ArrayList<DataIdentifier> functionOutputs = getFunctionParameters(ctx.outputParams);
- functionStmt.setOutputParams(functionOutputs);
-
- // set function name
- functionStmt.setName(ctx.name.getText());
-
-
- if(ctx.body.size() > 0) {
- // handle function body
- // Create arraylist of one statement block
- ArrayList<StatementBlock> body = new ArrayList<StatementBlock>();
- for(StatementContext stmtCtx : ctx.body) {
- body.add(getStatementBlock(stmtCtx.info.stmt));
- }
- functionStmt.setBody(body);
- functionStmt.mergeStatementBlocks();
- }
- else {
- helper.notifyErrorListeners("functions with no statements are not allowed", ctx.start);
- return;
- }
-
- ctx.info.stmt = functionStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- ctx.info.functionName = ctx.name.getText();
- }
-
- @Override
- public void exitExternalFunctionDefExpression(ExternalFunctionDefExpressionContext ctx) {
- ExternalFunctionStatement functionStmt = new ExternalFunctionStatement();
-
- ArrayList<DataIdentifier> functionInputs = getFunctionParameters(ctx.inputParams);
- functionStmt.setInputParams(functionInputs);
-
- // set function outputs
- ArrayList<DataIdentifier> functionOutputs = getFunctionParameters(ctx.outputParams);
- functionStmt.setOutputParams(functionOutputs);
-
- // set function name
- functionStmt.setName(ctx.name.getText());
-
- // set other parameters
- HashMap<String, String> otherParams = new HashMap<String,String>();
- boolean atleastOneClassName = false;
- for(StrictParameterizedKeyValueStringContext otherParamCtx : ctx.otherParams){
- String paramName = otherParamCtx.paramName.getText();
- String val = "";
- String text = otherParamCtx.paramVal.getText();
- // First unquote the string
- if( (text.startsWith("\"") && text.endsWith("\"")) ||
- (text.startsWith("\'") && text.endsWith("\'"))) {
- if(text.length() > 2) {
- val = text.substring(1, text.length()-1);
- }
- // Empty value allowed
- }
- else {
- helper.notifyErrorListeners("the value of user parameter for external function should be of type string", ctx.start);
- return;
- }
- otherParams.put(paramName, val);
- if(paramName.compareTo("classname") == 0) {
- atleastOneClassName = true;
- }
- }
- functionStmt.setOtherParams(otherParams);
- if(!atleastOneClassName) {
- helper.notifyErrorListeners("the parameter \'className\' needs to be passed for externalFunction", ctx.start);
- return;
- }
-
-// if(ctx.body.size() > 0) {
-// // handle function body
-// // Create arraylist of one statement block
-// ArrayList<StatementBlock> body = new ArrayList<StatementBlock>();
-// for(StatementContext stmtCtx : ctx.body) {
-// body.add(getStatementBlock(stmtCtx.info.stmt));
-// }
-// ((ExternalFunctionStatement) functionStmt).setBody(body);
-// ((ExternalFunctionStatement) functionStmt).mergeStatementBlocks();
-// }
-// else {
-// helper.notifyErrorListeners("functions with no statements are not allowed", ctx.start);
-// return;
-// }
-
- ctx.info.stmt = functionStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- ctx.info.functionName = ctx.name.getText();
- }
-
-
- @Override
- public void exitPathStatement(PathStatementContext ctx) {
- PathStatement stmt = new PathStatement(ctx.pathValue.getText());
- String filePath = ctx.pathValue.getText();
- if((filePath.startsWith("\"") && filePath.endsWith("\"")) ||
- filePath.startsWith("'") && filePath.endsWith("'")) {
- filePath = filePath.substring(1, filePath.length()-1);
- }
-
- _workingDir = filePath;
- ctx.info.stmt = stmt;
- }
-
- @Override
- public void exitIfdefAssignmentStatement(IfdefAssignmentStatementContext ctx) {
- if(!ctx.commandLineParam.getText().startsWith("$")) {
- helper.notifyErrorListeners("the first argument of ifdef function should be a commandline argument parameter (which starts with $)", ctx.commandLineParam.start);
- return;
- }
-
- if(ctx.targetList == null || ctx.targetList.size() != 1) {
- helper.notifyErrorListeners("incorrect parsing for ifdef function", ctx.start);
- return;
- }
- String targetListText = ctx.targetList.get(0).getText();
- if(targetListText.startsWith("$")) {
- helper.notifyErrorListeners("lhs of ifdef function cannot be a commandline parameters. Use local variable instead", ctx.start);
- return;
- }
-
- DataIdentifier target = null;
- if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
- target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
- Expression source = null;
- if(ctx.commandLineParam.dataInfo.expr != null) {
- // Since commandline parameter is set
- // The check of following is done in fillExpressionInfoCommandLineParameters:
- // Command line param cannot be empty string
- // If you want to pass space, please quote it
- source = ctx.commandLineParam.dataInfo.expr;
- }
- else {
- source = ctx.source.info.expr;
- }
-
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- try {
- ctx.info.stmt = new AssignmentStatement(target, source, line, col, line, col);
- setFileLineColumn(ctx.info.stmt, ctx);
- } catch (LanguageException e) {
- helper.notifyErrorListeners("invalid assignment for ifdef function", ctx.targetList.get(0).start);
- return;
- }
-
- }
- else {
- helper.notifyErrorListeners("incorrect lvalue in ifdef function... strange", ctx.targetList.get(0).start);
- return;
- }
-
- }
-
- // ----------------------------------------------------------------------
- @Override
- public void exitParameterizedExpression(ParameterizedExpressionContext ctx) { }
-
-
- @Override
- public void exitStrictParameterizedExpression(StrictParameterizedExpressionContext ctx) { }
-
- @Override
- public void exitTypedArgNoAssign(TypedArgNoAssignContext ctx) { }
- @Override
- public void enterIfdefAssignmentStatement(IfdefAssignmentStatementContext ctx) { }
- @Override
- public void enterMatrixDataTypeCheck(MatrixDataTypeCheckContext ctx) { }
- @Override
- public void exitMatrixDataTypeCheck(MatrixDataTypeCheckContext ctx) {
- if( ctx.ID().getText().compareTo("matrix") == 0
- || ctx.ID().getText().compareTo("Matrix") == 0
- || ctx.ID().getText().compareTo("Scalar") == 0
- || ctx.ID().getText().compareTo("scalar") == 0
- ) {
- // Do nothing
- }
- else {
- helper.notifyErrorListeners("incorrect datatype (expected matrix or scalar)", ctx.start);
- }
- }
-
- @Override
- public void enterBuiltinFunctionExpression(BuiltinFunctionExpressionContext ctx) {}
- @Override
- public void enterStrictParameterizedKeyValueString(StrictParameterizedKeyValueStringContext ctx) { }
- @Override
- public void exitStrictParameterizedKeyValueString(StrictParameterizedKeyValueStringContext ctx) {}
- @Override
- public void enterIterablePredicateColonExpression(IterablePredicateColonExpressionContext ctx) {}
- @Override
- public void enterIterablePredicateSeqExpression(IterablePredicateSeqExpressionContext ctx) { }
-
- @Override
- public void exitIterablePredicateColonExpression(IterablePredicateColonExpressionContext ctx) {
- ctx.info.from = ctx.from.info.expr;
- ctx.info.to = ctx.to.info.expr;
- ctx.info.increment = null;
- }
-
- @Override
- public void exitIterablePredicateSeqExpression(IterablePredicateSeqExpressionContext ctx) {
- if(ctx.ID().getText().compareTo("seq") != 0) {
- helper.notifyErrorListeners("incorrect function:\'" + ctx.ID().getText() + "\'. expected \'seq\'", ctx.start);
- return;
- }
- ctx.info.from = ctx.from.info.expr;
- ctx.info.to = ctx.to.info.expr;
- ctx.info.increment = ctx.increment.info.expr;
- }
-
- @Override
- public void enterConstFalseExpression(ConstFalseExpressionContext ctx) { }
- @Override
- public void enterConstTrueExpression(ConstTrueExpressionContext ctx) { }
-
- @Override
- public void exitConstFalseExpression(ConstFalseExpressionContext ctx) {
- boolean val = false;
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
-
- @Override
- public void exitConstTrueExpression(ConstTrueExpressionContext ctx) {
- boolean val = true;
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidatorHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidatorHelper.java b/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidatorHelper.java
deleted file mode 100644
index 91dbbf0..0000000
--- a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticValidatorHelper.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.parser.antlr4;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.antlr.v4.runtime.Token;
-
-import org.apache.sysml.parser.DMLProgram;
-import org.apache.sysml.parser.antlr4.DmlParser.FunctionCallAssignmentStatementContext;
-import org.apache.sysml.parser.antlr4.DmlParser.ParameterizedExpressionContext;
-import org.apache.sysml.parser.antlr4.DmlSyntacticErrorListener.CustomDmlErrorListener;
-
-public class DmlSyntacticValidatorHelper {
-
- private CustomDmlErrorListener _errorListener = null;
-
- public DmlSyntacticValidatorHelper(CustomDmlErrorListener errorListener) {
- this._errorListener = errorListener;
- }
-
- public void notifyErrorListeners(String message, int line, int charPositionInLine) {
- this._errorListener.validationError(line, charPositionInLine, message);
- }
-
- public void notifyErrorListeners(String message, Token op) {
- this._errorListener.validationError(op.getLine(), op.getCharPositionInLine(), message);
- }
-
- public void raiseWarning(String message, Token op) {
- this._errorListener.validationWarning(op.getLine(), op.getCharPositionInLine(), message);
- }
-
- public String getCurrentFileName() {
- return _errorListener.peekFileName();
- }
-
-// public static void setInfoForArithmeticOp(org.apache.sysml.parser.Expression current,
-// org.apache.sysml.parser.Expression left,
-// org.apache.sysml.parser.Expression right, String opStr) {
-// try {
-// // PLUS, MINUS, MULT, DIV, MODULUS, INTDIV, MATMULT, POW, INVALID
-// org.apache.sysml.parser.Expression.BinaryOp bop = org.apache.sysml.parser.Expression.getBinaryOp(opStr);
-// current = new org.apache.sysml.parser.BinaryExpression(bop);
-// ((org.apache.sysml.parser.BinaryExpression)current).setLeft(left);
-// ((org.apache.sysml.parser.BinaryExpression)current).setRight(right);
-// ((org.apache.sysml.parser.BinaryExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
-// }
-// catch(Exception e) {
-// System.out.println("In setInfoForArithmeticOp>>");
-// e.printStackTrace();
-// }
-// }
-
-// public static void setInfoForBooleanOp(org.apache.sysml.parser.Expression current,
-// org.apache.sysml.parser.Expression left,
-// org.apache.sysml.parser.Expression right, String opStr) {
-// org.apache.sysml.parser.Expression.BooleanOp bop = org.apache.sysml.parser.Expression.getBooleanOp(opStr);
-// current = new org.apache.sysml.parser.BooleanExpression(bop);
-// ((org.apache.sysml.parser.BooleanExpression)current).setLeft(left);
-// ((org.apache.sysml.parser.BooleanExpression)current).setRight(right);
-// ((org.apache.sysml.parser.BooleanExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
-// }
-
- public boolean validateBuiltinFunctions(FunctionCallAssignmentStatementContext ctx) {
- String functionName = ctx.name.getText().replaceAll(" ", "").trim();
- if(functionName.compareTo("write") == 0 || functionName.compareTo(DMLProgram.DEFAULT_NAMESPACE + "::write") == 0) {
- return validateBuiltinWriteFunction(ctx);
- }
- return true;
- }
-
- private boolean validateBuiltinWriteFunction(FunctionCallAssignmentStatementContext ctx) {
-
- return true;
- }
-
- public ArrayList<org.apache.sysml.parser.ParameterExpression> getParameterExpressionList(List<ParameterizedExpressionContext> paramExprs) {
- ArrayList<org.apache.sysml.parser.ParameterExpression> retVal = new ArrayList<org.apache.sysml.parser.ParameterExpression>();
- for(ParameterizedExpressionContext ctx : paramExprs) {
- String paramName = null;
- if(ctx.paramName != null && ctx.paramName.getText() != null && !ctx.paramName.getText().isEmpty()) {
- paramName = ctx.paramName.getText();
- }
- org.apache.sysml.parser.ParameterExpression myArg = new org.apache.sysml.parser.ParameterExpression(paramName, ctx.paramVal.info.expr);
- retVal.add(myArg);
- }
- return retVal;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/antlr4/ExpressionInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/ExpressionInfo.java b/src/main/java/org/apache/sysml/parser/antlr4/ExpressionInfo.java
deleted file mode 100644
index 3116290..0000000
--- a/src/main/java/org/apache/sysml/parser/antlr4/ExpressionInfo.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.parser.antlr4;
-
-
-public class ExpressionInfo {
-
- public org.apache.sysml.parser.Expression expr = null;
-
- // For parfor and for
- public org.apache.sysml.parser.Expression from = null;
- public org.apache.sysml.parser.Expression to = null;
- public org.apache.sysml.parser.Expression increment = null;
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/antlr4/StatementInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/StatementInfo.java b/src/main/java/org/apache/sysml/parser/antlr4/StatementInfo.java
deleted file mode 100644
index 7bd2edc..0000000
--- a/src/main/java/org/apache/sysml/parser/antlr4/StatementInfo.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.parser.antlr4;
-
-import java.util.HashMap;
-import org.apache.sysml.parser.DMLProgram;
-
-public class StatementInfo {
-
- public org.apache.sysml.parser.Statement stmt = null;
-
- // Valid only for import statements
- public HashMap<String,DMLProgram> namespaces = null;
-
- // Valid only for function statement
- //public String namespace = DMLProgram.DEFAULT_NAMESPACE;
- public String functionName = "";
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/dml/DMLParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/dml/DMLParserWrapper.java b/src/main/java/org/apache/sysml/parser/dml/DMLParserWrapper.java
new file mode 100644
index 0000000..b64f7d2
--- /dev/null
+++ b/src/main/java/org/apache/sysml/parser/dml/DMLParserWrapper.java
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.parser.dml;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.antlr.v4.runtime.BailErrorStrategy;
+import org.antlr.v4.runtime.DefaultErrorStrategy;
+import org.antlr.v4.runtime.atn.PredictionMode;
+import org.antlr.v4.runtime.misc.ParseCancellationException;
+import org.antlr.v4.runtime.tree.ParseTreeWalker;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.conf.ConfigurationManager;
+import org.apache.sysml.parser.AParserWrapper;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.ForStatement;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.FunctionStatementBlock;
+import org.apache.sysml.parser.IfStatement;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.ImportStatement;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.parser.ParForStatement;
+import org.apache.sysml.parser.ParForStatementBlock;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.WhileStatement;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.parser.dml.DmlParser.DmlprogramContext;
+import org.apache.sysml.parser.dml.DmlParser.FunctionStatementContext;
+import org.apache.sysml.parser.dml.DmlParser.StatementContext;
+import org.apache.sysml.parser.dml.DmlSyntacticErrorListener.CustomDmlErrorListener;
+import org.apache.sysml.runtime.util.LocalFileUtils;
+
+/**
+ * This is the main entry point for the Antlr4 parser.
+ * Dml.g4 is the grammar file which enforces syntactic structure of DML program.
+ * DmlSyntaticValidator on other hand captures little bit of semantic as well as does the job of translation of Antlr AST to DMLProgram.
+ * At a high-level, DmlSyntaticValidator implements call-back methods that are called by walker.walk(validator, tree)
+ * The callback methods are of two type: enterSomeASTNode() and exitSomeASTNode()
+ * It is important to note that almost every node in AST has either ExpressionInfo or StatementInfo object associated with it.
+ * The key design decision is that while "exiting" the node (i.e. callback to exitSomeASTNode), we use information in given AST node and construct an object of type Statement or Expression and put it in StatementInfo or ExpressionInfo respectively.
+ * This way it avoids any bugs due to lookahead and one only has to "think as an AST node", thereby making any changes to parse code much simpler :)
+ *
+ * Note: to add additional builtin function, one only needs to modify DmlSyntaticValidator (which is java file and provides full Eclipse tooling support) not g4.
+ *
+ * To separate logic of semantic validation, DmlSyntaticValidatorHelper contains functions that do semantic validation. Currently, there is no semantic validation as most of it is delegated to subsequent validation phase.
+ *
+ * Whenever there is a parse error, it goes through DmlSyntacticErrorListener. This allows us to pipe the error messages to any future pipeline as well as control the format in an elegant manner.
+ * There are three types of messages passed:
+ * - Syntactic errors: When passed DML script doesnot conform to syntatic structure enforced by Dml.g4
+ * - Validation errors: Errors due to translation of AST to DMLProgram
+ * - Validation warnings: Messages to inform users that there might be potential bug in their program
+ *
+ * As of this moment, Antlr4ParserWrapper is stateful and cannot be multithreaded. This is not big deal because each users calls SystemML in different process.
+ * If in future we intend to make it multi-threaded, look at cleanUpState method and resolve the dependency accordingly.
+ *
+ */
+public class DMLParserWrapper extends AParserWrapper
+{
+ private static final Log LOG = LogFactory.getLog(DMLScript.class.getName());
+
+ /**
+ * Custom wrapper to convert statement into statement blocks. Called by doParse and in DmlSyntacticValidator for for, parfor, while, ...
+ * @param current a statement
+ * @return corresponding statement block
+ */
+ public static StatementBlock getStatementBlock(Statement current) {
+ StatementBlock blk = null;
+ if(current instanceof ParForStatement) {
+ blk = new ParForStatementBlock();
+ blk.addStatement(current);
+ }
+ else if(current instanceof ForStatement) {
+ blk = new ForStatementBlock();
+ blk.addStatement(current);
+ }
+ else if(current instanceof IfStatement) {
+ blk = new IfStatementBlock();
+ blk.addStatement(current);
+ }
+ else if(current instanceof WhileStatement) {
+ blk = new WhileStatementBlock();
+ blk.addStatement(current);
+ }
+ else {
+ // This includes ImportStatement
+ blk = new StatementBlock();
+ blk.addStatement(current);
+ }
+ return blk;
+ }
+
+ /**
+ * Parses the passed file with command line parameters. You can either pass both (local file) or just dmlScript (hdfs) or just file name (import command)
+ * @param fileName either full path or null --> only used for better error handling
+ * @param dmlScript required
+ * @param argVals
+ * @return
+ * @throws ParseException
+ */
+ @Override
+ public DMLProgram parse(String fileName, String dmlScript, HashMap<String,String> argVals)
+ throws ParseException
+ {
+ DMLProgram prog = null;
+
+ if(dmlScript == null || dmlScript.trim().isEmpty()) {
+ throw new ParseException("Incorrect usage of parse. Please pass dmlScript not just filename");
+ }
+
+ // Set the pipeline required for ANTLR parsing
+ DMLParserWrapper parser = new DMLParserWrapper();
+ prog = parser.doParse(fileName, dmlScript, argVals);
+
+ if(prog == null) {
+ throw new ParseException("One or more errors found during parsing (could not construct AST for file: " + fileName + "). Cannot proceed ahead.");
+ }
+ return prog;
+ }
+
+ /**
+ * This function is supposed to be called directly only from DmlSyntacticValidator when it encounters 'import'
+ * @param fileName
+ * @return null if atleast one error
+ */
+ public DMLProgram doParse(String fileName, String dmlScript, HashMap<String,String> argVals) throws ParseException {
+ DMLProgram dmlPgm = null;
+
+ org.antlr.v4.runtime.ANTLRInputStream in;
+ try {
+ if(dmlScript == null) {
+ dmlScript = readDMLScript(fileName);
+ }
+
+ InputStream stream = new ByteArrayInputStream(dmlScript.getBytes());
+ in = new org.antlr.v4.runtime.ANTLRInputStream(stream);
+// else {
+// if(!(new File(fileName)).exists()) {
+// throw new ParseException("ERROR: Cannot open file:" + fileName);
+// }
+// in = new org.antlr.v4.runtime.ANTLRInputStream(new java.io.FileInputStream(fileName));
+// }
+ } catch (FileNotFoundException e) {
+ throw new ParseException("ERROR: Cannot find file:" + fileName, e);
+ } catch (IOException e) {
+ throw new ParseException("ERROR: Cannot open file:" + fileName, e);
+ } catch (LanguageException e) {
+ throw new ParseException("ERROR: " + e.getMessage(), e);
+ }
+
+ DmlprogramContext ast = null;
+ CustomDmlErrorListener errorListener = new CustomDmlErrorListener();
+
+ try {
+ DmlLexer lexer = new DmlLexer(in);
+ org.antlr.v4.runtime.CommonTokenStream tokens = new org.antlr.v4.runtime.CommonTokenStream(lexer);
+ DmlParser antlr4Parser = new DmlParser(tokens);
+
+ boolean tryOptimizedParsing = false; // For now no optimization, since it is not able to parse integer value.
+
+ if(tryOptimizedParsing) {
+ // Try faster and simpler SLL
+ antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
+ antlr4Parser.removeErrorListeners();
+ antlr4Parser.setErrorHandler(new BailErrorStrategy());
+ try{
+ ast = antlr4Parser.dmlprogram();
+ // If successful, no need to try out full LL(*) ... SLL was enough
+ }
+ catch(ParseCancellationException ex) {
+ // Error occurred, so now try full LL(*) for better error messages
+ tokens.reset();
+ antlr4Parser.reset();
+ if(fileName != null) {
+ errorListener.pushCurrentFileName(fileName);
+ // DmlSyntacticErrorListener.currentFileName.push(fileName);
+ }
+ else {
+ errorListener.pushCurrentFileName("MAIN_SCRIPT");
+ // DmlSyntacticErrorListener.currentFileName.push("MAIN_SCRIPT");
+ }
+ // Set our custom error listener
+ antlr4Parser.addErrorListener(errorListener);
+ antlr4Parser.setErrorHandler(new DefaultErrorStrategy());
+ antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL);
+ ast = antlr4Parser.dmlprogram();
+ }
+ }
+ else {
+ // Set our custom error listener
+ antlr4Parser.removeErrorListeners();
+ antlr4Parser.addErrorListener(errorListener);
+ errorListener.pushCurrentFileName(fileName);
+
+ // Now do the parsing
+ ast = antlr4Parser.dmlprogram();
+ }
+ }
+ catch(Exception e) {
+ throw new ParseException("ERROR: Cannot parse the program:" + fileName, e);
+ }
+
+
+ try {
+ // Now convert the parse tree into DMLProgram
+ // Do syntactic validation while converting
+ org.antlr.v4.runtime.tree.ParseTree tree = ast;
+ // And also do syntactic validation
+ org.antlr.v4.runtime.tree.ParseTreeWalker walker = new ParseTreeWalker();
+ DmlSyntacticValidatorHelper helper = new DmlSyntacticValidatorHelper(errorListener);
+ DmlSyntacticValidator validator = new DmlSyntacticValidator(helper, errorListener.peekFileName(), argVals);
+ walker.walk(validator, tree);
+ errorListener.popFileName();
+ if(errorListener.isAtleastOneError()) {
+ return null;
+ }
+ dmlPgm = createDMLProgram(ast);
+ }
+ catch(Exception e) {
+ throw new ParseException("ERROR: Cannot translate the parse tree into DMLProgram" + e.getMessage(), e);
+ }
+
+ return dmlPgm;
+ }
+
+ private DMLProgram createDMLProgram(DmlprogramContext ast) {
+
+ DMLProgram dmlPgm = new DMLProgram();
+
+ // First add all the functions
+ for(FunctionStatementContext fn : ast.functionBlocks) {
+ FunctionStatementBlock functionStmtBlk = new FunctionStatementBlock();
+ functionStmtBlk.addStatement(fn.info.stmt);
+ try {
+ // TODO: currently the logic of nested namespace is not clear.
+ String namespace = DMLProgram.DEFAULT_NAMESPACE;
+ dmlPgm.addFunctionStatementBlock(namespace, fn.info.functionName, functionStmtBlk);
+ } catch (LanguageException e) {
+ LOG.error("line: " + fn.start.getLine() + ":" + fn.start.getCharPositionInLine() + " cannot process the function " + fn.info.functionName);
+ return null;
+ }
+ }
+
+ // Then add all the statements
+ for(StatementContext stmtCtx : ast.blocks) {
+ org.apache.sysml.parser.Statement current = stmtCtx.info.stmt;
+ if(current == null) {
+ LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the statement");
+ return null;
+ }
+
+ if(current instanceof ImportStatement) {
+ // Handle import statements separately
+ if(stmtCtx.info.namespaces != null) {
+ // Add the DMLProgram entries into current program
+ for(Map.Entry<String, DMLProgram> entry : stmtCtx.info.namespaces.entrySet()) {
+ dmlPgm.getNamespaces().put(entry.getKey(), entry.getValue());
+
+// // Don't add DMLProgram into the current program, just add function statements
+ // dmlPgm.getNamespaces().put(entry.getKey(), entry.getValue());
+ // Add function statements to current dml program
+// DMLProgram importedPgm = entry.getValue();
+// try {
+// for(FunctionStatementBlock importedFnBlk : importedPgm.getFunctionStatementBlocks()) {
+// if(importedFnBlk.getStatements() != null && importedFnBlk.getStatements().size() == 1) {
+// String functionName = ((FunctionStatement)importedFnBlk.getStatement(0)).getName();
+// System.out.println("Adding function => " + entry.getKey() + "::" + functionName);
+// TODO:33
+// dmlPgm.addFunctionStatementBlock(entry.getKey(), functionName, importedFnBlk);
+// }
+// else {
+// LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " incorrect number of functions in the imported function block .... strange");
+// return null;
+// }
+// }
+// if(importedPgm.getStatementBlocks() != null && importedPgm.getStatementBlocks().size() > 0) {
+// LOG.warn("Only the functions can be imported from the namespace " + entry.getKey());
+// }
+// } catch (LanguageException e) {
+// LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot import functions from the file in the import statement: " + e.getMessage());
+// return null;
+// }
+ }
+ }
+ else {
+ LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the import statement");
+ return null;
+ }
+ }
+
+ // Now wrap statement into individual statement block
+ // merge statement will take care of merging these blocks
+ dmlPgm.addStatementBlock(getStatementBlock(current));
+ }
+
+ dmlPgm.mergeStatementBlocks();
+ return dmlPgm;
+ }
+
+ public static String readDMLScript( String script )
+ throws IOException, LanguageException
+ {
+ String dmlScriptStr = null;
+
+ //read DML script from file
+ if(script == null)
+ throw new LanguageException("DML script path was not specified!");
+
+ StringBuilder sb = new StringBuilder();
+ BufferedReader in = null;
+ try
+ {
+ //read from hdfs or gpfs file system
+ if( script.startsWith("hdfs:")
+ || script.startsWith("gpfs:") )
+ {
+ if( !LocalFileUtils.validateExternalFilename(script, true) )
+ throw new LanguageException("Invalid (non-trustworthy) hdfs filename.");
+ FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
+ Path scriptPath = new Path(script);
+ in = new BufferedReader(new InputStreamReader(fs.open(scriptPath)));
+ }
+ // from local file system
+ else
+ {
+ if( !LocalFileUtils.validateExternalFilename(script, false) )
+ throw new LanguageException("Invalid (non-trustworthy) local filename.");
+ in = new BufferedReader(new FileReader(script));
+ }
+
+ //core script reading
+ String tmp = null;
+ while ((tmp = in.readLine()) != null)
+ {
+ sb.append( tmp );
+ sb.append( "\n" );
+ }
+ }
+ catch (IOException ex)
+ {
+ LOG.error("Failed to read the script from the file system", ex);
+ throw ex;
+ }
+ finally
+ {
+ if( in != null )
+ in.close();
+ }
+
+ dmlScriptStr = sb.toString();
+
+ return dmlScriptStr;
+ }
+}
[8/8] incubator-systemml git commit: [SYSTEMML-148] Refactored
.parser.{antlr4, python} to .parser.{dml, pydml}.
Posted by du...@apache.org.
[SYSTEMML-148] Refactored .parser.{antlr4,python} to .parser.{dml,pydml}.
Closes #44.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/c04fc99f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/c04fc99f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/c04fc99f
Branch: refs/heads/master
Commit: c04fc99fc1e6d6cc5f5546dd86f51513da63714c
Parents: 8ba0fdc
Author: Nakul Jindal <na...@gmail.com>
Authored: Wed Jan 20 17:02:12 2016 -0800
Committer: Mike Dusenberry <mw...@us.ibm.com>
Committed: Wed Jan 20 17:02:12 2016 -0800
----------------------------------------------------------------------
.gitignore | 16 +-
pom.xml | 16 +-
.../org/apache/sysml/parser/AParserWrapper.java | 4 +-
.../sysml/parser/antlr4/DMLParserWrapper.java | 385 ----
.../java/org/apache/sysml/parser/antlr4/Dml.g4 | 201 --
.../antlr4/DmlSyntacticErrorListener.java | 115 -
.../parser/antlr4/DmlSyntacticValidator.java | 1533 -------------
.../antlr4/DmlSyntacticValidatorHelper.java | 108 -
.../sysml/parser/antlr4/ExpressionInfo.java | 31 -
.../sysml/parser/antlr4/StatementInfo.java | 36 -
.../sysml/parser/dml/DMLParserWrapper.java | 385 ++++
.../java/org/apache/sysml/parser/dml/Dml.g4 | 201 ++
.../parser/dml/DmlSyntacticErrorListener.java | 115 +
.../sysml/parser/dml/DmlSyntacticValidator.java | 1533 +++++++++++++
.../parser/dml/DmlSyntacticValidatorHelper.java | 108 +
.../apache/sysml/parser/dml/ExpressionInfo.java | 31 +
.../apache/sysml/parser/dml/StatementInfo.java | 36 +
.../sysml/parser/pydml/ExpressionInfo.java | 32 +
.../sysml/parser/pydml/PyDMLParserWrapper.java | 283 +++
.../java/org/apache/sysml/parser/pydml/Pydml.g4 | 384 ++++
.../pydml/PydmlSyntacticErrorListener.java | 114 +
.../parser/pydml/PydmlSyntacticValidator.java | 2040 +++++++++++++++++
.../pydml/PydmlSyntacticValidatorHelper.java | 101 +
.../sysml/parser/pydml/StatementInfo.java | 32 +
.../sysml/parser/python/ExpressionInfo.java | 32 -
.../sysml/parser/python/PyDMLParserWrapper.java | 283 ---
.../org/apache/sysml/parser/python/Pydml.g4 | 384 ----
.../python/PydmlSyntacticErrorListener.java | 114 -
.../parser/python/PydmlSyntacticValidator.java | 2042 ------------------
.../python/PydmlSyntacticValidatorHelper.java | 101 -
.../sysml/parser/python/StatementInfo.java | 32 -
31 files changed, 5413 insertions(+), 5415 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 7df2eea..2d11e27 100644
--- a/.gitignore
+++ b/.gitignore
@@ -41,14 +41,14 @@ src/main/java/Dml.tokens
src/main/java/DmlLexer.tokens
src/main/java/Pydml.tokens
src/main/java/PydmlLexer.tokens
-src/main/java/org/apache/sysml/parser/antlr4/DmlBaseListener.java
-src/main/java/org/apache/sysml/parser/antlr4/DmlLexer.java
-src/main/java/org/apache/sysml/parser/antlr4/DmlListener.java
-src/main/java/org/apache/sysml/parser/antlr4/DmlParser.java
-src/main/java/org/apache/sysml/parser/python/PydmlBaseListener.java
-src/main/java/org/apache/sysml/parser/python/PydmlLexer.java
-src/main/java/org/apache/sysml/parser/python/PydmlListener.java
-src/main/java/org/apache/sysml/parser/python/PydmlParser.java
+src/main/java/org/apache/sysml/parser/dml/DmlBaseListener.java
+src/main/java/org/apache/sysml/parser/dml/DmlLexer.java
+src/main/java/org/apache/sysml/parser/dml/DmlListener.java
+src/main/java/org/apache/sysml/parser/dml/DmlParser.java
+src/main/java/org/apache/sysml/parser/pydml/PydmlBaseListener.java
+src/main/java/org/apache/sysml/parser/pydml/PydmlLexer.java
+src/main/java/org/apache/sysml/parser/pydml/PydmlListener.java
+src/main/java/org/apache/sysml/parser/pydml/PydmlParser.java
src/test/scripts/**/in
src/test/scripts/**/out
src/test/scripts/**/expected
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index bba680c..dd4a586 100644
--- a/pom.xml
+++ b/pom.xml
@@ -474,14 +474,14 @@
<exclude>src/main/java/DmlLexer.tokens</exclude>
<exclude>src/main/java/Pydml.tokens</exclude>
<exclude>src/main/java/PydmlLexer.tokens</exclude>
- <exclude>src/main/java/org/apache/sysml/parser/antlr4/DmlBaseListener.java</exclude>
- <exclude>src/main/java/org/apache/sysml/parser/antlr4/DmlLexer.java</exclude>
- <exclude>src/main/java/org/apache/sysml/parser/antlr4/DmlListener.java</exclude>
- <exclude>src/main/java/org/apache/sysml/parser/antlr4/DmlParser.java</exclude>
- <exclude>src/main/java/org/apache/sysml/parser/python/PydmlBaseListener.java</exclude>
- <exclude>src/main/java/org/apache/sysml/parser/python/PydmlLexer.java</exclude>
- <exclude>src/main/java/org/apache/sysml/parser/python/PydmlListener.java</exclude>
- <exclude>src/main/java/org/apache/sysml/parser/python/PydmlParser.java</exclude>
+ <exclude>src/main/java/org/apache/sysml/parser/dml/DmlBaseListener.java</exclude>
+ <exclude>src/main/java/org/apache/sysml/parser/dml/DmlLexer.java</exclude>
+ <exclude>src/main/java/org/apache/sysml/parser/dml/DmlListener.java</exclude>
+ <exclude>src/main/java/org/apache/sysml/parser/dml/DmlParser.java</exclude>
+ <exclude>src/main/java/org/apache/sysml/parser/pydml/PydmlBaseListener.java</exclude>
+ <exclude>src/main/java/org/apache/sysml/parser/pydml/PydmlLexer.java</exclude>
+ <exclude>src/main/java/org/apache/sysml/parser/pydml/PydmlListener.java</exclude>
+ <exclude>src/main/java/org/apache/sysml/parser/pydml/PydmlParser.java</exclude>
<!-- Test Validation files -->
<exclude>src/test/scripts/functions/external/kMeans/kMeansWrapperOutput</exclude>
<exclude>src/test/scripts/functions/external/kMeans2/kMeansWrapperOutput1</exclude>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/AParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/AParserWrapper.java b/src/main/java/org/apache/sysml/parser/AParserWrapper.java
index fd600f2..6e67954 100644
--- a/src/main/java/org/apache/sysml/parser/AParserWrapper.java
+++ b/src/main/java/org/apache/sysml/parser/AParserWrapper.java
@@ -21,8 +21,8 @@ package org.apache.sysml.parser;
import java.util.HashMap;
-import org.apache.sysml.parser.antlr4.DMLParserWrapper;
-import org.apache.sysml.parser.python.PyDMLParserWrapper;
+import org.apache.sysml.parser.dml.DMLParserWrapper;
+import org.apache.sysml.parser.pydml.PyDMLParserWrapper;
/**
* Base class for all dml parsers in order to make the various compilation chains
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/antlr4/DMLParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/DMLParserWrapper.java b/src/main/java/org/apache/sysml/parser/antlr4/DMLParserWrapper.java
deleted file mode 100644
index 25b7b5e..0000000
--- a/src/main/java/org/apache/sysml/parser/antlr4/DMLParserWrapper.java
+++ /dev/null
@@ -1,385 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.parser.antlr4;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.antlr.v4.runtime.BailErrorStrategy;
-import org.antlr.v4.runtime.DefaultErrorStrategy;
-import org.antlr.v4.runtime.atn.PredictionMode;
-import org.antlr.v4.runtime.misc.ParseCancellationException;
-import org.antlr.v4.runtime.tree.ParseTreeWalker;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.sysml.api.DMLScript;
-import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.parser.AParserWrapper;
-import org.apache.sysml.parser.DMLProgram;
-import org.apache.sysml.parser.ForStatement;
-import org.apache.sysml.parser.ForStatementBlock;
-import org.apache.sysml.parser.FunctionStatementBlock;
-import org.apache.sysml.parser.IfStatement;
-import org.apache.sysml.parser.IfStatementBlock;
-import org.apache.sysml.parser.ImportStatement;
-import org.apache.sysml.parser.LanguageException;
-import org.apache.sysml.parser.ParForStatement;
-import org.apache.sysml.parser.ParForStatementBlock;
-import org.apache.sysml.parser.ParseException;
-import org.apache.sysml.parser.Statement;
-import org.apache.sysml.parser.StatementBlock;
-import org.apache.sysml.parser.WhileStatement;
-import org.apache.sysml.parser.WhileStatementBlock;
-import org.apache.sysml.parser.antlr4.DmlParser.DmlprogramContext;
-import org.apache.sysml.parser.antlr4.DmlParser.FunctionStatementContext;
-import org.apache.sysml.parser.antlr4.DmlParser.StatementContext;
-import org.apache.sysml.parser.antlr4.DmlSyntacticErrorListener.CustomDmlErrorListener;
-import org.apache.sysml.runtime.util.LocalFileUtils;
-
-/**
- * This is the main entry point for the Antlr4 parser.
- * Dml.g4 is the grammar file which enforces syntactic structure of DML program.
- * DmlSyntaticValidator on other hand captures little bit of semantic as well as does the job of translation of Antlr AST to DMLProgram.
- * At a high-level, DmlSyntaticValidator implements call-back methods that are called by walker.walk(validator, tree)
- * The callback methods are of two type: enterSomeASTNode() and exitSomeASTNode()
- * It is important to note that almost every node in AST has either ExpressionInfo or StatementInfo object associated with it.
- * The key design decision is that while "exiting" the node (i.e. callback to exitSomeASTNode), we use information in given AST node and construct an object of type Statement or Expression and put it in StatementInfo or ExpressionInfo respectively.
- * This way it avoids any bugs due to lookahead and one only has to "think as an AST node", thereby making any changes to parse code much simpler :)
- *
- * Note: to add additional builtin function, one only needs to modify DmlSyntaticValidator (which is java file and provides full Eclipse tooling support) not g4.
- *
- * To separate logic of semantic validation, DmlSyntaticValidatorHelper contains functions that do semantic validation. Currently, there is no semantic validation as most of it is delegated to subsequent validation phase.
- *
- * Whenever there is a parse error, it goes through DmlSyntacticErrorListener. This allows us to pipe the error messages to any future pipeline as well as control the format in an elegant manner.
- * There are three types of messages passed:
- * - Syntactic errors: When passed DML script doesnot conform to syntatic structure enforced by Dml.g4
- * - Validation errors: Errors due to translation of AST to DMLProgram
- * - Validation warnings: Messages to inform users that there might be potential bug in their program
- *
- * As of this moment, Antlr4ParserWrapper is stateful and cannot be multithreaded. This is not big deal because each users calls SystemML in different process.
- * If in future we intend to make it multi-threaded, look at cleanUpState method and resolve the dependency accordingly.
- *
- */
-public class DMLParserWrapper extends AParserWrapper
-{
- private static final Log LOG = LogFactory.getLog(DMLScript.class.getName());
-
- /**
- * Custom wrapper to convert statement into statement blocks. Called by doParse and in DmlSyntacticValidator for for, parfor, while, ...
- * @param current a statement
- * @return corresponding statement block
- */
- public static StatementBlock getStatementBlock(Statement current) {
- StatementBlock blk = null;
- if(current instanceof ParForStatement) {
- blk = new ParForStatementBlock();
- blk.addStatement(current);
- }
- else if(current instanceof ForStatement) {
- blk = new ForStatementBlock();
- blk.addStatement(current);
- }
- else if(current instanceof IfStatement) {
- blk = new IfStatementBlock();
- blk.addStatement(current);
- }
- else if(current instanceof WhileStatement) {
- blk = new WhileStatementBlock();
- blk.addStatement(current);
- }
- else {
- // This includes ImportStatement
- blk = new StatementBlock();
- blk.addStatement(current);
- }
- return blk;
- }
-
- /**
- * Parses the passed file with command line parameters. You can either pass both (local file) or just dmlScript (hdfs) or just file name (import command)
- * @param fileName either full path or null --> only used for better error handling
- * @param dmlScript required
- * @param argVals
- * @return
- * @throws ParseException
- */
- @Override
- public DMLProgram parse(String fileName, String dmlScript, HashMap<String,String> argVals)
- throws ParseException
- {
- DMLProgram prog = null;
-
- if(dmlScript == null || dmlScript.trim().isEmpty()) {
- throw new ParseException("Incorrect usage of parse. Please pass dmlScript not just filename");
- }
-
- // Set the pipeline required for ANTLR parsing
- DMLParserWrapper parser = new DMLParserWrapper();
- prog = parser.doParse(fileName, dmlScript, argVals);
-
- if(prog == null) {
- throw new ParseException("One or more errors found during parsing (could not construct AST for file: " + fileName + "). Cannot proceed ahead.");
- }
- return prog;
- }
-
- /**
- * This function is supposed to be called directly only from DmlSyntacticValidator when it encounters 'import'
- * @param fileName
- * @return null if atleast one error
- */
- public DMLProgram doParse(String fileName, String dmlScript, HashMap<String,String> argVals) throws ParseException {
- DMLProgram dmlPgm = null;
-
- org.antlr.v4.runtime.ANTLRInputStream in;
- try {
- if(dmlScript == null) {
- dmlScript = readDMLScript(fileName);
- }
-
- InputStream stream = new ByteArrayInputStream(dmlScript.getBytes());
- in = new org.antlr.v4.runtime.ANTLRInputStream(stream);
-// else {
-// if(!(new File(fileName)).exists()) {
-// throw new ParseException("ERROR: Cannot open file:" + fileName);
-// }
-// in = new org.antlr.v4.runtime.ANTLRInputStream(new java.io.FileInputStream(fileName));
-// }
- } catch (FileNotFoundException e) {
- throw new ParseException("ERROR: Cannot find file:" + fileName, e);
- } catch (IOException e) {
- throw new ParseException("ERROR: Cannot open file:" + fileName, e);
- } catch (LanguageException e) {
- throw new ParseException("ERROR: " + e.getMessage(), e);
- }
-
- DmlprogramContext ast = null;
- CustomDmlErrorListener errorListener = new CustomDmlErrorListener();
-
- try {
- DmlLexer lexer = new DmlLexer(in);
- org.antlr.v4.runtime.CommonTokenStream tokens = new org.antlr.v4.runtime.CommonTokenStream(lexer);
- DmlParser antlr4Parser = new DmlParser(tokens);
-
- boolean tryOptimizedParsing = false; // For now no optimization, since it is not able to parse integer value.
-
- if(tryOptimizedParsing) {
- // Try faster and simpler SLL
- antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
- antlr4Parser.removeErrorListeners();
- antlr4Parser.setErrorHandler(new BailErrorStrategy());
- try{
- ast = antlr4Parser.dmlprogram();
- // If successful, no need to try out full LL(*) ... SLL was enough
- }
- catch(ParseCancellationException ex) {
- // Error occurred, so now try full LL(*) for better error messages
- tokens.reset();
- antlr4Parser.reset();
- if(fileName != null) {
- errorListener.pushCurrentFileName(fileName);
- // DmlSyntacticErrorListener.currentFileName.push(fileName);
- }
- else {
- errorListener.pushCurrentFileName("MAIN_SCRIPT");
- // DmlSyntacticErrorListener.currentFileName.push("MAIN_SCRIPT");
- }
- // Set our custom error listener
- antlr4Parser.addErrorListener(errorListener);
- antlr4Parser.setErrorHandler(new DefaultErrorStrategy());
- antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL);
- ast = antlr4Parser.dmlprogram();
- }
- }
- else {
- // Set our custom error listener
- antlr4Parser.removeErrorListeners();
- antlr4Parser.addErrorListener(errorListener);
- errorListener.pushCurrentFileName(fileName);
-
- // Now do the parsing
- ast = antlr4Parser.dmlprogram();
- }
- }
- catch(Exception e) {
- throw new ParseException("ERROR: Cannot parse the program:" + fileName, e);
- }
-
-
- try {
- // Now convert the parse tree into DMLProgram
- // Do syntactic validation while converting
- org.antlr.v4.runtime.tree.ParseTree tree = ast;
- // And also do syntactic validation
- org.antlr.v4.runtime.tree.ParseTreeWalker walker = new ParseTreeWalker();
- DmlSyntacticValidatorHelper helper = new DmlSyntacticValidatorHelper(errorListener);
- DmlSyntacticValidator validator = new DmlSyntacticValidator(helper, errorListener.peekFileName(), argVals);
- walker.walk(validator, tree);
- errorListener.popFileName();
- if(errorListener.isAtleastOneError()) {
- return null;
- }
- dmlPgm = createDMLProgram(ast);
- }
- catch(Exception e) {
- throw new ParseException("ERROR: Cannot translate the parse tree into DMLProgram" + e.getMessage(), e);
- }
-
- return dmlPgm;
- }
-
- private DMLProgram createDMLProgram(DmlprogramContext ast) {
-
- DMLProgram dmlPgm = new DMLProgram();
-
- // First add all the functions
- for(FunctionStatementContext fn : ast.functionBlocks) {
- FunctionStatementBlock functionStmtBlk = new FunctionStatementBlock();
- functionStmtBlk.addStatement(fn.info.stmt);
- try {
- // TODO: currently the logic of nested namespace is not clear.
- String namespace = DMLProgram.DEFAULT_NAMESPACE;
- dmlPgm.addFunctionStatementBlock(namespace, fn.info.functionName, functionStmtBlk);
- } catch (LanguageException e) {
- LOG.error("line: " + fn.start.getLine() + ":" + fn.start.getCharPositionInLine() + " cannot process the function " + fn.info.functionName);
- return null;
- }
- }
-
- // Then add all the statements
- for(StatementContext stmtCtx : ast.blocks) {
- org.apache.sysml.parser.Statement current = stmtCtx.info.stmt;
- if(current == null) {
- LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the statement");
- return null;
- }
-
- if(current instanceof ImportStatement) {
- // Handle import statements separately
- if(stmtCtx.info.namespaces != null) {
- // Add the DMLProgram entries into current program
- for(Map.Entry<String, DMLProgram> entry : stmtCtx.info.namespaces.entrySet()) {
- dmlPgm.getNamespaces().put(entry.getKey(), entry.getValue());
-
-// // Don't add DMLProgram into the current program, just add function statements
- // dmlPgm.getNamespaces().put(entry.getKey(), entry.getValue());
- // Add function statements to current dml program
-// DMLProgram importedPgm = entry.getValue();
-// try {
-// for(FunctionStatementBlock importedFnBlk : importedPgm.getFunctionStatementBlocks()) {
-// if(importedFnBlk.getStatements() != null && importedFnBlk.getStatements().size() == 1) {
-// String functionName = ((FunctionStatement)importedFnBlk.getStatement(0)).getName();
-// System.out.println("Adding function => " + entry.getKey() + "::" + functionName);
-// TODO:33
-// dmlPgm.addFunctionStatementBlock(entry.getKey(), functionName, importedFnBlk);
-// }
-// else {
-// LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " incorrect number of functions in the imported function block .... strange");
-// return null;
-// }
-// }
-// if(importedPgm.getStatementBlocks() != null && importedPgm.getStatementBlocks().size() > 0) {
-// LOG.warn("Only the functions can be imported from the namespace " + entry.getKey());
-// }
-// } catch (LanguageException e) {
-// LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot import functions from the file in the import statement: " + e.getMessage());
-// return null;
-// }
- }
- }
- else {
- LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the import statement");
- return null;
- }
- }
-
- // Now wrap statement into individual statement block
- // merge statement will take care of merging these blocks
- dmlPgm.addStatementBlock(getStatementBlock(current));
- }
-
- dmlPgm.mergeStatementBlocks();
- return dmlPgm;
- }
-
- public static String readDMLScript( String script )
- throws IOException, LanguageException
- {
- String dmlScriptStr = null;
-
- //read DML script from file
- if(script == null)
- throw new LanguageException("DML script path was not specified!");
-
- StringBuilder sb = new StringBuilder();
- BufferedReader in = null;
- try
- {
- //read from hdfs or gpfs file system
- if( script.startsWith("hdfs:")
- || script.startsWith("gpfs:") )
- {
- if( !LocalFileUtils.validateExternalFilename(script, true) )
- throw new LanguageException("Invalid (non-trustworthy) hdfs filename.");
- FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
- Path scriptPath = new Path(script);
- in = new BufferedReader(new InputStreamReader(fs.open(scriptPath)));
- }
- // from local file system
- else
- {
- if( !LocalFileUtils.validateExternalFilename(script, false) )
- throw new LanguageException("Invalid (non-trustworthy) local filename.");
- in = new BufferedReader(new FileReader(script));
- }
-
- //core script reading
- String tmp = null;
- while ((tmp = in.readLine()) != null)
- {
- sb.append( tmp );
- sb.append( "\n" );
- }
- }
- catch (IOException ex)
- {
- LOG.error("Failed to read the script from the file system", ex);
- throw ex;
- }
- finally
- {
- if( in != null )
- in.close();
- }
-
- dmlScriptStr = sb.toString();
-
- return dmlScriptStr;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/antlr4/Dml.g4
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/Dml.g4 b/src/main/java/org/apache/sysml/parser/antlr4/Dml.g4
deleted file mode 100644
index 400a412..0000000
--- a/src/main/java/org/apache/sysml/parser/antlr4/Dml.g4
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-grammar Dml;
-
-@header
-{
- // Commenting the package name and explicitly passing it in build.xml to maintain compatibility with maven plugin
- // package org.apache.sysml.antlr4;
-}
-
-// DML Program is a list of expression
-// For now, we only allow global function definitions (not nested or inside a while block)
-dmlprogram: (blocks+=statement | functionBlocks+=functionStatement)* EOF;
-
-statement returns [ StatementInfo info ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new StatementInfo();
-} :
- // ------------------------------------------
- // ImportStatement
- 'source' '(' filePath = STRING ')' 'as' namespace=ID ';'* # ImportStatement
- | 'setwd' '(' pathValue = STRING ')' ';'* # PathStatement
- // ------------------------------------------
- // Treat function call as AssignmentStatement or MultiAssignmentStatement
- // For backward compatibility and also since the behavior of foo() * A + foo() ... where foo returns A
- // Convert FunctionCallIdentifier(paramExprs, ..) -> source
- | // TODO: Throw an informative error if user doesnot provide the optional assignment
- ( targetList+=dataIdentifier ('='|'<-') )? name=ID '(' (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? ')' ';'* # FunctionCallAssignmentStatement
- | '[' targetList+=dataIdentifier (',' targetList+=dataIdentifier)* ']' ('='|'<-') name=ID '(' (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? ')' ';'* # FunctionCallMultiAssignmentStatement
- // {notifyErrorListeners("Too many parentheses");}
- // ------------------------------------------
- // AssignmentStatement
- | targetList+=dataIdentifier op=('<-'|'=') 'ifdef' '(' commandLineParam=dataIdentifier ',' source=expression ')' ';'* # IfdefAssignmentStatement
- | targetList+=dataIdentifier op=('<-'|'=') source=expression ';'* # AssignmentStatement
- // ------------------------------------------
- // We don't support block statement
- // | '{' body+=expression ';'* ( body+=expression ';'* )* '}' # BlockStatement
- // ------------------------------------------
- // IfStatement
- | 'if' '(' predicate=expression ')' (ifBody+=statement ';'* | '{' (ifBody+=statement ';'*)* '}') ('else' (elseBody+=statement ';'* | '{' (elseBody+=statement ';'*)* '}'))? # IfStatement
- // ------------------------------------------
- // ForStatement & ParForStatement
- | 'for' '(' iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* ')' (body+=statement ';'* | '{' (body+=statement ';'* )* '}') # ForStatement
- // Convert strictParameterizedExpression to HashMap<String, String> for parForParams
- | 'parfor' '(' iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* ')' (body+=statement ';'* | '{' (body+=statement ';'*)* '}') # ParForStatement
- | 'while' '(' predicate=expression ')' (body+=statement ';'* | '{' (body+=statement ';'*)* '}') # WhileStatement
- // ------------------------------------------
-;
-
-iterablePredicate returns [ ExpressionInfo info ]
- @init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new ExpressionInfo();
- } :
- from=expression ':' to=expression #IterablePredicateColonExpression
- | ID '(' from=expression ',' to=expression ',' increment=expression ')' #IterablePredicateSeqExpression
- ;
-
-functionStatement returns [ StatementInfo info ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new StatementInfo();
-} :
- // ------------------------------------------
- // FunctionStatement & ExternalFunctionStatement
- // small change: only allow typed arguments here ... instead of data identifier
- name=ID ('<-'|'=') 'function' '(' ( inputParams+=typedArgNoAssign (',' inputParams+=typedArgNoAssign)* )? ')' ( 'return' '(' ( outputParams+=typedArgNoAssign (',' outputParams+=typedArgNoAssign)* )? ')' )? '{' (body+=statement ';'*)* '}' # InternalFunctionDefExpression
- | name=ID ('<-'|'=') 'externalFunction' '(' ( inputParams+=typedArgNoAssign (',' inputParams+=typedArgNoAssign)* )? ')' ( 'return' '(' ( outputParams+=typedArgNoAssign (',' outputParams+=typedArgNoAssign)* )? ')' )? 'implemented' 'in' '(' ( otherParams+=strictParameterizedKeyValueString (',' otherParams+=strictParameterizedKeyValueString)* )? ')' ';'* # ExternalFunctionDefExpression
- // ------------------------------------------
-;
-
-
-// Other data identifiers are typedArgNoAssign, parameterizedExpression and strictParameterizedExpression
-dataIdentifier returns [ ExpressionInfo dataInfo ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $dataInfo = new ExpressionInfo();
- // $dataInfo.expr = new org.apache.sysml.parser.DataIdentifier();
-} :
- // ------------------------------------------
- // IndexedIdentifier
- name=ID '[' (rowLower=expression (':' rowUpper=expression)?)? ',' (colLower=expression (':' colUpper=expression)?)? ']' # IndexedExpression
- // ------------------------------------------
- | ID # SimpleDataIdentifierExpression
- | COMMANDLINE_NAMED_ID # CommandlineParamExpression
- | COMMANDLINE_POSITION_ID # CommandlinePositionExpression
-;
-expression returns [ ExpressionInfo info ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new ExpressionInfo();
- // $info.expr = new org.apache.sysml.parser.BinaryExpression(org.apache.sysml.parser.Expression.BinaryOp.INVALID);
-} :
- // ------------------------------------------
- // BinaryExpression
- // power
- <assoc=right> left=expression op='^' right=expression # PowerExpression
- // unary plus and minus
- | op=('-'|'+') left=expression # UnaryExpression
- // sequence - since we are only using this into for
- //| left=expression op=':' right=expression # SequenceExpression
- // matrix multiply
- | left=expression op='%*%' right=expression # MatrixMulExpression
- // modulus and integer division
- | left=expression op=('%/%' | '%%' ) right=expression # ModIntDivExpression
- // arithmetic multiply and divide
- | left=expression op=('*'|'/') right=expression # MultDivExpression
- // arithmetic addition and subtraction
- | left=expression op=('+'|'-') right=expression # AddSubExpression
- // ------------------------------------------
- // RelationalExpression
- | left=expression op=('>'|'>='|'<'|'<='|'=='|'!=') right=expression # RelationalExpression
- // ------------------------------------------
- // BooleanExpression
- // boolean not
- | op='!' left=expression # BooleanNotExpression
- // boolean and
- | left=expression op=('&'|'&&') right=expression # BooleanAndExpression
- // boolean or
- | left=expression op=('|'|'||') right=expression # BooleanOrExpression
-
- // ---------------------------------
- // only applicable for builtin function expressions
- | name=ID '(' (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? ')' ';'* # BuiltinFunctionExpression
-
- // 4. Atomic
- | '(' left=expression ')' # AtomicExpression
-
- // Should you allow indexed expression here ?
- // | '[' targetList+=expression (',' targetList+=expression)* ']' # MultiIdExpression
-
- // | BOOLEAN # ConstBooleanIdExpression
- | 'TRUE' # ConstTrueExpression
- | 'FALSE' # ConstFalseExpression
- | INT # ConstIntIdExpression
- | DOUBLE # ConstDoubleIdExpression
- | STRING # ConstStringIdExpression
- | dataIdentifier # DataIdExpression
- // Special
- // | 'NULL' | 'NA' | 'Inf' | 'NaN'
-;
-
-typedArgNoAssign : paramType=ml_type paramName=ID;
-parameterizedExpression : (paramName=ID '=')? paramVal=expression;
-strictParameterizedExpression : paramName=ID '=' paramVal=expression ;
-strictParameterizedKeyValueString : paramName=ID '=' paramVal=STRING ;
-ID : (ALPHABET (ALPHABET|DIGIT|'_')* '::')? ALPHABET (ALPHABET|DIGIT|'_')*
- // Special ID cases:
- // | 'matrix' // --> This is a special case which causes lot of headache
- | 'as.scalar' | 'as.matrix' | 'as.double' | 'as.integer' | 'as.logical' | 'index.return' | 'lower.tail'
-;
-// Unfortunately, we have datatype name clashing with builtin function name: matrix :(
-// Therefore, ugly work around for checking datatype
-ml_type : valueType | dataType '[' valueType ']';
-// Note to reduce number of keywords, these are case-sensitive,
-// To allow case-insenstive, 'int' becomes: ('i' | 'I') ('n' | 'N') ('t' | 'T')
-valueType: 'int' | 'integer' | 'string' | 'boolean' | 'double'
- | 'Int' | 'Integer' | 'String' | 'Boolean' | 'Double';
-dataType:
- // 'scalar' # ScalarDataTypeDummyCheck
- // |
- ID # MatrixDataTypeCheck //{ if($ID.text.compareTo("matrix") != 0) { notifyErrorListeners("incorrect datatype"); } }
- //| 'matrix' //---> See ID, this causes lot of headache
- ;
-INT : DIGIT+ [Ll]?;
-// BOOLEAN : 'TRUE' | 'FALSE';
-DOUBLE: DIGIT+ '.' DIGIT* EXP? [Ll]?
-| DIGIT+ EXP? [Ll]?
-| '.' DIGIT+ EXP? [Ll]?
-;
-DIGIT: '0'..'9';
-ALPHABET : [a-zA-Z] ;
-fragment EXP : ('E' | 'e') ('+' | '-')? INT ;
-COMMANDLINE_NAMED_ID: '$' ALPHABET (ALPHABET|DIGIT|'_')*;
-COMMANDLINE_POSITION_ID: '$' DIGIT+;
-
-// supports single and double quoted string with escape characters
-STRING: '"' ( ESC | ~[\\"] )*? '"' | '\'' ( ESC | ~[\\'] )*? '\'';
-fragment ESC : '\\' [abtnfrv"'\\] ;
-// Comments, whitespaces and new line
-LINE_COMMENT : '#' .*? '\r'? '\n' -> skip ;
-MULTILINE_BLOCK_COMMENT : '/*' .*? '*/' -> skip ;
-WHITESPACE : (' ' | '\t' | '\r' | '\n')+ -> skip ;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticErrorListener.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticErrorListener.java b/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticErrorListener.java
deleted file mode 100644
index a7ed89d..0000000
--- a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticErrorListener.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.parser.antlr4;
-
-import org.antlr.v4.runtime.BaseErrorListener;
-import org.antlr.v4.runtime.RecognitionException;
-import org.antlr.v4.runtime.Recognizer;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.sysml.api.DMLScript;
-
-import java.util.Stack;
-
-public class DmlSyntacticErrorListener {
-
-
- private static final Log LOG = LogFactory.getLog(DMLScript.class.getName());
-
- public static class CustomDmlErrorListener extends BaseErrorListener {
-
- private boolean atleastOneError = false;
- private Stack<String> currentFileName = new Stack<String>();
-
- public void pushCurrentFileName(String currentFilePath) {
- currentFileName.push(currentFilePath);
- }
-
- public String peekFileName() {
- return currentFileName.peek();
- }
-
- public String popFileName() {
- return currentFileName.pop();
- }
-
- public void validationError(int line, int charPositionInLine, String msg) {
- try {
- setAtleastOneError(true);
- // Print error messages with file name
- if(currentFileName == null || currentFileName.empty()) {
- LOG.error("line "+line+":"+charPositionInLine+" "+msg);
- }
- else {
- String fileName = currentFileName.peek();
- LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg);
- }
- }
- catch(Exception e1) {
- LOG.error("ERROR: while customizing error message:" + e1);
- }
- }
-
- public void validationWarning(int line, int charPositionInLine, String msg) {
- try {
- //atleastOneError = true; ---> not an error, just warning
- // Print error messages with file name
- if(currentFileName == null || currentFileName.empty())
- LOG.warn("line "+line+":"+charPositionInLine+" "+msg);
- else {
- String fileName = currentFileName.peek();
- LOG.warn(fileName + " line "+line+":"+charPositionInLine+" "+msg);
- }
- }
- catch(Exception e1) {
- LOG.warn("ERROR: while customizing error message:" + e1);
- }
- }
-
- @Override
- public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol,
- int line, int charPositionInLine,
- String msg, RecognitionException e)
- {
- try {
- setAtleastOneError(true);
- // Print error messages with file name
- if(currentFileName == null || currentFileName.empty())
- LOG.error("line "+line+":"+charPositionInLine+" "+msg);
- else {
- String fileName = currentFileName.peek();
- LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg);
- }
- }
- catch(Exception e1) {
- LOG.error("ERROR: while customizing error message:" + e1);
- }
- }
-
- public boolean isAtleastOneError() {
- return atleastOneError;
- }
-
- public void setAtleastOneError(boolean atleastOneError) {
- this.atleastOneError = atleastOneError;
- }
- }
-}
[2/8] incubator-systemml git commit: [SYSTEMML-148] Refactored
.parser.{antlr4, python} to .parser.{dml, pydml}.
Posted by du...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
deleted file mode 100644
index e7a2b6e..0000000
--- a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidator.java
+++ /dev/null
@@ -1,2042 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.parser.python;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.antlr.v4.runtime.ParserRuleContext;
-import org.antlr.v4.runtime.Token;
-import org.antlr.v4.runtime.tree.ErrorNode;
-import org.antlr.v4.runtime.tree.TerminalNode;
-import org.apache.commons.lang.StringUtils;
-import org.apache.sysml.parser.AssignmentStatement;
-import org.apache.sysml.parser.BinaryExpression;
-import org.apache.sysml.parser.BooleanExpression;
-import org.apache.sysml.parser.BooleanIdentifier;
-import org.apache.sysml.parser.BuiltinFunctionExpression;
-import org.apache.sysml.parser.ConditionalPredicate;
-import org.apache.sysml.parser.ConstIdentifier;
-import org.apache.sysml.parser.DMLProgram;
-import org.apache.sysml.parser.DataExpression;
-import org.apache.sysml.parser.DataIdentifier;
-import org.apache.sysml.parser.DoubleIdentifier;
-import org.apache.sysml.parser.Expression;
-import org.apache.sysml.parser.Expression.DataOp;
-import org.apache.sysml.parser.Expression.DataType;
-import org.apache.sysml.parser.Expression.ValueType;
-import org.apache.sysml.parser.AParserWrapper;
-import org.apache.sysml.parser.ExternalFunctionStatement;
-import org.apache.sysml.parser.ForStatement;
-import org.apache.sysml.parser.FunctionCallIdentifier;
-import org.apache.sysml.parser.FunctionStatement;
-import org.apache.sysml.parser.IfStatement;
-import org.apache.sysml.parser.ImportStatement;
-import org.apache.sysml.parser.IndexedIdentifier;
-import org.apache.sysml.parser.IntIdentifier;
-import org.apache.sysml.parser.IterablePredicate;
-import org.apache.sysml.parser.LanguageException;
-import org.apache.sysml.parser.MultiAssignmentStatement;
-import org.apache.sysml.parser.OutputStatement;
-import org.apache.sysml.parser.ParForStatement;
-import org.apache.sysml.parser.ParameterExpression;
-import org.apache.sysml.parser.ParameterizedBuiltinFunctionExpression;
-import org.apache.sysml.parser.ParseException;
-import org.apache.sysml.parser.PathStatement;
-import org.apache.sysml.parser.PrintStatement;
-import org.apache.sysml.parser.RelationalExpression;
-import org.apache.sysml.parser.Statement;
-import org.apache.sysml.parser.StatementBlock;
-import org.apache.sysml.parser.StringIdentifier;
-import org.apache.sysml.parser.WhileStatement;
-//import org.apache.sysml.parser.antlr4.ExpressionInfo;
-//import org.apache.sysml.parser.antlr4.StatementInfo;
-import org.apache.sysml.parser.python.PydmlParser.AddSubExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.AssignmentStatementContext;
-import org.apache.sysml.parser.python.PydmlParser.AtomicExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.BooleanAndExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.BooleanNotExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.BooleanOrExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.BuiltinFunctionExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.CommandlineParamExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.CommandlinePositionExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.ConstDoubleIdExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.ConstFalseExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.ConstIntIdExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.ConstStringIdExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.ConstTrueExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.DataIdExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.DataIdentifierContext;
-import org.apache.sysml.parser.python.PydmlParser.ExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.ExternalFunctionDefExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.ForStatementContext;
-import org.apache.sysml.parser.python.PydmlParser.FunctionCallAssignmentStatementContext;
-import org.apache.sysml.parser.python.PydmlParser.FunctionCallMultiAssignmentStatementContext;
-import org.apache.sysml.parser.python.PydmlParser.FunctionStatementContext;
-import org.apache.sysml.parser.python.PydmlParser.IfStatementContext;
-import org.apache.sysml.parser.python.PydmlParser.IfdefAssignmentStatementContext;
-import org.apache.sysml.parser.python.PydmlParser.IgnoreNewLineContext;
-import org.apache.sysml.parser.python.PydmlParser.ImportStatementContext;
-import org.apache.sysml.parser.python.PydmlParser.IndexedExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.InternalFunctionDefExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.IterablePredicateColonExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.IterablePredicateSeqExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.MatrixDataTypeCheckContext;
-import org.apache.sysml.parser.python.PydmlParser.Ml_typeContext;
-import org.apache.sysml.parser.python.PydmlParser.ModIntDivExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.MultDivExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.ParForStatementContext;
-import org.apache.sysml.parser.python.PydmlParser.ParameterizedExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.PathStatementContext;
-import org.apache.sysml.parser.python.PydmlParser.PmlprogramContext;
-import org.apache.sysml.parser.python.PydmlParser.PowerExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.RelationalExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.SimpleDataIdentifierExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.StatementContext;
-import org.apache.sysml.parser.python.PydmlParser.StrictParameterizedExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.StrictParameterizedKeyValueStringContext;
-import org.apache.sysml.parser.python.PydmlParser.TypedArgNoAssignContext;
-import org.apache.sysml.parser.python.PydmlParser.UnaryExpressionContext;
-import org.apache.sysml.parser.python.PydmlParser.ValueDataTypeCheckContext;
-import org.apache.sysml.parser.python.PydmlParser.WhileStatementContext;
-
-/**
- * TODO: Refactor duplicated parser code dml/pydml (entire package).
- *
- */
-public class PydmlSyntacticValidator implements PydmlListener
-{
- private PydmlSyntacticValidatorHelper helper = null;
-
- private String _workingDir = "."; //current working directory
- private String _currentPath = null; //current file path
- private HashMap<String,String> argVals = null;
-
- public PydmlSyntacticValidator(PydmlSyntacticValidatorHelper helper, String currentPath, HashMap<String,String> argVals) {
- this.helper = helper;
- this.argVals = argVals;
-
- _currentPath = currentPath;
- }
-
- // Functions we have to implement but don't really need it
- @Override
- public void enterAddSubExpression(AddSubExpressionContext ctx) { }
- @Override
- public void enterAssignmentStatement(AssignmentStatementContext ctx) {}
- @Override
- public void enterAtomicExpression(AtomicExpressionContext ctx) { }
- @Override
- public void enterBooleanAndExpression(BooleanAndExpressionContext ctx) { }
- @Override
- public void enterBooleanNotExpression(BooleanNotExpressionContext ctx) { }
- @Override
- public void enterBooleanOrExpression(BooleanOrExpressionContext ctx) { }
- @Override
- public void enterCommandlineParamExpression(CommandlineParamExpressionContext ctx) { }
- @Override
- public void enterCommandlinePositionExpression(CommandlinePositionExpressionContext ctx) { }
- @Override
- public void enterConstDoubleIdExpression(ConstDoubleIdExpressionContext ctx) { }
- @Override
- public void enterConstIntIdExpression(ConstIntIdExpressionContext ctx) { }
- @Override
- public void enterConstStringIdExpression(ConstStringIdExpressionContext ctx) { }
- @Override
- public void enterDataIdExpression(DataIdExpressionContext ctx) { }
-
- @Override
- public void enterIgnoreNewLine(IgnoreNewLineContext ctx) { }
- @Override
- public void enterPmlprogram(PmlprogramContext ctx) { }
- @Override
- public void exitPmlprogram(PmlprogramContext ctx) { }
-
- @Override
- public void enterEveryRule(ParserRuleContext arg0) {
- if(arg0 instanceof StatementContext) {
- if(((StatementContext) arg0).info == null) {
- ((StatementContext) arg0).info = new StatementInfo();
- }
- }
- if(arg0 instanceof FunctionStatementContext) {
- if(((FunctionStatementContext) arg0).info == null) {
- ((FunctionStatementContext) arg0).info = new StatementInfo();
- }
- }
- if(arg0 instanceof ExpressionContext) {
- if(((ExpressionContext) arg0).info == null) {
- ((ExpressionContext) arg0).info = new ExpressionInfo();
- }
- }
- if(arg0 instanceof DataIdentifierContext) {
- if(((DataIdentifierContext) arg0).dataInfo == null) {
- ((DataIdentifierContext) arg0).dataInfo = new ExpressionInfo();
- }
- }
- }
- @Override
- public void enterExternalFunctionDefExpression(ExternalFunctionDefExpressionContext ctx) { }
- @Override
- public void enterForStatement(ForStatementContext ctx) {}
- @Override
- public void enterFunctionCallAssignmentStatement(FunctionCallAssignmentStatementContext ctx) { }
- @Override
- public void enterFunctionCallMultiAssignmentStatement(FunctionCallMultiAssignmentStatementContext ctx) { }
- @Override
- public void enterIfStatement(IfStatementContext ctx) { }
- @Override
- public void enterImportStatement(ImportStatementContext ctx) { }
- @Override
- public void enterIndexedExpression(IndexedExpressionContext ctx) { }
- @Override
- public void enterInternalFunctionDefExpression(InternalFunctionDefExpressionContext ctx) { }
- @Override
- public void enterMl_type(Ml_typeContext ctx) { }
- @Override
- public void enterModIntDivExpression(ModIntDivExpressionContext ctx) { }
- @Override
- public void enterMultDivExpression(MultDivExpressionContext ctx) { }
- @Override
- public void enterParameterizedExpression(ParameterizedExpressionContext ctx) { }
- @Override
- public void enterParForStatement(ParForStatementContext ctx) { }
- @Override
- public void enterPathStatement(PathStatementContext ctx) { }
- @Override
- public void enterPowerExpression(PowerExpressionContext ctx) { }
- @Override
- public void enterRelationalExpression(RelationalExpressionContext ctx) { }
- @Override
- public void enterSimpleDataIdentifierExpression(SimpleDataIdentifierExpressionContext ctx) { }
- @Override
- public void enterStrictParameterizedExpression(StrictParameterizedExpressionContext ctx) { }
- @Override
- public void enterTypedArgNoAssign(TypedArgNoAssignContext ctx) { }
- @Override
- public void enterUnaryExpression(UnaryExpressionContext ctx) { }
- @Override
- public void enterWhileStatement(WhileStatementContext ctx) { }
-
- @Override
- public void visitErrorNode(ErrorNode arg0) { }
- @Override
- public void visitTerminal(TerminalNode arg0) { }
- @Override
- public void exitEveryRule(ParserRuleContext arg0) {}
- // --------------------------------------------------------------------
- private void setFileLineColumn(Expression expr, ParserRuleContext ctx) {
- // expr.setFilename(helper.getCurrentFileName());
- String txt = ctx.getText();
- expr.setFilename(_currentPath);
- expr.setBeginLine(ctx.start.getLine());
- expr.setBeginColumn(ctx.start.getCharPositionInLine());
- expr.setEndLine(ctx.stop.getLine());
- expr.setEndColumn(ctx.stop.getCharPositionInLine());
- if(expr.getBeginColumn() == expr.getEndColumn() && expr.getBeginLine() == expr.getEndLine() && txt.length() > 1) {
- expr.setEndColumn(expr.getBeginColumn() + txt.length() - 1);
- }
- }
-
- private void setFileLineColumn(Statement stmt, ParserRuleContext ctx) {
- String txt = ctx.getText();
- stmt.setFilename(helper.getCurrentFileName());
- stmt.setBeginLine(ctx.start.getLine());
- stmt.setBeginColumn(ctx.start.getCharPositionInLine());
- stmt.setEndLine(ctx.stop.getLine());
- stmt.setEndColumn(ctx.stop.getCharPositionInLine());
- if(stmt.getBeginColumn() == stmt.getEndColumn() && stmt.getBeginLine() == stmt.getEndLine() && txt.length() > 1) {
- stmt.setEndColumn(stmt.getBeginColumn() + txt.length() - 1);
- }
- }
-
- // For now do no type checking, let validation handle it.
- // This way parser doesn't have to open metadata file
- @Override
- public void exitAddSubExpression(AddSubExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
- // Addition and subtraction operator same as DML
- Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
-
-
- @Override
- public void exitModIntDivExpression(ModIntDivExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
- String dmlOperator = "";
- if(ctx.op.getText().compareTo("//") == 0) {
- dmlOperator = "%/%";
- }
- else if(ctx.op.getText().compareTo("%") == 0) {
- dmlOperator = "%%";
- }
- else {
- helper.notifyErrorListeners("Incorrect operator (expected // or %)", ctx.op);
- return;
- }
- Expression.BinaryOp bop = Expression.getBinaryOp(dmlOperator);
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- @Override
- public void exitUnaryExpression(UnaryExpressionContext ctx) {
- if(ctx.left.info.expr != null) {
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- if(ctx.left.info.expr instanceof IntIdentifier) {
- if(ctx.op.getText().compareTo("-") == 0) {
- ((IntIdentifier) ctx.left.info.expr).multiplyByMinusOne();
- }
- ctx.info.expr = ctx.left.info.expr;
- }
- else if(ctx.left.info.expr instanceof DoubleIdentifier) {
- if(ctx.op.getText().compareTo("-") == 0) {
- ((DoubleIdentifier) ctx.left.info.expr).multiplyByMinusOne();
- }
- ctx.info.expr = ctx.left.info.expr;
- }
- else {
- Expression right = new IntIdentifier(1, fileName, line, col, line, col);
- if(ctx.op.getText().compareTo("-") == 0) {
- right = new IntIdentifier(-1, fileName, line, col, line, col);
- }
-
- Expression.BinaryOp bop = Expression.getBinaryOp("*");
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(right);
- }
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- @Override
- public void exitMultDivExpression(MultDivExpressionContext ctx) {
- Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
- @Override
- public void exitPowerExpression(PowerExpressionContext ctx) {
- String dmlOperator = "";
- if(ctx.op.getText().compareTo("**") == 0) {
- dmlOperator = "^";
- }
- else {
- helper.notifyErrorListeners("Incorrect operator (expected **)", ctx.op);
- return;
- }
-
- Expression.BinaryOp bop = Expression.getBinaryOp(dmlOperator);
- ctx.info.expr = new BinaryExpression(bop);
- ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
- // TODO:
-// @Override
-// public void exitMatrixMulExpression(MatrixMulExpressionContext ctx) {
-// Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
-// ctx.info.expr = new BinaryExpression(bop);
-// ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
-// ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
-// setFileLineColumn(ctx.info.expr, ctx);
-// }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitRelationalExpression(RelationalExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
- Expression.RelationalOp rop = Expression.getRelationalOp(ctx.op.getText());
- ctx.info.expr = new RelationalExpression(rop);
- ((RelationalExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((RelationalExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitBooleanAndExpression(BooleanAndExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
- String dmlOperator = "";
- if(ctx.op.getText().compareTo("&") == 0 || ctx.op.getText().compareTo("and") == 0) {
- dmlOperator = "&";
- }
- else {
- helper.notifyErrorListeners("Incorrect operator (expected &)", ctx.op);
- return;
- }
-
- Expression.BooleanOp bop = Expression.getBooleanOp(dmlOperator);
- ctx.info.expr = new BooleanExpression(bop);
- ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BooleanExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- @Override
- public void exitBooleanOrExpression(BooleanOrExpressionContext ctx) {
- if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
- String dmlOperator = "";
- if(ctx.op.getText().compareTo("|") == 0 || ctx.op.getText().compareTo("or") == 0) {
- dmlOperator = "|";
- }
- else {
- helper.notifyErrorListeners("Incorrect operator (expected |)", ctx.op);
- return;
- }
-
- Expression.BooleanOp bop = Expression.getBooleanOp(dmlOperator);
- ctx.info.expr = new BooleanExpression(bop);
- ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- ((BooleanExpression)ctx.info.expr).setRight(ctx.right.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- @Override
- public void exitBooleanNotExpression(BooleanNotExpressionContext ctx) {
- if(ctx.left.info.expr != null) {
- Expression.BooleanOp bop = Expression.getBooleanOp(ctx.op.getText());
- ctx.info.expr = new BooleanExpression(bop);
- ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitAtomicExpression(AtomicExpressionContext ctx) {
- ctx.info.expr = ctx.left.info.expr;
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
- @Override
- public void exitConstDoubleIdExpression(ConstDoubleIdExpressionContext ctx) {
- try {
- double val = Double.parseDouble(ctx.getText());
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new DoubleIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- catch(Exception e) {
- helper.notifyErrorListeners("cannot parse the float value: \'" + ctx.getText() + "\'", ctx.getStart());
- return;
- }
- }
-
- @Override
- public void exitConstIntIdExpression(ConstIntIdExpressionContext ctx) {
- try {
- long val = Long.parseLong(ctx.getText());
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new IntIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
- catch(Exception e) {
- helper.notifyErrorListeners("cannot parse the int value: \'" + ctx.getText() + "\'", ctx.getStart());
- return;
- }
- }
-
- @Override
- public void exitConstStringIdExpression(ConstStringIdExpressionContext ctx) {
- String val = "";
- String text = ctx.getText();
- if( (text.startsWith("\"") && text.endsWith("\"")) ||
- (text.startsWith("\'") && text.endsWith("\'"))) {
- if(text.length() > 2) {
- val = text.substring(1, text.length()-1);
- }
- }
- else {
- helper.notifyErrorListeners("something wrong while parsing string ... strange", ctx.start);
- return;
- }
-
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new StringIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitDataIdExpression(DataIdExpressionContext ctx) {
- ctx.info.expr = ctx.dataIdentifier().dataInfo.expr;
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- ctx.info.expr.setAllPositions(helper.getCurrentFileName(), line, col, line, col);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
- @Override
- public void exitSimpleDataIdentifierExpression(SimpleDataIdentifierExpressionContext ctx) {
- // This is either a function, or variable with namespace
- // By default, it assigns to a data type
- ctx.dataInfo.expr = new DataIdentifier(ctx.getText());
- setFileLineColumn(ctx.dataInfo.expr, ctx);
- }
-
-
- private Expression incrementByOne(Expression expr, ParserRuleContext ctx) {
- // For maintaining semantic consistency, we have decided to keep 1-based indexing
- // If in future, PyDML becomes more popular than DML, this can be switched.
- return expr;
- }
-
- @Override
- public void exitIndexedExpression(IndexedExpressionContext ctx) {
- ctx.dataInfo.expr = new IndexedIdentifier(ctx.name.getText(), false, false);
- setFileLineColumn(ctx.dataInfo.expr, ctx);
- try {
- ArrayList< ArrayList<Expression> > exprList = new ArrayList< ArrayList<Expression> >();
-
- ArrayList<Expression> rowIndices = new ArrayList<Expression>();
- ArrayList<Expression> colIndices = new ArrayList<Expression>();
-
- boolean isRowLower = (ctx.rowLower != null && !ctx.rowLower.isEmpty() && (ctx.rowLower.info.expr != null));
- boolean isRowUpper = (ctx.rowUpper != null && !ctx.rowUpper.isEmpty() && (ctx.rowUpper.info.expr != null));
- boolean isColLower = (ctx.colLower != null && !ctx.colLower.isEmpty() && (ctx.colLower.info.expr != null));
- boolean isColUpper = (ctx.colUpper != null && !ctx.colUpper.isEmpty() && (ctx.colUpper.info.expr != null));
-
- if(!isRowLower && !isRowUpper) {
- // both not set
- rowIndices.add(null); rowIndices.add(null);
- }
- else if(isRowLower && isRowUpper) {
- // both set
- rowIndices.add(incrementByOne(ctx.rowLower.info.expr, ctx));
- rowIndices.add(ctx.rowUpper.info.expr);
- }
- else if(isRowLower && !isRowUpper) {
- // only row set
- rowIndices.add(incrementByOne(ctx.rowLower.info.expr, ctx));
- }
- else {
- helper.notifyErrorListeners("incorrect index expression for row", ctx.start);
- return;
- }
-
- if(!isColLower && !isColUpper) {
- // both not set
- colIndices.add(null); colIndices.add(null);
- }
- else if(isColLower && isColUpper) {
- colIndices.add(incrementByOne(ctx.colLower.info.expr, ctx));
- colIndices.add(ctx.colUpper.info.expr);
- }
- else if(isColLower && !isColUpper) {
- colIndices.add(incrementByOne(ctx.colLower.info.expr, ctx));
- }
- else {
- helper.notifyErrorListeners("incorrect index expression for column", ctx.start);
- return;
- }
- exprList.add(rowIndices);
- exprList.add(colIndices);
- ((IndexedIdentifier) ctx.dataInfo.expr).setIndices(exprList);
- }
- catch(Exception e) {
- helper.notifyErrorListeners("cannot set the indices", ctx.start);
- return;
- }
- }
-
- private ConstIdentifier getConstIdFromString(String varValue, Token start) {
- // Both varName and varValue are correct
- int linePosition = start.getLine();
- int charPosition = start.getCharPositionInLine();
- try {
- long val = Long.parseLong(varValue);
- return new IntIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- }
- catch(Exception e) {
- try {
- double val = Double.parseDouble(varValue);
- return new DoubleIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- }
- catch(Exception e1) {
- try {
- if(varValue.compareTo("True") == 0 || varValue.compareTo("False") == 0) {
- boolean val = false;
- if(varValue.compareTo("True") == 0) {
- val = true;
- }
- return new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- }
- else {
- String val = "";
- String text = varValue;
- if( (text.startsWith("\"") && text.endsWith("\"")) ||
- (text.startsWith("\'") && text.endsWith("\'"))) {
- if(text.length() > 2) {
- val = text.substring(1, text.length()-1);
- }
- }
- else {
- // the commandline parameters can be passed without any quotes
- val = text;
- }
- return new StringIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- }
- }
- catch(Exception e3) {
- helper.notifyErrorListeners("unable to cast the commandline parameter into int/float/bool/str", start);
- return null;
- }
- }
- }
- }
-
- private void fillExpressionInfoCommandLineParameters(String varName, ExpressionInfo dataInfo, Token start) {
-
- if(!varName.startsWith("$")) {
- helper.notifyErrorListeners("commandline param doesnot start with $ ... strange", start);
- return;
- }
-
- String varValue = null;
- for(Map.Entry<String, String> arg : this.argVals.entrySet()) {
- if(arg.getKey().trim().compareTo(varName) == 0) {
- if(varValue != null) {
- helper.notifyErrorListeners("multiple values passed for the parameter " + varName + " via commandline", start);
- return;
- }
- else {
- varValue = arg.getValue().trim();
- }
- }
- }
-
- if(varValue == null) {
- // helper.notifyErrorListeners("the parameter " + varName + " either needs to be passed through commandline or initialized to default value", start);
- return;
- }
-
- // Command line param cannot be empty string
- // If you want to pass space, please quote it
- if(varValue.trim().compareTo("") == 0)
- return;
-
- dataInfo.expr = getConstIdFromString(varValue, start);
- }
-
- @Override
- public void exitCommandlineParamExpression(CommandlineParamExpressionContext ctx) {
- handleCommandlineArgumentExpression(ctx);
- }
-
- @Override
- public void exitCommandlinePositionExpression(CommandlinePositionExpressionContext ctx) {
- handleCommandlineArgumentExpression(ctx);
- }
-
- /**
- *
- * @param ctx
- */
- private void handleCommandlineArgumentExpression(DataIdentifierContext ctx)
- {
- String varName = ctx.getText().trim();
- fillExpressionInfoCommandLineParameters(varName, ctx.dataInfo, ctx.start);
-
- if(ctx.dataInfo.expr == null) {
- if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) {
- String msg = "The parameter " + varName + " either needs to be passed "
- + "through commandline or initialized to default value.";
- if( AParserWrapper.IGNORE_UNSPECIFIED_ARGS ) {
- ctx.dataInfo.expr = getConstIdFromString(" ", ctx.start);
- helper.raiseWarning(msg, ctx.start);
- }
- else {
- helper.notifyErrorListeners(msg, ctx.start);
- }
- }
- }
- }
-
- // --------------------------------------------------------------------
-
- @Override
- public void exitImportStatement(ImportStatementContext ctx)
- {
- //prepare import filepath
- String filePath = ctx.filePath.getText();
- String namespace = DMLProgram.DEFAULT_NAMESPACE;
- if(ctx.namespace != null && ctx.namespace.getText() != null && !ctx.namespace.getText().isEmpty()) {
- namespace = ctx.namespace.getText();
- }
- if((filePath.startsWith("\"") && filePath.endsWith("\"")) ||
- filePath.startsWith("'") && filePath.endsWith("'")) {
- filePath = filePath.substring(1, filePath.length()-1);
- }
-
- //concatenate working directory to filepath
- filePath = _workingDir + File.separator + filePath;
-
- DMLProgram prog = null;
- try {
- prog = (new PyDMLParserWrapper()).doParse(filePath, null, argVals);
- } catch (ParseException e) {
- helper.notifyErrorListeners("Exception found during importing a program from file " + filePath, ctx.start);
- return;
- }
- // Custom logic whether to proceed ahead or not. Better than the current exception handling mechanism
- if(prog == null) {
- helper.notifyErrorListeners("One or more errors found during importing a program from file " + filePath, ctx.start);
- return;
- }
- else {
- ctx.info.namespaces = new HashMap<String, DMLProgram>();
- ctx.info.namespaces.put(namespace, prog);
- ctx.info.stmt = new ImportStatement();
- ((ImportStatement) ctx.info.stmt).setCompletePath(filePath);
- ((ImportStatement) ctx.info.stmt).setFilePath(ctx.filePath.getText());
- ((ImportStatement) ctx.info.stmt).setNamespace(namespace);
- }
- }
-
- @Override
- public void exitAssignmentStatement(AssignmentStatementContext ctx) {
- if(ctx.targetList == null || ctx.targetList.size() != 1) {
- helper.notifyErrorListeners("incorrect parsing for assignment", ctx.start);
- return;
- }
- String targetListText = ctx.targetList.get(0).getText();
- if(targetListText.startsWith("$")) {
- helper.notifyErrorListeners("assignment of commandline parameters is not allowed. (Quickfix: try using someLocalVariable=ifdef(" + targetListText + ", default value))", ctx.start);
- return;
- }
-
- DataIdentifier target = null;
- if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
- target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
- Expression source = ctx.source.info.expr;
-
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- try {
- ctx.info.stmt = new AssignmentStatement(target, source, line, col, line, col);
- setFileLineColumn(ctx.info.stmt, ctx);
- } catch (LanguageException e) {
- // TODO: extract more meaningful info from this exception.
- helper.notifyErrorListeners("invalid assignment", ctx.targetList.get(0).start);
- return;
- }
- }
- else {
- helper.notifyErrorListeners("incorrect lvalue ... strange", ctx.targetList.get(0).start);
- return;
- }
-
- }
-
-
- private void setAssignmentStatement(DataIdentifier target, Expression expression, StatementContext ctx) {
- try {
- ctx.info.stmt = new AssignmentStatement(target, expression, ctx.start.getLine(), ctx.start.getCharPositionInLine(), ctx.start.getLine(), ctx.start.getCharPositionInLine());
- setFileLineColumn(ctx.info.stmt, ctx);
- } catch (LanguageException e) {
- // TODO: extract more meaningful info from this exception.
- helper.notifyErrorListeners("invalid function call", ctx.start);
- return;
- }
- }
-
- private void setPrintStatement(FunctionCallAssignmentStatementContext ctx, String functionName) {
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners(functionName + "() has only one parameter", ctx.start);
- return;
- }
- Expression expr = paramExpression.get(0).getExpr();
- if(expr == null) {
- helper.notifyErrorListeners("cannot process " + functionName + "() function", ctx.start);
- return;
- }
- try {
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- ctx.info.stmt = new PrintStatement(functionName, expr, line, col, line, col);
- } catch (LanguageException e) {
- helper.notifyErrorListeners("cannot process " + functionName + "() function", ctx.start);
- return;
- }
- }
-
- private void setOutputStatement(FunctionCallAssignmentStatementContext ctx) {
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
- if(paramExpression.size() < 2){
- helper.notifyErrorListeners("incorrect usage of load function (atleast 2 arguments required)", ctx.start);
- return;
- }
- if(paramExpression.get(0).getExpr() instanceof DataIdentifier) {
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- HashMap<String, Expression> varParams = new HashMap<String, Expression>();
- varParams.put(DataExpression.IO_FILENAME, paramExpression.get(1).getExpr());
- for(int i = 2; i < paramExpression.size(); i++) {
- // DataExpression.FORMAT_TYPE, DataExpression.DELIM_DELIMITER, DataExpression.DELIM_HAS_HEADER_ROW, DataExpression.DELIM_SPARSE
- varParams.put(paramExpression.get(i).getName(), paramExpression.get(i).getExpr());
- }
-
- DataExpression dataExpression = new DataExpression(DataOp.WRITE, varParams, fileName, line, col, line, col);
- ctx.info.stmt = new OutputStatement((DataIdentifier) paramExpression.get(0).getExpr(), DataOp.WRITE, fileName, line, col, line, col);
- setFileLineColumn(ctx.info.stmt, ctx);
- ((OutputStatement)ctx.info.stmt).setExprParams(dataExpression);
- return;
- }
-
- helper.notifyErrorListeners("incorrect usage of write function", ctx.start);
- return;
-
- }
-
- private boolean inDefaultNamespace(String namespace) {
- return namespace.compareTo(DMLProgram.DEFAULT_NAMESPACE) == 0;
- }
-
- // Returns 0, 1 or -1 (for error)
- private int getAxis(ParameterizedExpressionContext ctx) {
- if(ctx.paramName != null && ctx.paramName.getText() != null && !ctx.paramName.getText().isEmpty()) {
- if(ctx.paramName.getText().compareTo("axis") != 0) {
- return -1;
- }
- }
-
- String val = ctx.paramVal.getText();
- if(val != null && val.compareTo("0") == 0) {
- return 0;
- }
- else if(val != null && val.compareTo("1") == 0) {
- return 1;
- }
-
- return -1;
- }
-
- private String getPythonAggFunctionNames(String functionName, int axis) {
- if(axis != 0 && axis != 1) {
- return functionName;
- }
- // axis=0 maps to column-wise computation and axis=1 maps to row-wise computation
-
- if(functionName.compareTo("sum") == 0) {
- return axis == 0 ? "colSums" : "rowSums";
- }
- else if(functionName.compareTo("mean") == 0) {
- return axis == 0 ? "colMeans" : "rowMeans";
- }
- else if(functionName.compareTo("avg") == 0) {
- return axis == 0 ? "colMeans" : "rowMeans";
- }
- else if(functionName.compareTo("max") == 0) {
- return axis == 0 ? "colMaxs" : "rowMaxs";
- }
- else if(functionName.compareTo("min") == 0) {
- return axis == 0 ? "colMins" : "rowMins";
- }
- else if(functionName.compareTo("argmin") == 0) {
- return axis == 0 ? "Not Supported" : "rowIndexMin";
- }
- else if(functionName.compareTo("argmax") == 0) {
- return axis == 0 ? "Not Supported" : "rowIndexMax";
- }
- else if(functionName.compareTo("cumsum") == 0) {
- return axis == 0 ? "cumsum" : "Not Supported";
- }
- else if(functionName.compareTo("transpose") == 0) {
- return axis == 0 ? "Not Supported" : "Not Supported";
- }
- else if(functionName.compareTo("trace") == 0) {
- return axis == 0 ? "Not Supported" : "Not Supported";
- }
- else {
- return functionName;
- }
- }
-
- private ConvertedDMLSyntax convertPythonBuiltinFunctionToDMLSyntax(String namespace, String functionName, ArrayList<ParameterExpression> paramExpression,
- List<ParameterizedExpressionContext> paramCtx, Token fnName, String fileName, int line, int col) {
- // ===========================================================================================
- // Check function name, namespace, parameters (#params & possible values) and throw useful hints
- if(inDefaultNamespace(namespace) && functionName.compareTo("len") == 0) {
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts 1 arguments", fnName);
- return null;
- }
- functionName = "length";
- }
- else if(functionName.compareTo("sum") == 0 || functionName.compareTo("mean") == 0 || functionName.compareTo("avg") == 0 ||
- functionName.compareTo("min") == 0 || functionName.compareTo("max") == 0 ||
- functionName.compareTo("argmax") == 0 || functionName.compareTo("argmin") == 0 ||
- functionName.compareTo("cumsum") == 0 || functionName.compareTo("transpose") == 0 || functionName.compareTo("trace") == 0) {
- // 0 maps row-wise computation and 1 maps to column-wise computation
-
- // can mean sum of all cells or row-wise or columnwise sum
- if(inDefaultNamespace(namespace) && paramExpression.size() == 1) {
- // sum(x) => sum(x)
- // otherwise same function name
- if(functionName.compareTo("avg") == 0) {
- functionName = "mean";
- }
- else if(functionName.compareTo("transpose") == 0) {
- functionName = "t";
- }
- else if(functionName.compareTo("argmax") == 0 || functionName.compareTo("argmin") == 0 || functionName.compareTo("cumsum") == 0) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for entire matrix is not supported", fnName);
- return null;
- }
- }
- else if(!inDefaultNamespace(namespace) && paramExpression.size() == 0) {
- // x.sum() => sum(x)
- paramExpression = new ArrayList<ParameterExpression>();
- paramExpression.add(new ParameterExpression(null, new DataIdentifier(namespace)));
- // otherwise same function name
- if(functionName.compareTo("avg") == 0) {
- functionName = "mean";
- }
- else if(functionName.compareTo("transpose") == 0) {
- functionName = "t";
- }
- else if(functionName.compareTo("argmax") == 0 || functionName.compareTo("argmin") == 0 || functionName.compareTo("cumsum") == 0) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for entire matrix is not supported", fnName);
- return null;
- }
- }
- else if(inDefaultNamespace(namespace) && paramExpression.size() == 2) {
- // sum(x, axis=1) => rowSums(x)
- int axis = getAxis(paramCtx.get(1));
- if(axis == -1 && (functionName.compareTo("min") == 0 || functionName.compareTo("max") == 0 )) {
- // Do nothing
- // min(2, 3)
- }
- else if(axis == -1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for given arguments is not supported", fnName);
- return null;
- }
- else {
- ArrayList<ParameterExpression> temp = new ArrayList<ParameterExpression>();
- temp.add(paramExpression.get(0));
- paramExpression = temp;
- functionName = getPythonAggFunctionNames(functionName, axis);
- if(functionName.compareTo("Not Supported") == 0) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for given arguments is not supported", fnName);
- return null;
- }
- }
- }
- else if(!inDefaultNamespace(namespace) && paramExpression.size() == 1) {
- // x.sum(axis=1) => rowSums(x)
- int axis = getAxis(paramCtx.get(0));
- if(axis == -1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for given arguments is not supported", fnName);
- return null;
- }
- else {
- paramExpression = new ArrayList<ParameterExpression>();
- paramExpression.add(new ParameterExpression(null, new DataIdentifier(namespace)));
- functionName = getPythonAggFunctionNames(functionName, axis);
- if(functionName.compareTo("Not Supported") == 0) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for given arguments is not supported", fnName);
- return null;
- }
- }
- }
- else {
- helper.notifyErrorListeners("Incorrect number of arguments for the builtin function \'" + functionName + "\'.", fnName);
- return null;
- }
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("concatenate") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts 2 arguments (Note: concatenate append columns of two matrices)", fnName);
- return null;
- }
- functionName = "append";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("minimum") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts 2 arguments", fnName);
- return null;
- }
- functionName = "min";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("maximum") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts 2 arguments", fnName);
- return null;
- }
- functionName = "max";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(!inDefaultNamespace(namespace) && functionName.compareTo("shape") == 0) {
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts only 1 argument (0 or 1)", fnName);
- return null;
- }
-
- int axis = getAxis(paramCtx.get(0));
- if(axis == -1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts only 1 argument (0 or 1)", fnName);
- return null;
- }
- paramExpression = new ArrayList<ParameterExpression>();
- paramExpression.add(new ParameterExpression(null, new DataIdentifier(namespace)));
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- if(axis == 0) {
- functionName = "nrow";
- }
- else if(axis == 1) {
- functionName = "ncol";
- }
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("random.normal") == 0) {
- if(paramExpression.size() != 3) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (number of rows, number of columns, sparsity)", fnName);
- return null;
- }
- paramExpression.get(0).setName("rows");
- paramExpression.get(1).setName("cols");
- paramExpression.get(2).setName("sparsity");
- paramExpression.add(new org.apache.sysml.parser.ParameterExpression("pdf", new StringIdentifier("normal", fileName, line, col, line, col)));
- functionName = "rand";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("random.uniform") == 0) {
- if(paramExpression.size() != 5) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 5 arguments (number of rows, number of columns, sparsity, min, max)", fnName);
- return null;
- }
- paramExpression.get(0).setName("rows");
- paramExpression.get(1).setName("cols");
- paramExpression.get(2).setName("sparsity");
- paramExpression.get(3).setName("min");
- paramExpression.get(4).setName("max");
- paramExpression.add(new org.apache.sysml.parser.ParameterExpression("pdf", new StringIdentifier("uniform", fileName, line, col, line, col)));
- functionName = "rand";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("full") == 0) {
- if(paramExpression.size() != 3) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (constant float value, number of rows, number of columns)", fnName);
- return null;
- }
- paramExpression.get(1).setName("rows");
- paramExpression.get(2).setName("cols");
- functionName = "matrix";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("matrix") == 0) {
- // This can either be string initializer or as.matrix function
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 1 argument (either str or float value)", fnName);
- return null;
- }
-
- if(paramExpression.get(0).getExpr() instanceof StringIdentifier) {
- String initializerString = ((StringIdentifier)paramExpression.get(0).getExpr()).getValue().trim();
- if(!initializerString.startsWith("[") || !initializerString.endsWith("]")) {
- helper.notifyErrorListeners("Incorrect initializer string for builtin function \'" + functionName + "\' (Eg: matrix(\"[1 2 3; 4 5 6]\"))", fnName);
- return null;
- }
- int rows = StringUtils.countMatches(initializerString, ";") + 1;
-
- // Make sure user doesnot have pretty string
- initializerString = initializerString.replaceAll("; ", ";");
- initializerString = initializerString.replaceAll(" ;", ";");
- initializerString = initializerString.replaceAll("\\[ ", "\\[");
- initializerString = initializerString.replaceAll(" \\]", "\\]");
-
- // Each row has ncol-1 spaces
- // #spaces = nrow * (ncol-1)
- // ncol = (#spaces / nrow) + 1
- int cols = (StringUtils.countMatches(initializerString, " ") / rows) + 1;
-
- initializerString = initializerString.replaceAll(";", " ");
- initializerString = initializerString.replaceAll("\\[", "");
- initializerString = initializerString.replaceAll("\\]", "");
- paramExpression = new ArrayList<ParameterExpression>();
- paramExpression.add(new ParameterExpression(null, new StringIdentifier(initializerString, fileName, line, col, line, col)));
- paramExpression.add(new ParameterExpression("rows", new IntIdentifier(rows, fileName, line, col, line, col)));
- paramExpression.add(new ParameterExpression("cols", new IntIdentifier(cols, fileName, line, col, line, col)));
- }
- else {
- functionName = "as.matrix";
- }
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("scalar") == 0) {
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 1 argument", fnName);
- return null;
- }
- functionName = "as.scalar";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("float") == 0) {
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 1 argument", fnName);
- return null;
- }
- functionName = "as.double";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("int") == 0) {
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 1 argument", fnName);
- return null;
- }
- functionName = "as.integer";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("bool") == 0) {
- if(paramExpression.size() != 1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 1 argument", fnName);
- return null;
- }
- functionName = "as.logical";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(!inDefaultNamespace(namespace) && functionName.compareTo("reshape") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (number of rows, number of columns)", fnName);
- return null;
- }
- paramExpression.get(0).setName("rows");
- paramExpression.get(1).setName("cols");
-
- ArrayList<ParameterExpression> temp = new ArrayList<ParameterExpression>();
- temp.add(new ParameterExpression(null, new DataIdentifier(namespace)));
- temp.add(paramExpression.get(0));
- temp.add(paramExpression.get(1));
- paramExpression = temp;
-
- functionName = "matrix";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("removeEmpty") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (matrix, axis=0 or 1)", fnName);
- return null;
- }
- int axis = getAxis(paramCtx.get(1));
- if(axis == -1) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (matrix, axis=0 or 1)", fnName);
- return null;
- }
- StringIdentifier marginVal = null;
- if(axis == 0) {
- marginVal = new StringIdentifier("rows", fileName, line, col, line, col);
- }
- else {
- marginVal = new StringIdentifier("cols", fileName, line, col, line, col);
- }
- paramExpression.get(0).setName("target");
- paramExpression.get(1).setName("margin");
- paramExpression.get(1).setExpr(marginVal);
- functionName = "removeEmpty";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("replace") == 0) {
- if(paramExpression.size() != 3) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (matrix, scalar value that should be replaced (pattern), scalar value (replacement))", fnName);
- return null;
- }
- paramExpression.get(0).setName("target");
- paramExpression.get(1).setName("pattern");
- paramExpression.get(2).setName("replacement");
- functionName = "replace";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("range") == 0) {
- if(paramExpression.size() != 3) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (matrix, scalar value that should be replaced (pattern), scalar value (replacement))", fnName);
- return null;
- }
- functionName = "seq";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("norm.cdf") == 0) {
- if(paramExpression.size() != 3) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (target, mean, sd)", fnName);
- return null;
- }
- functionName = "cumulativeProbability";
- paramExpression.get(0).setName("target");
- paramExpression.get(1).setName("mean");
- paramExpression.get(2).setName("sd");
- paramExpression.add(new ParameterExpression("dist", new StringIdentifier("normal", fileName, line, col, line, col)));
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("expon.cdf") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (target, mean)", fnName);
- return null;
- }
- functionName = "cumulativeProbability";
- paramExpression.get(0).setName("target");
- paramExpression.get(1).setName("mean");
- paramExpression.add(new ParameterExpression("dist", new StringIdentifier("exp", fileName, line, col, line, col)));
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("chi.cdf") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (target, df)", fnName);
- return null;
- }
- functionName = "cumulativeProbability";
- paramExpression.get(0).setName("target");
- paramExpression.get(1).setName("df");
- paramExpression.add(new ParameterExpression("dist", new StringIdentifier("chisq", fileName, line, col, line, col)));
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("f.cdf") == 0) {
- if(paramExpression.size() != 3) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (target, df1, df2)", fnName);
- return null;
- }
- functionName = "cumulativeProbability";
- paramExpression.get(0).setName("target");
- paramExpression.get(1).setName("df1");
- paramExpression.get(2).setName("df2");
- paramExpression.add(new ParameterExpression("dist", new StringIdentifier("f", fileName, line, col, line, col)));
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("t.cdf") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (target, df)", fnName);
- return null;
- }
- functionName = "cumulativeProbability";
- paramExpression.get(0).setName("target");
- paramExpression.get(1).setName("df");
- paramExpression.add(new ParameterExpression("dist", new StringIdentifier("t", fileName, line, col, line, col)));
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("percentile") == 0) {
- if(paramExpression.size() != 2 && paramExpression.size() != 3) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts either 2 or 3 arguments", fnName);
- return null;
- }
- functionName = "quantile";
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("arcsin") == 0) {
- functionName = "asin";
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("arccos") == 0) {
- functionName = "acos";
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("arctan") == 0) {
- functionName = "atan";
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("load") == 0) {
- functionName = "read";
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("eigen") == 0) {
- functionName = "eig";
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("power") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments", fnName);
- return null;
- }
- }
- else if(inDefaultNamespace(namespace) && functionName.compareTo("dot") == 0) {
- if(paramExpression.size() != 2) {
- helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments", fnName);
- return null;
- }
- }
-
- ConvertedDMLSyntax retVal = new ConvertedDMLSyntax();
- retVal.namespace = namespace;
- retVal.functionName = functionName;
- retVal.paramExpression = paramExpression;
- return retVal;
- }
-
- class ConvertedDMLSyntax {
- public String namespace;
- public String functionName;
- public ArrayList<ParameterExpression> paramExpression;
- };
-
- private Expression getOperatorExpression(String namespace, String functionName, ArrayList<ParameterExpression> paramExpression) {
- String dmlOperator = null;
-
- if(inDefaultNamespace(namespace) && functionName.compareTo("dot") == 0) {
- if(paramExpression.size() == 2) {
- dmlOperator = "%*%";
- }
- }
-
- if(dmlOperator != null) {
- Expression.BinaryOp bop = Expression.getBinaryOp(dmlOperator);
- Expression expr = new BinaryExpression(bop);
- ((BinaryExpression)expr).setLeft(paramExpression.get(0).getExpr());
- ((BinaryExpression)expr).setRight(paramExpression.get(1).getExpr());
- return expr;
- }
-
- return null;
- }
-
- @Override
- public void exitFunctionCallAssignmentStatement(FunctionCallAssignmentStatementContext ctx) {
- ArrayList<String> names = helper.getQualifiedNames(ctx.name.getText());
- if(names == null) {
- helper.notifyErrorListeners("incorrect function name (only namespace.functionName allowed. Hint: If you are trying to use builtin functions, you can skip the namespace)", ctx.name);
- return;
- }
- String namespace = names.get(0);
- String functionName = names.get(1);
-
- if((functionName.compareTo("print") == 0 || functionName.compareTo("stop") == 0 ) && namespace.compareTo(DMLProgram.DEFAULT_NAMESPACE) == 0) {
- setPrintStatement(ctx, functionName);
- return;
- }
- else if(functionName.compareTo("save") == 0 && namespace.compareTo(DMLProgram.DEFAULT_NAMESPACE) == 0) {
- setOutputStatement(ctx);
- return;
- }
-
- boolean ignoreLValue = false;
- if(ctx.targetList == null || ctx.targetList.size() == 0 || ctx.targetList.get(0).isEmpty()) {
- helper.notifyErrorListeners("function call needs to have lvalue (Quickfix: change it to \'tmpVar = " + functionName + "(...)\')", ctx.name);
- return;
- }
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
-
- ConvertedDMLSyntax convertedSyntax = convertPythonBuiltinFunctionToDMLSyntax(namespace, functionName, paramExpression, ctx.paramExprs, ctx.name, fileName, line, col);
- if(convertedSyntax == null) {
- return;
- }
- else {
- namespace = convertedSyntax.namespace;
- functionName = convertedSyntax.functionName;
- paramExpression = convertedSyntax.paramExpression;
- }
-
- // ===========================================================================================
- FunctionCallIdentifier functCall = new FunctionCallIdentifier(paramExpression);
- try {
- functCall.setFunctionName(functionName);
- functCall.setFunctionNamespace(namespace);
- } catch (ParseException e1) {
- helper.notifyErrorListeners("unable to process function " + functionName, ctx.start);
- return;
- }
-
- DataIdentifier target = null;
- if(!ignoreLValue) {
- if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
- target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
- }
- else {
- helper.notifyErrorListeners("incorrect lvalue ... strange", ctx.targetList.get(0).start);
- //target = new DataIdentifier(); // so as not to avoid null pointer
- return;
- }
- }
-
- Expression operatorExpr = getOperatorExpression(namespace, functionName, paramExpression);
- if(operatorExpr != null) {
- setFileLineColumn(operatorExpr, ctx);
- setAssignmentStatement(target, operatorExpr, ctx);
- return;
- }
-
- //Note: In contrast to the dml parser, namespace and function names are separated by '.' not '::'.
- //Hence, we have to include a whitelist of function names to handle builtins like 'as.scalar'.
- String[] whitelist = new String[]{"as.matrix","as.scalar","as.double","as.integer","as.logical"};
- boolean isWhitelisted = Arrays.asList(whitelist).contains(functionName);
-
- if( !functionName.contains(".") || isWhitelisted
- || functionName.startsWith(DMLProgram.DEFAULT_NAMESPACE) )
- {
- // In global namespace, so it can be a builtin function
- if(!helper.validateBuiltinFunctions(ctx)) {
- return; // it is a built-in function and validation failed, so donot proceed ahead.
- }
- // Double verification: verify passed function name is a (non-parameterized) built-in function.
- try
- {
- // builtin functions
- BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (bife != null) {
- setAssignmentStatement(target, bife, ctx);
- return;
- }
-
- // parameterized builtin functions
- ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (pbife != null) {
- setAssignmentStatement(target, pbife, ctx);
- return;
- }
-
- // built-in data expressions, e.g. read
- DataExpression dbife = DataExpression.getDataExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (dbife != null){
- setAssignmentStatement(target, dbife, ctx);
- return;
- }
- }
- catch(Exception e) {
- helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
- return ;
- }
- }
-
- setAssignmentStatement(target, functCall, ctx);
- }
-
-
- @Override
- public void exitBuiltinFunctionExpression(BuiltinFunctionExpressionContext ctx) {
- // Double verification: verify passed function name is a (non-parameterized) built-in function.
- ArrayList<String> names = helper.getQualifiedNames(ctx.name.getText());
- if(names == null) {
- helper.notifyErrorListeners("incorrect function name (only namespace.functionName allowed. Hint: If you are trying to use builtin functions, you can skip the namespace)", ctx.name);
- return;
- }
- String namespace = names.get(0);
- String functionName = names.get(1);
-
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
-
- ConvertedDMLSyntax convertedSyntax = convertPythonBuiltinFunctionToDMLSyntax(namespace, functionName, paramExpression, ctx.paramExprs, ctx.name, fileName, line, col);
- if(convertedSyntax == null) {
- return;
- }
- else {
- namespace = convertedSyntax.namespace;
- functionName = convertedSyntax.functionName;
- paramExpression = convertedSyntax.paramExpression;
- // System.out.println(ctx.name.getText() + ">>" + namespace + " " + functionName);
- }
-
- Expression operatorExpr = getOperatorExpression(namespace, functionName, paramExpression);
- if(operatorExpr != null) {
- ctx.info.expr = operatorExpr;
- setFileLineColumn(operatorExpr, ctx);
- return;
- }
-
- try {
- BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, paramExpression, fileName, line, col, line, col);
- if (bife != null){
- // It is a builtin function
- ctx.info.expr = bife;
- return;
- }
-
- ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, paramExpression, fileName, line, col, line, col);
- if (pbife != null){
- // It is a parameterized builtin function
- ctx.info.expr = pbife;
- return;
- }
-
- // built-in read, rand ...
- DataExpression dbife = DataExpression.getDataExpression(functionName, paramExpression, fileName, line, col, line, col);
- if (dbife != null){
- ctx.info.expr = dbife;
- return;
- }
- } catch(Exception e) {
- helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
- return ;
- }
- helper.notifyErrorListeners("only builtin functions allowed as part of expression", ctx.start);
- }
-
- private void setMultiAssignmentStatement(ArrayList<DataIdentifier> target, Expression expression, StatementContext ctx) {
- ctx.info.stmt = new MultiAssignmentStatement(target, expression);
- ctx.info.stmt.setAllPositions(helper.getCurrentFileName(), ctx.start.getLine(), ctx.start.getCharPositionInLine(), ctx.start.getLine(), ctx.start.getCharPositionInLine());
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
- @Override
- public void exitFunctionCallMultiAssignmentStatement(
- FunctionCallMultiAssignmentStatementContext ctx) {
- ArrayList<String> names = helper.getQualifiedNames(ctx.name.getText());
- if(names == null) {
- helper.notifyErrorListeners("incorrect function name (only namespace.functionName allowed. Hint: If you are trying to use builtin functions, you can skip the namespace)", ctx.name);
- return;
- }
- String namespace = names.get(0);
- String functionName = names.get(1);
-
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
- ConvertedDMLSyntax convertedSyntax = convertPythonBuiltinFunctionToDMLSyntax(namespace, functionName, paramExpression, ctx.paramExprs, ctx.name, fileName, line, col);
- if(convertedSyntax == null) {
- return;
- }
- else {
- namespace = convertedSyntax.namespace;
- functionName = convertedSyntax.functionName;
- paramExpression = convertedSyntax.paramExpression;
- }
-
- // No need to support dot() function since it will never return multi-assignment function
-
- FunctionCallIdentifier functCall = new FunctionCallIdentifier(paramExpression);
- try {
- functCall.setFunctionName(functionName);
- functCall.setFunctionNamespace(namespace);
- } catch (ParseException e1) {
- helper.notifyErrorListeners("unable to process function " + functionName, ctx.start);
- return;
- }
-
- ArrayList<DataIdentifier> targetList = new ArrayList<DataIdentifier>();
- for(DataIdentifierContext dataCtx : ctx.targetList) {
- if(dataCtx.dataInfo.expr instanceof DataIdentifier) {
- targetList.add((DataIdentifier) dataCtx.dataInfo.expr);
- }
- else {
- helper.notifyErrorListeners("incorrect lvalue ... strange", dataCtx.start);
- //target = new DataIdentifier(); // so as not to avoid null pointer
- return;
- }
- }
-
- if(!functionName.contains(".") || functionName.startsWith(DMLProgram.DEFAULT_NAMESPACE)) {
- // In global namespace, so it can be a builtin function
- // Double verification: verify passed function name is a (non-parameterized) built-in function.
- try {
- BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (bife != null){
- // It is a builtin function
- setMultiAssignmentStatement(targetList, bife, ctx);
- return;
- }
-
- ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (pbife != null){
- // It is a parameterized builtin function
- setMultiAssignmentStatement(targetList, pbife, ctx);
- return;
- }
-
- // built-in read, rand ...
- DataExpression dbife = DataExpression.getDataExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
- if (dbife != null){
- setMultiAssignmentStatement(targetList, dbife, ctx);
- return;
- }
- } catch(Exception e) {
- helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
- return;
- }
- }
-
- setMultiAssignmentStatement(targetList, functCall, ctx);
- }
-
- private StatementBlock getStatementBlock(Statement current) {
- return PyDMLParserWrapper.getStatementBlock(current);
- }
-
- @Override
- public void exitIfStatement(IfStatementContext ctx) {
- IfStatement ifStmt = new IfStatement();
- ConditionalPredicate predicate = new ConditionalPredicate(ctx.predicate.info.expr);
- ifStmt.setConditionalPredicate(predicate);
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- ifStmt.setAllPositions(fileName, line, col, line, col);
-
- if(ctx.ifBody.size() > 0) {
- for(StatementContext stmtCtx : ctx.ifBody) {
- ifStmt.addStatementBlockIfBody(getStatementBlock(stmtCtx.info.stmt));
- }
- ifStmt.mergeStatementBlocksIfBody();
- }
-
- if(ctx.elseBody.size() > 0) {
- for(StatementContext stmtCtx : ctx.elseBody) {
- ifStmt.addStatementBlockElseBody(getStatementBlock(stmtCtx.info.stmt));
- }
- ifStmt.mergeStatementBlocksElseBody();
- }
-
- ctx.info.stmt = ifStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
- @Override
- public void exitWhileStatement(WhileStatementContext ctx) {
- WhileStatement whileStmt = new WhileStatement();
- ConditionalPredicate predicate = new ConditionalPredicate(ctx.predicate.info.expr);
- whileStmt.setPredicate(predicate);
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- whileStmt.setAllPositions(fileName, line, col, line, col);
-
- if(ctx.body.size() > 0) {
- for(StatementContext stmtCtx : ctx.body) {
- whileStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
- }
- whileStmt.mergeStatementBlocks();
- }
-
- ctx.info.stmt = whileStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
- @Override
- public void exitForStatement(ForStatementContext ctx) {
- ForStatement forStmt = new ForStatement();
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText());
- HashMap<String, String> parForParamValues = null;
- Expression incrementExpr = new IntIdentifier(1, fileName, line, col, line, col);
- if(ctx.iterPred.info.increment != null) {
- incrementExpr = ctx.iterPred.info.increment;
- }
- IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, fileName, line, col, line, col);
- forStmt.setPredicate(predicate);
-
- if(ctx.body.size() > 0) {
- for(StatementContext stmtCtx : ctx.body) {
- forStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
- }
- forStmt.mergeStatementBlocks();
- }
- ctx.info.stmt = forStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
- @Override
- public void exitParForStatement(ParForStatementContext ctx) {
- ParForStatement parForStmt = new ParForStatement();
- String fileName = helper.getCurrentFileName();
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
-
- DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText());
- HashMap<String, String> parForParamValues = new HashMap<String, String>();
- if(ctx.parForParams != null && ctx.parForParams.size() > 0) {
- for(StrictParameterizedExpressionContext parForParamCtx : ctx.parForParams) {
- parForParamValues.put(parForParamCtx.paramName.getText(), parForParamCtx.paramVal.getText());
- }
- }
-
- Expression incrementExpr = new IntIdentifier(1, fileName, line, col, line, col);
-
- if( ctx.iterPred.info.increment != null ) {
- incrementExpr = ctx.iterPred.info.increment;
- }
- IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, fileName, line, col, line, col);
- parForStmt.setPredicate(predicate);
- if(ctx.body.size() > 0) {
- for(StatementContext stmtCtx : ctx.body) {
- parForStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
- }
- parForStmt.mergeStatementBlocks();
- }
- ctx.info.stmt = parForStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- }
-
-
-
-
- // ----------------------------------------------------------------------
- @Override
- public void exitMl_type(Ml_typeContext ctx) { }
-
- private ArrayList<DataIdentifier> getFunctionParameters(List<TypedArgNoAssignContext> ctx) {
- ArrayList<DataIdentifier> retVal = new ArrayList<DataIdentifier>();
- for(TypedArgNoAssignContext paramCtx : ctx) {
- DataIdentifier dataId = new DataIdentifier(paramCtx.paramName.getText());
- String dataType = null;
- String valueType = null;
-
- if(paramCtx.paramType == null || paramCtx.paramType.dataType() == null
- || paramCtx.paramType.dataType().getText() == null || paramCtx.paramType.dataType().getText().isEmpty()) {
- dataType = "scalar";
- }
- else {
- dataType = paramCtx.paramType.dataType().getText();
- }
-
- if(dataType.compareTo("matrix") == 0) {
- // matrix
- dataId.setDataType(DataType.MATRIX);
- }
- else if(dataType.compareTo("scalar") == 0) {
- // scalar
- dataId.setDataType(DataType.SCALAR);
- }
- else {
- helper.notifyErrorListeners("invalid datatype " + dataType, paramCtx.start);
- return null;
- }
-
- valueType = paramCtx.paramType.valueType().getText();
- if(valueType.compareTo("int") == 0) {
- dataId.setValueType(ValueType.INT);
- }
- else if(valueType.compareTo("str") == 0) {
- dataId.setValueType(ValueType.STRING);
- }
- else if(valueType.compareTo("bool") == 0) {
- dataId.setValueType(ValueType.BOOLEAN);
- }
- else if(valueType.compareTo("float") == 0) {
- dataId.setValueType(ValueType.DOUBLE);
- }
- else {
- helper.notifyErrorListeners("invalid valuetype " + valueType, paramCtx.start);
- return null;
- }
- retVal.add(dataId);
- }
- return retVal;
- }
-
- @Override
- public void exitInternalFunctionDefExpression(InternalFunctionDefExpressionContext ctx) {
- FunctionStatement functionStmt = new FunctionStatement();
-
- ArrayList<DataIdentifier> functionInputs = getFunctionParameters(ctx.inputParams);
- functionStmt.setInputParams(functionInputs);
-
- // set function outputs
- ArrayList<DataIdentifier> functionOutputs = getFunctionParameters(ctx.outputParams);
- functionStmt.setOutputParams(functionOutputs);
-
- // set function name
- functionStmt.setName(ctx.name.getText());
-
-
- if(ctx.body.size() > 0) {
- // handle function body
- // Create arraylist of one statement block
- ArrayList<StatementBlock> body = new ArrayList<StatementBlock>();
- for(StatementContext stmtCtx : ctx.body) {
- body.add(getStatementBlock(stmtCtx.info.stmt));
- }
- functionStmt.setBody(body);
- functionStmt.mergeStatementBlocks();
- }
- else {
- helper.notifyErrorListeners("functions with no statements are not allowed", ctx.start);
- return;
- }
-
- ctx.info.stmt = functionStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- ctx.info.functionName = ctx.name.getText();
- }
-
- @Override
- public void exitExternalFunctionDefExpression(ExternalFunctionDefExpressionContext ctx) {
- ExternalFunctionStatement functionStmt = new ExternalFunctionStatement();
-
- ArrayList<DataIdentifier> functionInputs = getFunctionParameters(ctx.inputParams);
- functionStmt.setInputParams(functionInputs);
-
- // set function outputs
- ArrayList<DataIdentifier> functionOutputs = getFunctionParameters(ctx.outputParams);
- functionStmt.setOutputParams(functionOutputs);
-
- // set function name
- functionStmt.setName(ctx.name.getText());
-
- // set other parameters
- HashMap<String, String> otherParams = new HashMap<String,String>();
- boolean atleastOneClassName = false;
- for(StrictParameterizedKeyValueStringContext otherParamCtx : ctx.otherParams){
- String paramName = otherParamCtx.paramName.getText();
- String val = "";
- String text = otherParamCtx.paramVal.getText();
- // First unquote the string
- if( (text.startsWith("\"") && text.endsWith("\"")) ||
- (text.startsWith("\'") && text.endsWith("\'"))) {
- if(text.length() > 2) {
- val = text.substring(1, text.length()-1);
- }
- // Empty value allowed
- }
- else {
- helper.notifyErrorListeners("the value of user parameter for external function should be of type str", ctx.start);
- return;
- }
- otherParams.put(paramName, val);
- if(paramName.compareTo("classname") == 0) {
- atleastOneClassName = true;
- }
- }
- functionStmt.setOtherParams(otherParams);
- if(!atleastOneClassName) {
- helper.notifyErrorListeners("the parameter \'className\' needs to be passed for defExternal", ctx.start);
- return;
- }
-
- ctx.info.stmt = functionStmt;
- setFileLineColumn(ctx.info.stmt, ctx);
- ctx.info.functionName = ctx.name.getText();
- }
-
-
- @Override
- public void exitPathStatement(PathStatementContext ctx) {
- PathStatement stmt = new PathStatement(ctx.pathValue.getText());
- String filePath = ctx.pathValue.getText();
- if((filePath.startsWith("\"") && filePath.endsWith("\"")) ||
- filePath.startsWith("'") && filePath.endsWith("'")) {
- filePath = filePath.substring(1, filePath.length()-1);
- }
-
- _workingDir = filePath;
- ctx.info.stmt = stmt;
- }
-
- @Override
- public void exitIfdefAssignmentStatement(IfdefAssignmentStatementContext ctx) {
- if(!ctx.commandLineParam.getText().startsWith("$")) {
- helper.notifyErrorListeners("the first argument of ifdef function should be a commandline argument parameter (which starts with $)", ctx.commandLineParam.start);
- return;
- }
-
- if(ctx.targetList == null || ctx.targetList.size() != 1) {
- helper.notifyErrorListeners("incorrect parsing for ifdef function", ctx.start);
- return;
- }
- String targetListText = ctx.targetList.get(0).getText();
- if(targetListText.startsWith("$")) {
- helper.notifyErrorListeners("lhs of ifdef function cannot be a commandline parameters. Use local variable instead", ctx.start);
- return;
- }
-
- DataIdentifier target = null;
- if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
- target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
- Expression source = null;
- if(ctx.commandLineParam.dataInfo.expr != null) {
- // Since commandline parameter is set
- // The check of following is done in fillExpressionInfoCommandLineParameters:
- // Command line param cannot be empty string
- // If you want to pass space, please quote it
- source = ctx.commandLineParam.dataInfo.expr;
- }
- else {
- source = ctx.source.info.expr;
- }
-
- int line = ctx.start.getLine();
- int col = ctx.start.getCharPositionInLine();
- try {
- ctx.info.stmt = new AssignmentStatement(target, source, line, col, line, col);
- setFileLineColumn(ctx.info.stmt, ctx);
- } catch (LanguageException e) {
- helper.notifyErrorListeners("invalid assignment for ifdef function", ctx.targetList.get(0).start);
- return;
- }
-
- }
- else {
- helper.notifyErrorListeners("incorrect lvalue in ifdef function... strange", ctx.targetList.get(0).start);
- return;
- }
-
- }
-
- // ----------------------------------------------------------------------
- @Override
- public void exitParameterizedExpression(ParameterizedExpressionContext ctx) { }
-
-
- @Override
- public void exitStrictParameterizedExpression(StrictParameterizedExpressionContext ctx) { }
-
- @Override
- public void exitTypedArgNoAssign(TypedArgNoAssignContext ctx) { }
- @Override
- public void enterIfdefAssignmentStatement(IfdefAssignmentStatementContext ctx) { }
- @Override
- public void enterMatrixDataTypeCheck(MatrixDataTypeCheckContext ctx) { }
- @Override
- public void exitMatrixDataTypeCheck(MatrixDataTypeCheckContext ctx) {
- if( ctx.ID().getText().compareTo("matrix") == 0
- || ctx.ID().getText().compareTo("scalar") == 0
- ) {
- // Do nothing
- }
- else if(ctx.ID().getText().compareTo("Matrix") == 0)
- helper.notifyErrorListeners("incorrect datatype (Hint: use matrix instead of Matrix)", ctx.start);
- else if(ctx.ID().getText().compareTo("Scalar") == 0)
- helper.notifyErrorListeners("incorrect datatype (Hint: use scalar instead of Scalar)", ctx.start);
- else if( ctx.ID().getText().compareTo("int") == 0
- || ctx.ID().getText().compareTo("str") == 0
- || ctx.ID().getText().compareTo("bool") == 0
- || ctx.ID().getText().compareTo("float") == 0
- ) {
- helper.notifyErrorListeners("expected datatype but found a valuetype (Hint: use matrix or scalar instead of " + ctx.ID().getText() + ")", ctx.start);
- }
- else {
- helper.notifyErrorListeners("incorrect datatype (expected matrix or scalar)", ctx.start);
- }
- }
-
- @Override
- public void enterBuiltinFunctionExpression(BuiltinFunctionExpressionContext ctx) {}
- @Override
- public void enterStrictParameterizedKeyValueString(StrictParameterizedKeyValueStringContext ctx) { }
- @Override
- public void exitStrictParameterizedKeyValueString(StrictParameterizedKeyValueStringContext ctx) {}
- @Override
- public void enterIterablePredicateColonExpression(IterablePredicateColonExpressionContext ctx) {}
- @Override
- public void enterIterablePredicateSeqExpression(IterablePredicateSeqExpressionContext ctx) { }
-
- @Override
- public void exitIterablePredicateColonExpression(IterablePredicateColonExpressionContext ctx) {
- ctx.info.from = ctx.from.info.expr;
- ctx.info.to = ctx.to.info.expr;
- ctx.info.increment = null;
- }
-
- @Override
- public void exitIterablePredicateSeqExpression(IterablePredicateSeqExpressionContext ctx) {
- if(ctx.ID().getText().compareTo("range") != 0) {
- helper.notifyErrorListeners("incorrect function:\'" + ctx.ID().getText() + "\'. expected \'range\'", ctx.start);
- return;
- }
- ctx.info.from = ctx.from.info.expr;
- ctx.info.to = ctx.to.info.expr;
- ctx.info.increment = ctx.increment.info.expr;
- }
-
- @Override
- public void enterConstFalseExpression(ConstFalseExpressionContext ctx) { }
- @Override
- public void enterConstTrueExpression(ConstTrueExpressionContext ctx) { }
-
- @Override
- public void exitConstFalseExpression(ConstFalseExpressionContext ctx) {
- boolean val = false;
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
-
- @Override
- public void exitConstTrueExpression(ConstTrueExpressionContext ctx) {
- boolean val = true;
- int linePosition = ctx.start.getLine();
- int charPosition = ctx.start.getCharPositionInLine();
- ctx.info.expr = new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
- setFileLineColumn(ctx.info.expr, ctx);
- }
-
- @Override
- public void exitIgnoreNewLine(IgnoreNewLineContext ctx) {
- // Introduce empty StatementInfo
- // This is later ignored by PyDMLParserWrapper
- try {
- ctx.info.stmt = new AssignmentStatement(null, null, 0, 0, 0, 0);
- ctx.info.stmt.setEmptyNewLineStatement(true);
- } catch (LanguageException e) {
- e.printStackTrace();
- }
-
- }
- @Override
- public void enterValueDataTypeCheck(ValueDataTypeCheckContext ctx) { }
- @Override
- public void exitValueDataTypeCheck(ValueDataTypeCheckContext ctx) {
- if( ctx.ID().getText().compareTo("int") == 0
- || ctx.ID().getText().compareTo("str") == 0
- || ctx.ID().getText().compareTo("bool") == 0
- || ctx.ID().getText().compareTo("float") == 0
- ) {
- // Do nothing
- }
- else if(ctx.ID().getText().compareTo("integer") == 0)
- helper.notifyErrorListeners("incorrect valuetype (Hint: use int instead of integer)", ctx.start);
- else if(ctx.ID().getText().compareTo("double") == 0)
- helper.notifyErrorListeners("incorrect valuetype (Hint: use float instead of double)", ctx.start);
- else if(ctx.ID().getText().compareTo("boolean") == 0)
- helper.notifyErrorListeners("incorrect valuetype (Hint: use bool instead of boolean)", ctx.start);
- else if(ctx.ID().getText().compareTo("string") == 0)
- helper.notifyErrorListeners("incorrect valuetype (Hint: use str instead of string)", ctx.start);
- else {
- helper.notifyErrorListeners("incorrect valuetype (expected int, str, bool or float)", ctx.start);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java
deleted file mode 100644
index f014684..0000000
--- a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticValidatorHelper.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.parser.python;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.antlr.v4.runtime.Token;
-import org.apache.sysml.parser.DMLProgram;
-import org.apache.sysml.parser.python.PydmlParser.FunctionCallAssignmentStatementContext;
-import org.apache.sysml.parser.python.PydmlParser.ParameterizedExpressionContext;
-import org.apache.sysml.parser.python.PydmlSyntacticErrorListener.CustomDmlErrorListener;
-
-
-public class PydmlSyntacticValidatorHelper
-{
- private CustomDmlErrorListener _errorListener = null;
- public PydmlSyntacticValidatorHelper(CustomDmlErrorListener errorListener) {
- this._errorListener = errorListener;
- }
- public void notifyErrorListeners(String message, int line, int charPositionInLine) {
- this._errorListener.validationError(line, charPositionInLine, message);
- }
-
- public void notifyErrorListeners(String message, Token op) {
- this._errorListener.validationError(op.getLine(), op.getCharPositionInLine(), message);
- }
-
- public void raiseWarning(String message, Token op) {
- this._errorListener.validationWarning(op.getLine(), op.getCharPositionInLine(), message);
- }
-
- public String getCurrentFileName() {
- return _errorListener.peekFileName();
- }
-
- // Returns list of two elements <namespace, function names>, else null
- public ArrayList<String> getQualifiedNames(String fullyQualifiedFunctionName) {
- String [] fnNames = fullyQualifiedFunctionName.split("\\."); // instead of ::
- String functionName = "";
- String namespace = "";
- if(fnNames.length == 1) {
- namespace = DMLProgram.DEFAULT_NAMESPACE;
- functionName = fnNames[0].trim();
- }
- else if(fnNames.length == 2) {
- namespace = fnNames[0].trim();
- functionName = fnNames[1].trim();
- }
- else
- return null;
-
- ArrayList<String> retVal = new ArrayList<String>();
- retVal.add(namespace);
- retVal.add(functionName);
- return retVal;
- }
-
- public boolean validateBuiltinFunctions(FunctionCallAssignmentStatementContext ctx) {
- String functionName = ctx.name.getText().replaceAll(" ", "").trim();
- if(functionName.compareTo("write") == 0 || functionName.compareTo(DMLProgram.DEFAULT_NAMESPACE + ".write") == 0) {
- return validateBuiltinWriteFunction(ctx);
- }
- return true;
- }
-
- private boolean validateBuiltinWriteFunction(FunctionCallAssignmentStatementContext ctx) {
-
- return true;
- }
-
- public ArrayList<org.apache.sysml.parser.ParameterExpression> getParameterExpressionList(List<ParameterizedExpressionContext> paramExprs) {
- ArrayList<org.apache.sysml.parser.ParameterExpression> retVal = new ArrayList<org.apache.sysml.parser.ParameterExpression>();
- for(ParameterizedExpressionContext ctx : paramExprs) {
- String paramName = null;
- if(ctx.paramName != null && ctx.paramName.getText() != null && !ctx.paramName.getText().isEmpty()) {
- paramName = ctx.paramName.getText();
- }
- org.apache.sysml.parser.ParameterExpression myArg = new org.apache.sysml.parser.ParameterExpression(paramName, ctx.paramVal.info.expr);
- retVal.add(myArg);
- }
- return retVal;
- }
-}
[5/8] incubator-systemml git commit: [SYSTEMML-148] Refactored
.parser.{antlr4, python} to .parser.{dml, pydml}.
Posted by du...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/pydml/PyDMLParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/pydml/PyDMLParserWrapper.java b/src/main/java/org/apache/sysml/parser/pydml/PyDMLParserWrapper.java
new file mode 100644
index 0000000..25ee006
--- /dev/null
+++ b/src/main/java/org/apache/sysml/parser/pydml/PyDMLParserWrapper.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.parser.pydml;
+
+import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.antlr.v4.runtime.ANTLRInputStream;
+import org.antlr.v4.runtime.BailErrorStrategy;
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.DefaultErrorStrategy;
+import org.antlr.v4.runtime.atn.PredictionMode;
+import org.antlr.v4.runtime.misc.ParseCancellationException;
+import org.antlr.v4.runtime.tree.ParseTree;
+import org.antlr.v4.runtime.tree.ParseTreeWalker;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.parser.AParserWrapper;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.ForStatement;
+import org.apache.sysml.parser.ForStatementBlock;
+import org.apache.sysml.parser.FunctionStatementBlock;
+import org.apache.sysml.parser.IfStatement;
+import org.apache.sysml.parser.IfStatementBlock;
+import org.apache.sysml.parser.ImportStatement;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.parser.ParForStatement;
+import org.apache.sysml.parser.ParForStatementBlock;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.WhileStatement;
+import org.apache.sysml.parser.WhileStatementBlock;
+import org.apache.sysml.parser.dml.DMLParserWrapper;
+import org.apache.sysml.parser.pydml.PydmlParser.FunctionStatementContext;
+import org.apache.sysml.parser.pydml.PydmlParser.PmlprogramContext;
+import org.apache.sysml.parser.pydml.PydmlParser.StatementContext;
+import org.apache.sysml.parser.pydml.PydmlSyntacticErrorListener.CustomDmlErrorListener;
+
+/**
+ * Logic of this wrapper is similar to DMLParserWrapper.
+ *
+ * Note: ExpressionInfo and StatementInfo are simply wrapper objects and are reused in both DML and PyDML parsers.
+ *
+ */
+public class PyDMLParserWrapper extends AParserWrapper
+{
+ private static final Log LOG = LogFactory.getLog(DMLScript.class.getName());
+
+ /**
+ * Custom wrapper to convert statement into statement blocks. Called by doParse and in PydmlSyntacticValidator for for, parfor, while, ...
+ * @param current a statement
+ * @return corresponding statement block
+ */
+ public static StatementBlock getStatementBlock(org.apache.sysml.parser.Statement current) {
+ StatementBlock blk = null;
+ if(current instanceof ParForStatement) {
+ blk = new ParForStatementBlock();
+ blk.addStatement(current);
+ }
+ else if(current instanceof ForStatement) {
+ blk = new ForStatementBlock();
+ blk.addStatement(current);
+ }
+ else if(current instanceof IfStatement) {
+ blk = new IfStatementBlock();
+ blk.addStatement(current);
+ }
+ else if(current instanceof WhileStatement) {
+ blk = new WhileStatementBlock();
+ blk.addStatement(current);
+ }
+ else {
+ // This includes ImportStatement
+ blk = new StatementBlock();
+ blk.addStatement(current);
+ }
+ return blk;
+ }
+
+ /**
+ * Parses the passed file with command line parameters. You can either pass both (local file) or just dmlScript (hdfs) or just file name (import command)
+ * @param fileName either full path or null --> only used for better error handling
+ * @param dmlScript required
+ * @param argVals
+ * @return
+ * @throws ParseException
+ */
+ @Override
+ public DMLProgram parse(String fileName, String dmlScript, HashMap<String,String> argVals) throws ParseException {
+ DMLProgram prog = null;
+
+ if(dmlScript == null || dmlScript.trim().isEmpty()) {
+ throw new ParseException("Incorrect usage of parse. Please pass dmlScript not just filename");
+ }
+
+ // Set the pipeline required for ANTLR parsing
+ PyDMLParserWrapper parser = new PyDMLParserWrapper();
+ prog = parser.doParse(fileName, dmlScript, argVals);
+
+ if(prog == null) {
+ throw new ParseException("One or more errors found during parsing. (could not construct AST for file: " + fileName + "). Cannot proceed ahead.");
+ }
+ return prog;
+ }
+
+ /**
+ * This function is supposed to be called directly only from PydmlSyntacticValidator when it encounters 'import'
+ * @param fileName
+ * @return null if atleast one error
+ */
+ public DMLProgram doParse(String fileName, String dmlScript, HashMap<String,String> argVals) throws ParseException {
+ DMLProgram dmlPgm = null;
+
+ ANTLRInputStream in;
+ try {
+ if(dmlScript == null) {
+ dmlScript = DMLParserWrapper.readDMLScript(fileName);
+ }
+
+ InputStream stream = new ByteArrayInputStream(dmlScript.getBytes());
+ in = new org.antlr.v4.runtime.ANTLRInputStream(stream);
+ }
+ catch (FileNotFoundException e) {
+ throw new ParseException("ERROR: Cannot find file:" + fileName, e);
+ }
+ catch (IOException e) {
+ throw new ParseException("ERROR: Cannot open file:" + fileName, e);
+ }
+ catch (LanguageException e) {
+ throw new ParseException("ERROR: " + e.getMessage(), e);
+ }
+
+ PmlprogramContext ast = null;
+ CustomDmlErrorListener errorListener = new CustomDmlErrorListener();
+
+ try {
+ PydmlLexer lexer = new PydmlLexer(in);
+ CommonTokenStream tokens = new CommonTokenStream(lexer);
+ PydmlParser antlr4Parser = new PydmlParser(tokens);
+
+ boolean tryOptimizedParsing = false; // For now no optimization, since it is not able to parse integer value.
+
+ if(tryOptimizedParsing) {
+ // Try faster and simpler SLL
+ antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
+ antlr4Parser.removeErrorListeners();
+ antlr4Parser.setErrorHandler(new BailErrorStrategy());
+ try{
+ ast = antlr4Parser.pmlprogram();
+ // If successful, no need to try out full LL(*) ... SLL was enough
+ }
+ catch(ParseCancellationException ex) {
+ // Error occurred, so now try full LL(*) for better error messages
+ tokens.reset();
+ antlr4Parser.reset();
+ if(fileName != null) {
+ errorListener.pushCurrentFileName(fileName);
+ }
+ else {
+ errorListener.pushCurrentFileName("MAIN_SCRIPT");
+ }
+ // Set our custom error listener
+ antlr4Parser.addErrorListener(errorListener);
+ antlr4Parser.setErrorHandler(new DefaultErrorStrategy());
+ antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL);
+ ast = antlr4Parser.pmlprogram();
+ }
+ }
+ else {
+ // Set our custom error listener
+ antlr4Parser.removeErrorListeners();
+ antlr4Parser.addErrorListener(errorListener);
+ errorListener.pushCurrentFileName(fileName);
+
+ // Now do the parsing
+ ast = antlr4Parser.pmlprogram();
+ }
+ }
+ catch(Exception e) {
+ throw new ParseException("ERROR: Cannot parse the program:" + fileName, e);
+ }
+
+
+ try {
+ // Now convert the parse tree into DMLProgram
+ // Do syntactic validation while converting
+ ParseTree tree = ast;
+ // And also do syntactic validation
+ ParseTreeWalker walker = new ParseTreeWalker();
+ PydmlSyntacticValidatorHelper helper = new PydmlSyntacticValidatorHelper(errorListener);
+ PydmlSyntacticValidator validator = new PydmlSyntacticValidator(helper, fileName, argVals);
+ walker.walk(validator, tree);
+ errorListener.popFileName();
+ if(errorListener.isAtleastOneError()) {
+ return null;
+ }
+ dmlPgm = createDMLProgram(ast);
+ }
+ catch(Exception e) {
+ throw new ParseException("ERROR: Cannot translate the parse tree into DMLProgram" + e.getMessage(), e);
+ }
+
+ return dmlPgm;
+ }
+
+
+ private DMLProgram createDMLProgram(PmlprogramContext ast) {
+
+ DMLProgram dmlPgm = new DMLProgram();
+
+ // First add all the functions
+ for(FunctionStatementContext fn : ast.functionBlocks) {
+ FunctionStatementBlock functionStmtBlk = new FunctionStatementBlock();
+ functionStmtBlk.addStatement(fn.info.stmt);
+ try {
+ // TODO: currently the logic of nested namespace is not clear.
+ String namespace = DMLProgram.DEFAULT_NAMESPACE;
+ dmlPgm.addFunctionStatementBlock(namespace, fn.info.functionName, functionStmtBlk);
+ } catch (LanguageException e) {
+ LOG.error("line: " + fn.start.getLine() + ":" + fn.start.getCharPositionInLine() + " cannot process the function " + fn.info.functionName);
+ return null;
+ }
+ }
+
+ // Then add all the statements
+ for(StatementContext stmtCtx : ast.blocks) {
+ Statement current = stmtCtx.info.stmt;
+ if(current == null) {
+ LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the statement");
+ return null;
+ }
+
+ // Ignore Newline logic
+ if(current.isEmptyNewLineStatement()) {
+ continue;
+ }
+
+ if(current instanceof ImportStatement) {
+ // Handle import statements separately
+ if(stmtCtx.info.namespaces != null) {
+ // Add the DMLProgram entries into current program
+ for(Map.Entry<String, DMLProgram> entry : stmtCtx.info.namespaces.entrySet()) {
+ dmlPgm.getNamespaces().put(entry.getKey(), entry.getValue());
+ }
+ }
+ else {
+ LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the import statement");
+ return null;
+ }
+ }
+
+ // Now wrap statement into individual statement block
+ // merge statement will take care of merging these blocks
+ dmlPgm.addStatementBlock(getStatementBlock(current));
+ }
+
+ dmlPgm.mergeStatementBlocks();
+ return dmlPgm;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/pydml/Pydml.g4
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/pydml/Pydml.g4 b/src/main/java/org/apache/sysml/parser/pydml/Pydml.g4
new file mode 100644
index 0000000..425859c
--- /dev/null
+++ b/src/main/java/org/apache/sysml/parser/pydml/Pydml.g4
@@ -0,0 +1,384 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grammar Pydml;
+
+@header
+{
+ // package org.apache.sysml.parser.pydml;
+ //import org.apache.sysml.parser.dml.StatementInfo;
+ //import org.apache.sysml.parser.dml.ExpressionInfo;
+}
+
+// This antlr grammar is based on Python 3.3 language reference: https://docs.python.org/3.3/reference/grammar.html
+
+tokens { INDENT, DEDENT }
+
+@lexer::members {
+ private boolean debugIndentRules = false;
+
+ // Indentation level stack
+ private java.util.Stack<Integer> indents = new java.util.Stack<Integer>();
+
+ // Extra tokens queue (see the NEWLINE rule).
+ private java.util.Queue<Token> tokens = new java.util.LinkedList<Token>();
+
+ // Number of opened braces, brackets and parenthesis.
+ private int opened = 0;
+
+ // This is only used to set the line number for dedent
+ private Token lastToken = null;
+
+
+ @Override
+ public void emit(Token t) {
+ if(debugIndentRules)
+ System.out.println("Emitted token:" + t);
+
+ super.setToken(t);
+ tokens.offer(t);
+ }
+
+
+ @Override
+ public Token nextToken() {
+ if (_input.LA(1) == EOF && !this.indents.isEmpty()) {
+ if(debugIndentRules)
+ System.out.println("EOF reached and expecting some DEDENTS, so emitting them");
+
+ tokens.poll();
+ this.emit(commonToken(PydmlParser.NEWLINE, "\n"));
+
+ // Now emit as much DEDENT tokens as needed.
+ while (!indents.isEmpty()) {
+ if(debugIndentRules)
+ System.out.println("Emitting (inserted) DEDENTS");
+
+ this.emit(createDedent());
+ indents.pop();
+ }
+ // Put the EOF back on the token stream.
+ this.emit(commonToken(PydmlParser.EOF, "<EOF>"));
+ }
+ Token next = super.nextToken();
+ if (next.getChannel() == Token.DEFAULT_CHANNEL) {
+ // Keep track of the last token on the default channel.
+ this.lastToken = next;
+ }
+ Token retVal = tokens.isEmpty() ? next : tokens.poll();
+
+ if(debugIndentRules)
+ System.out.println("Returning nextToken: [" + retVal + "]<<" + tokens.isEmpty());
+
+ return retVal;
+ }
+
+ private Token createDedent() {
+ CommonToken dedent = commonToken(PydmlParser.DEDENT, "");
+ dedent.setLine(this.lastToken.getLine());
+ return dedent;
+ }
+
+ private CommonToken commonToken(int type, String text) {
+ // Nike: Main change: This logic was screwed up and was emitting additional 3 characters, so commenting it for now.
+ // int start = this.getCharIndex();
+ // int stop = start + text.length();
+ // return new CommonToken(this._tokenFactorySourcePair, type, DEFAULT_TOKEN_CHANNEL, start, stop);
+ return new CommonToken(type, text); // Main change
+ }
+
+ // Calculates the indentation level from the spaces:
+ // "Tabs are replaced (from left to right) by one to eight spaces
+ // such that the total number of characters up to and including
+ // the replacement is a multiple of eight [...]"
+ // https://docs.python.org/3.1/reference/lexical_analysis.html#indentation
+ static int getIndentationCount(String spaces) {
+ int count = 0;
+ for (char ch : spaces.toCharArray()) {
+ switch (ch) {
+ case '\t':
+ count += 8 - (count % 8);
+ break;
+ default:
+ // A normal space char.
+ count++;
+ }
+ }
+ return count;
+ }
+}
+
+
+// 2. Modify this g4 by comparing it with Java:
+// - https://pythonconquerstheuniverse.wordpress.com/2009/10/03/python-java-a-side-by-side-comparison/
+// - http://www.cs.gordon.edu/courses/cps122/handouts-2014/From%20Python%20to%20Java%20Lecture/A%20Comparison%20of%20the%20Syntax%20of%20Python%20and%20Java.pdf
+// - http://cs.joensuu.fi/~pviktor/python/slides/cheatsheet.pdf
+// - http://www.interfaceware.com/manual/chameleon/scripts/quickreference.pdf
+
+// DML Program is a list of expression
+// For now, we only allow global function definitions (not nested or inside a while block)
+pmlprogram: (blocks+=statement | functionBlocks+=functionStatement)* NEWLINE* EOF;
+
+
+
+statement returns [ StatementInfo info ]
+@init {
+ // This actions occurs regardless of how many alternatives in this rule
+ $info = new StatementInfo();
+} :
+ // ------------------------------------------
+ // ImportStatement
+ 'source' OPEN_PAREN filePath = STRING CLOSE_PAREN 'as' namespace=ID NEWLINE # ImportStatement
+ | 'setwd' OPEN_PAREN pathValue = STRING CLOSE_PAREN NEWLINE # PathStatement
+ // ------------------------------------------
+ // AssignmentStatement
+ | targetList+=dataIdentifier '=' 'ifdef' OPEN_PAREN commandLineParam=dataIdentifier ',' source=expression CLOSE_PAREN NEWLINE # IfdefAssignmentStatement
+ // ------------------------------------------
+ // Treat function call as AssignmentStatement or MultiAssignmentStatement
+ // For backward compatibility and also since the behavior of foo() * A + foo() ... where foo returns A
+ // Convert FunctionCallIdentifier(paramExprs, ..) -> source
+ | // TODO: Throw an informative error if user doesnot provide the optional assignment
+ ( targetList+=dataIdentifier '=' )? name=ID OPEN_PAREN (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? CLOSE_PAREN NEWLINE # FunctionCallAssignmentStatement
+ | OPEN_BRACK targetList+=dataIdentifier (',' targetList+=dataIdentifier)* CLOSE_BRACK '=' name=ID OPEN_PAREN (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? CLOSE_PAREN NEWLINE # FunctionCallMultiAssignmentStatement
+ // {notifyErrorListeners("Too many parentheses");}
+ // We don't support block statement
+ // | '{' body+=expression ';'* ( body+=expression ';'* )* '}' # BlockStatement
+ // ------------------------------------------
+ | targetList+=dataIdentifier '=' source=expression NEWLINE # AssignmentStatement
+ // IfStatement
+ // | 'if' OPEN_PAREN predicate=expression CLOSE_PAREN (ifBody+=statement ';'* | NEWLINE INDENT (ifBody+=statement)+ DEDENT ) ('else' (elseBody+=statement ';'* | '{' (elseBody+=statement ';'*)* '}'))? # IfStatement
+ | 'if' (OPEN_PAREN predicate=expression CLOSE_PAREN | predicate=expression) ':' NEWLINE INDENT (ifBody+=statement)+ DEDENT ('else' ':' NEWLINE INDENT (elseBody+=statement)+ DEDENT )? # IfStatement
+ // ------------------------------------------
+ // ForStatement & ParForStatement
+ | 'for' (OPEN_PAREN iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* CLOSE_PAREN | iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* ) ':' NEWLINE INDENT (body+=statement)+ DEDENT # ForStatement
+ // Convert strictParameterizedExpression to HashMap<String, String> for parForParams
+ | 'parfor' (OPEN_PAREN iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* CLOSE_PAREN | iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* ) ':' NEWLINE INDENT (body+=statement)+ DEDENT # ParForStatement
+ | 'while' ( OPEN_PAREN predicate=expression CLOSE_PAREN | predicate=expression ) ':' NEWLINE INDENT (body+=statement)+ DEDENT # WhileStatement
+ // ------------------------------------------
+ | NEWLINE #IgnoreNewLine
+;
+
+iterablePredicate returns [ ExpressionInfo info ]
+ @init {
+ // This actions occurs regardless of how many alternatives in this rule
+ $info = new ExpressionInfo();
+ } :
+ from=expression ':' to=expression #IterablePredicateColonExpression
+ | ID OPEN_PAREN from=expression ',' to=expression ',' increment=expression CLOSE_PAREN #IterablePredicateSeqExpression
+ ;
+
+functionStatement returns [ StatementInfo info ]
+@init {
+ // This actions occurs regardless of how many alternatives in this rule
+ $info = new StatementInfo();
+} :
+ // ------------------------------------------
+ // FunctionStatement & ExternalFunctionStatement
+ // small change: only allow typed arguments here ... instead of data identifier
+ 'def' name=ID OPEN_PAREN ( inputParams+=typedArgNoAssign (',' inputParams+=typedArgNoAssign)* )? CLOSE_PAREN ( '->' OPEN_PAREN ( outputParams+=typedArgNoAssign (',' outputParams+=typedArgNoAssign)* )? CLOSE_PAREN )? ':' NEWLINE INDENT (body+=statement)+ DEDENT # InternalFunctionDefExpression
+ | 'defExternal' name=ID OPEN_PAREN ( inputParams+=typedArgNoAssign (',' inputParams+=typedArgNoAssign)* )? CLOSE_PAREN ( '->' OPEN_PAREN ( outputParams+=typedArgNoAssign (',' outputParams+=typedArgNoAssign)* )? CLOSE_PAREN )? 'implemented' 'in' OPEN_PAREN ( otherParams+=strictParameterizedKeyValueString (',' otherParams+=strictParameterizedKeyValueString)* )? CLOSE_PAREN NEWLINE # ExternalFunctionDefExpression
+ // ------------------------------------------
+;
+
+
+// Other data identifiers are typedArgNoAssign, parameterizedExpression and strictParameterizedExpression
+dataIdentifier returns [ ExpressionInfo dataInfo ]
+@init {
+ // This actions occurs regardless of how many alternatives in this rule
+ $dataInfo = new ExpressionInfo();
+ // $dataInfo.expr = new org.apache.sysml.parser.DataIdentifier();
+} :
+ // ------------------------------------------
+ // IndexedIdentifier
+ name=ID OPEN_BRACK (rowLower=expression (':' rowUpper=expression)?)? ',' (colLower=expression (':' colUpper=expression)?)? CLOSE_BRACK # IndexedExpression
+ // ------------------------------------------
+ | ID # SimpleDataIdentifierExpression
+ | COMMANDLINE_NAMED_ID # CommandlineParamExpression
+ | COMMANDLINE_POSITION_ID # CommandlinePositionExpression
+;
+expression returns [ ExpressionInfo info ]
+@init {
+ // This actions occurs regardless of how many alternatives in this rule
+ $info = new ExpressionInfo();
+ // $info.expr = new org.apache.sysml.parser.BinaryExpression(org.apache.sysml.parser.Expression.BinaryOp.INVALID);
+} :
+ // ------------------------------------------
+ // BinaryExpression
+ // power
+ <assoc=right> left=expression op='**' right=expression # PowerExpression
+ // unary plus and minus
+ | op=('-'|'+') left=expression # UnaryExpression
+ // sequence - since we are only using this into for loop => Array not supported
+ //| left=expression op=':' right=expression # SequenceExpression
+ // matrix multiply
+ // | left=expression op='*' right=expression # MatrixMulExpression
+ // modulus and integer division
+ | left=expression op=('//' | '%' ) right=expression # ModIntDivExpression
+ // arithmetic multiply and divide
+ | left=expression op=('*'|'/') right=expression # MultDivExpression
+ // arithmetic addition and subtraction
+ | left=expression op=('+'|'-') right=expression # AddSubExpression
+ // ------------------------------------------
+ // RelationalExpression
+ | left=expression op=('>'|'>='|'<'|'<='|'=='|'!=') right=expression # RelationalExpression
+ // ------------------------------------------
+ // BooleanExpression
+ // boolean not
+ | op='!' left=expression # BooleanNotExpression
+ // boolean and
+ | left=expression op=('&'|'and') right=expression # BooleanAndExpression
+ // boolean or
+ | left=expression op=('|'|'or') right=expression # BooleanOrExpression
+
+ // ---------------------------------
+ // only applicable for builtin function expressions
+ // Add following additional functions and check number of parameters:
+ // power, full, matrix, reshape, dot
+ // Also take care whether there is y.transpose() => which sometinamespace
+ | name=ID OPEN_PAREN (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? CLOSE_PAREN ';'* # BuiltinFunctionExpression
+
+ // 4. Atomic
+ | OPEN_PAREN left=expression CLOSE_PAREN # AtomicExpression
+
+ // Should you allow indexed expression here ?
+ // | OPEN_BRACK targetList+=expression (',' targetList+=expression)* CLOSE_BRACK # MultiIdExpression
+
+ // | BOOLEAN # ConstBooleanIdExpression
+ | 'True' # ConstTrueExpression
+ | 'False' # ConstFalseExpression
+ | INT # ConstIntIdExpression
+ | DOUBLE # ConstDoubleIdExpression
+ | STRING # ConstStringIdExpression
+ | dataIdentifier # DataIdExpression
+ // Special
+ // | 'NULL' | 'NA' | 'Inf' | 'NaN'
+;
+
+typedArgNoAssign : paramName=ID ':' paramType=ml_type ;
+parameterizedExpression : (paramName=ID '=')? paramVal=expression;
+strictParameterizedExpression : paramName=ID '=' paramVal=expression ;
+strictParameterizedKeyValueString : paramName=ID '=' paramVal=STRING ;
+// sometimes this is matrix object and sometimes its namespace
+ID : (ALPHABET (ALPHABET|DIGIT|'_')* '.')? ALPHABET (ALPHABET|DIGIT|'_')*
+ // Special ID cases:
+ // | 'matrix' // --> This is a special case which causes lot of headache
+ // | 'scalar' | 'float' | 'int' | 'bool' // corresponds to as.scalar, as.double, as.integer and as.logical
+ | 'index.return'
+;
+// Unfortunately, we have datatype name clashing with builtin function name: matrix :(
+// Therefore, ugly work around for checking datatype
+ml_type : valueType | dataType OPEN_BRACK valueType CLOSE_BRACK;
+// Note to reduce number of keywords, these are case-sensitive,
+// To allow case-insenstive, 'int' becomes: ('i' | 'I') ('n' | 'N') ('t' | 'T')
+valueType:
+ ID # ValueDataTypeCheck
+ // 'int' | 'str' | 'bool' | 'float'
+;
+dataType:
+ // 'scalar' # ScalarDataTypeDummyCheck
+ // |
+ ID # MatrixDataTypeCheck //{ if($ID.text.compareTo("matrix") != 0) { notifyErrorListeners("incorrect datatype"); } }
+ //| 'matrix' //---> See ID, this causes lot of headache
+ ;
+INT : DIGIT+ [Ll]?;
+// BOOLEAN : 'TRUE' | 'FALSE';
+DOUBLE: DIGIT+ '.' DIGIT* EXP? [Ll]?
+| DIGIT+ EXP? [Ll]?
+| '.' DIGIT+ EXP? [Ll]?
+;
+DIGIT: '0'..'9';
+ALPHABET : [a-zA-Z] ;
+fragment EXP : ('E' | 'e') ('+' | '-')? INT ;
+COMMANDLINE_NAMED_ID: '$' ALPHABET (ALPHABET|DIGIT|'_')*;
+COMMANDLINE_POSITION_ID: '$' DIGIT+;
+
+// supports single and double quoted string with escape characters
+STRING: '"' ( ESC | ~[\\"] )*? '"' | '\'' ( ESC | ~[\\'] )*? '\'';
+fragment ESC : '\\' [abtnfrv"'\\] ;
+// Comments, whitespaces and new line
+// LINE_COMMENT : '#' .*? '\r'? '\n' -> skip ;
+// MULTILINE_BLOCK_COMMENT : '/*' .*? '*/' -> skip ;
+// WHITESPACE : (' ' | '\r' | '\n')+ -> skip ;
+
+OPEN_BRACK : '[' {opened++;};
+CLOSE_BRACK : ']' {opened--;};
+OPEN_PAREN : '(' {opened++;};
+CLOSE_PAREN : ')' {opened--;};
+// OPEN_BRACE : '{' {opened++;};
+// CLOSE_BRACE : '}' {opened--;};
+
+fragment SPACES : [ \t]+ ;
+fragment COMMENT : '#' ~[\r\n]* ;
+fragment LINE_JOINING : '\\' SPACES? ( '\r'? '\n' | '\r' ) ;
+
+NEWLINE : ( '\r'? '\n' | '\r' ) SPACES?
+{
+ String newLine = getText().replaceAll("[^\r\n]+", "");
+ String spaces = getText().replaceAll("[\r\n]+", "");
+ int next = _input.LA(1);
+ if (opened > 0 || next == '\r' || next == '\n' || next == '#') {
+ // If we're inside a list or on a blank line, ignore all indents,
+ // dedents and line breaks.
+ skip();
+ if(debugIndentRules) {
+ if(next == '\r' || next == '\n') {
+ System.out.println("4.1 Skipping (blank lines)");
+ }
+ else if(next == '#') {
+ System.out.println("4.2 Skipping (comment)");
+ }
+ else {
+ System.out.println("4.2 Skipping something else");
+ }
+ }
+ }
+ else {
+ emit(commonToken(NEWLINE, newLine));
+
+ int indent = getIndentationCount(spaces);
+ int previous = indents.isEmpty() ? 0 : indents.peek();
+ if (indent == previous) {
+ if(debugIndentRules)
+ System.out.println("3. Skipping identation as of same size:" + next);
+
+ // skip indents of the same size as the present indent-size
+ skip();
+ }
+ else if (indent > previous) {
+ if(debugIndentRules)
+ System.out.println("1. Indent:" + next);
+
+ indents.push(indent);
+ emit(commonToken(PydmlParser.INDENT, spaces));
+ }
+ else {
+ // Possibly emit more than 1 DEDENT token.
+ while(!indents.isEmpty() && indents.peek() > indent) {
+ if(debugIndentRules)
+ System.out.println("2. Dedent:" + next);
+
+ this.emit(createDedent());
+ indents.pop();
+ }
+ }
+ }
+}
+;
+
+SKIP : ( SPACES | COMMENT | LINE_JOINING ) -> skip ;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticErrorListener.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticErrorListener.java b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticErrorListener.java
new file mode 100644
index 0000000..8b2cc34
--- /dev/null
+++ b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticErrorListener.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.parser.pydml;
+
+import java.util.Stack;
+
+import org.antlr.v4.runtime.BaseErrorListener;
+import org.antlr.v4.runtime.RecognitionException;
+import org.antlr.v4.runtime.Recognizer;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.sysml.api.DMLScript;
+
+public class PydmlSyntacticErrorListener
+{
+ private static final Log LOG = LogFactory.getLog(DMLScript.class.getName());
+
+ public static class CustomDmlErrorListener extends BaseErrorListener {
+
+ private boolean atleastOneError = false;
+ private Stack<String> currentFileName = new Stack<String>();
+
+ public void pushCurrentFileName(String currentFilePath) {
+ currentFileName.push(currentFilePath);
+ }
+
+ public String peekFileName() {
+ return currentFileName.peek();
+ }
+
+ public String popFileName() {
+ return currentFileName.pop();
+ }
+
+ public void validationError(int line, int charPositionInLine, String msg) {
+ try {
+ setAtleastOneError(true);
+ // Print error messages with file name
+ if(currentFileName == null || currentFileName.empty()) {
+ LOG.error("line "+line+":"+charPositionInLine+" "+msg);
+ }
+ else {
+ String fileName = currentFileName.peek();
+ LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg);
+ }
+ }
+ catch(Exception e1) {
+ LOG.error("ERROR: while customizing error message:" + e1);
+ }
+ }
+
+ public void validationWarning(int line, int charPositionInLine, String msg) {
+ try {
+ //atleastOneError = true; ---> not an error, just warning
+ // Print error messages with file name
+ if(currentFileName == null || currentFileName.empty())
+ LOG.warn("line "+line+":"+charPositionInLine+" "+msg);
+ else {
+ String fileName = currentFileName.peek();
+ LOG.warn(fileName + " line "+line+":"+charPositionInLine+" "+msg);
+ }
+ }
+ catch(Exception e1) {
+ LOG.warn("ERROR: while customizing error message:" + e1);
+ }
+ }
+
+ @Override
+ public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol,
+ int line, int charPositionInLine,
+ String msg, RecognitionException e)
+ {
+ try {
+ setAtleastOneError(true);
+ // Print error messages with file name
+ if(currentFileName == null || currentFileName.empty())
+ LOG.error("line "+line+":"+charPositionInLine+" "+msg);
+ else {
+ String fileName = currentFileName.peek();
+ LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg);
+ }
+ }
+ catch(Exception e1) {
+ LOG.error("ERROR: while customizing error message:" + e1);
+ }
+ }
+
+ public boolean isAtleastOneError() {
+ return atleastOneError;
+ }
+
+ public void setAtleastOneError(boolean atleastOneError) {
+ this.atleastOneError = atleastOneError;
+ }
+ }
+}
[3/8] incubator-systemml git commit: [SYSTEMML-148] Refactored
.parser.{antlr4, python} to .parser.{dml, pydml}.
Posted by du...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/pydml/StatementInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/pydml/StatementInfo.java b/src/main/java/org/apache/sysml/parser/pydml/StatementInfo.java
new file mode 100644
index 0000000..c19f1cd
--- /dev/null
+++ b/src/main/java/org/apache/sysml/parser/pydml/StatementInfo.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.parser.pydml;
+
+/**
+ * This class exists solely to prevent compiler warnings.
+ *
+ * <p>
+ * The ExpressionInfo and StatementInfo classes are shared among both parsers
+ * (R-like and Python-like dialects), and Antlr-generated code assumes that
+ * these classes are present in the parser's namespace.
+ */
+class StatementInfo extends org.apache.sysml.parser.dml.StatementInfo {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/python/ExpressionInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/ExpressionInfo.java b/src/main/java/org/apache/sysml/parser/python/ExpressionInfo.java
deleted file mode 100644
index 71962cf..0000000
--- a/src/main/java/org/apache/sysml/parser/python/ExpressionInfo.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.parser.python;
-
-/**
- * This class exists solely to prevent compiler warnings.
- *
- * <p>
- * The ExpressionInfo and StatementInfo classes are shared among both parsers
- * (R-like and Python-like dialects), and Antlr-generated code assumes that
- * these classes are present in the parser's namespace.
- */
-class ExpressionInfo extends org.apache.sysml.parser.antlr4.ExpressionInfo {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java b/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java
deleted file mode 100644
index e52bb73..0000000
--- a/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java
+++ /dev/null
@@ -1,283 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.parser.python;
-
-import java.io.ByteArrayInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.antlr.v4.runtime.ANTLRInputStream;
-import org.antlr.v4.runtime.BailErrorStrategy;
-import org.antlr.v4.runtime.CommonTokenStream;
-import org.antlr.v4.runtime.DefaultErrorStrategy;
-import org.antlr.v4.runtime.atn.PredictionMode;
-import org.antlr.v4.runtime.misc.ParseCancellationException;
-import org.antlr.v4.runtime.tree.ParseTree;
-import org.antlr.v4.runtime.tree.ParseTreeWalker;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.sysml.api.DMLScript;
-import org.apache.sysml.parser.AParserWrapper;
-import org.apache.sysml.parser.DMLProgram;
-import org.apache.sysml.parser.ForStatement;
-import org.apache.sysml.parser.ForStatementBlock;
-import org.apache.sysml.parser.FunctionStatementBlock;
-import org.apache.sysml.parser.IfStatement;
-import org.apache.sysml.parser.IfStatementBlock;
-import org.apache.sysml.parser.ImportStatement;
-import org.apache.sysml.parser.LanguageException;
-import org.apache.sysml.parser.ParForStatement;
-import org.apache.sysml.parser.ParForStatementBlock;
-import org.apache.sysml.parser.ParseException;
-import org.apache.sysml.parser.Statement;
-import org.apache.sysml.parser.StatementBlock;
-import org.apache.sysml.parser.WhileStatement;
-import org.apache.sysml.parser.WhileStatementBlock;
-import org.apache.sysml.parser.antlr4.DMLParserWrapper;
-import org.apache.sysml.parser.python.PydmlParser.FunctionStatementContext;
-import org.apache.sysml.parser.python.PydmlParser.PmlprogramContext;
-import org.apache.sysml.parser.python.PydmlParser.StatementContext;
-import org.apache.sysml.parser.python.PydmlSyntacticErrorListener.CustomDmlErrorListener;
-
-/**
- * Logic of this wrapper is similar to DMLParserWrapper.
- *
- * Note: ExpressionInfo and StatementInfo are simply wrapper objects and are reused in both DML and PyDML parsers.
- *
- */
-public class PyDMLParserWrapper extends AParserWrapper
-{
- private static final Log LOG = LogFactory.getLog(DMLScript.class.getName());
-
- /**
- * Custom wrapper to convert statement into statement blocks. Called by doParse and in PydmlSyntacticValidator for for, parfor, while, ...
- * @param current a statement
- * @return corresponding statement block
- */
- public static StatementBlock getStatementBlock(org.apache.sysml.parser.Statement current) {
- StatementBlock blk = null;
- if(current instanceof ParForStatement) {
- blk = new ParForStatementBlock();
- blk.addStatement(current);
- }
- else if(current instanceof ForStatement) {
- blk = new ForStatementBlock();
- blk.addStatement(current);
- }
- else if(current instanceof IfStatement) {
- blk = new IfStatementBlock();
- blk.addStatement(current);
- }
- else if(current instanceof WhileStatement) {
- blk = new WhileStatementBlock();
- blk.addStatement(current);
- }
- else {
- // This includes ImportStatement
- blk = new StatementBlock();
- blk.addStatement(current);
- }
- return blk;
- }
-
- /**
- * Parses the passed file with command line parameters. You can either pass both (local file) or just dmlScript (hdfs) or just file name (import command)
- * @param fileName either full path or null --> only used for better error handling
- * @param dmlScript required
- * @param argVals
- * @return
- * @throws ParseException
- */
- @Override
- public DMLProgram parse(String fileName, String dmlScript, HashMap<String,String> argVals) throws ParseException {
- DMLProgram prog = null;
-
- if(dmlScript == null || dmlScript.trim().isEmpty()) {
- throw new ParseException("Incorrect usage of parse. Please pass dmlScript not just filename");
- }
-
- // Set the pipeline required for ANTLR parsing
- PyDMLParserWrapper parser = new PyDMLParserWrapper();
- prog = parser.doParse(fileName, dmlScript, argVals);
-
- if(prog == null) {
- throw new ParseException("One or more errors found during parsing. (could not construct AST for file: " + fileName + "). Cannot proceed ahead.");
- }
- return prog;
- }
-
- /**
- * This function is supposed to be called directly only from PydmlSyntacticValidator when it encounters 'import'
- * @param fileName
- * @return null if atleast one error
- */
- public DMLProgram doParse(String fileName, String dmlScript, HashMap<String,String> argVals) throws ParseException {
- DMLProgram dmlPgm = null;
-
- ANTLRInputStream in;
- try {
- if(dmlScript == null) {
- dmlScript = DMLParserWrapper.readDMLScript(fileName);
- }
-
- InputStream stream = new ByteArrayInputStream(dmlScript.getBytes());
- in = new org.antlr.v4.runtime.ANTLRInputStream(stream);
- }
- catch (FileNotFoundException e) {
- throw new ParseException("ERROR: Cannot find file:" + fileName, e);
- }
- catch (IOException e) {
- throw new ParseException("ERROR: Cannot open file:" + fileName, e);
- }
- catch (LanguageException e) {
- throw new ParseException("ERROR: " + e.getMessage(), e);
- }
-
- PmlprogramContext ast = null;
- CustomDmlErrorListener errorListener = new CustomDmlErrorListener();
-
- try {
- PydmlLexer lexer = new PydmlLexer(in);
- CommonTokenStream tokens = new CommonTokenStream(lexer);
- PydmlParser antlr4Parser = new PydmlParser(tokens);
-
- boolean tryOptimizedParsing = false; // For now no optimization, since it is not able to parse integer value.
-
- if(tryOptimizedParsing) {
- // Try faster and simpler SLL
- antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
- antlr4Parser.removeErrorListeners();
- antlr4Parser.setErrorHandler(new BailErrorStrategy());
- try{
- ast = antlr4Parser.pmlprogram();
- // If successful, no need to try out full LL(*) ... SLL was enough
- }
- catch(ParseCancellationException ex) {
- // Error occurred, so now try full LL(*) for better error messages
- tokens.reset();
- antlr4Parser.reset();
- if(fileName != null) {
- errorListener.pushCurrentFileName(fileName);
- }
- else {
- errorListener.pushCurrentFileName("MAIN_SCRIPT");
- }
- // Set our custom error listener
- antlr4Parser.addErrorListener(errorListener);
- antlr4Parser.setErrorHandler(new DefaultErrorStrategy());
- antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL);
- ast = antlr4Parser.pmlprogram();
- }
- }
- else {
- // Set our custom error listener
- antlr4Parser.removeErrorListeners();
- antlr4Parser.addErrorListener(errorListener);
- errorListener.pushCurrentFileName(fileName);
-
- // Now do the parsing
- ast = antlr4Parser.pmlprogram();
- }
- }
- catch(Exception e) {
- throw new ParseException("ERROR: Cannot parse the program:" + fileName, e);
- }
-
-
- try {
- // Now convert the parse tree into DMLProgram
- // Do syntactic validation while converting
- ParseTree tree = ast;
- // And also do syntactic validation
- ParseTreeWalker walker = new ParseTreeWalker();
- PydmlSyntacticValidatorHelper helper = new PydmlSyntacticValidatorHelper(errorListener);
- PydmlSyntacticValidator validator = new PydmlSyntacticValidator(helper, fileName, argVals);
- walker.walk(validator, tree);
- errorListener.popFileName();
- if(errorListener.isAtleastOneError()) {
- return null;
- }
- dmlPgm = createDMLProgram(ast);
- }
- catch(Exception e) {
- throw new ParseException("ERROR: Cannot translate the parse tree into DMLProgram" + e.getMessage(), e);
- }
-
- return dmlPgm;
- }
-
-
- private DMLProgram createDMLProgram(PmlprogramContext ast) {
-
- DMLProgram dmlPgm = new DMLProgram();
-
- // First add all the functions
- for(FunctionStatementContext fn : ast.functionBlocks) {
- FunctionStatementBlock functionStmtBlk = new FunctionStatementBlock();
- functionStmtBlk.addStatement(fn.info.stmt);
- try {
- // TODO: currently the logic of nested namespace is not clear.
- String namespace = DMLProgram.DEFAULT_NAMESPACE;
- dmlPgm.addFunctionStatementBlock(namespace, fn.info.functionName, functionStmtBlk);
- } catch (LanguageException e) {
- LOG.error("line: " + fn.start.getLine() + ":" + fn.start.getCharPositionInLine() + " cannot process the function " + fn.info.functionName);
- return null;
- }
- }
-
- // Then add all the statements
- for(StatementContext stmtCtx : ast.blocks) {
- Statement current = stmtCtx.info.stmt;
- if(current == null) {
- LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the statement");
- return null;
- }
-
- // Ignore Newline logic
- if(current.isEmptyNewLineStatement()) {
- continue;
- }
-
- if(current instanceof ImportStatement) {
- // Handle import statements separately
- if(stmtCtx.info.namespaces != null) {
- // Add the DMLProgram entries into current program
- for(Map.Entry<String, DMLProgram> entry : stmtCtx.info.namespaces.entrySet()) {
- dmlPgm.getNamespaces().put(entry.getKey(), entry.getValue());
- }
- }
- else {
- LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the import statement");
- return null;
- }
- }
-
- // Now wrap statement into individual statement block
- // merge statement will take care of merging these blocks
- dmlPgm.addStatementBlock(getStatementBlock(current));
- }
-
- dmlPgm.mergeStatementBlocks();
- return dmlPgm;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/python/Pydml.g4
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/Pydml.g4 b/src/main/java/org/apache/sysml/parser/python/Pydml.g4
deleted file mode 100644
index 7fbe42d..0000000
--- a/src/main/java/org/apache/sysml/parser/python/Pydml.g4
+++ /dev/null
@@ -1,384 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-grammar Pydml;
-
-@header
-{
- // package org.apache.sysml.python;
- //import org.apache.sysml.parser.antlr4.StatementInfo;
- //import org.apache.sysml.parser.antlr4.ExpressionInfo;
-}
-
-// This antlr grammar is based on Python 3.3 language reference: https://docs.python.org/3.3/reference/grammar.html
-
-tokens { INDENT, DEDENT }
-
-@lexer::members {
- private boolean debugIndentRules = false;
-
- // Indentation level stack
- private java.util.Stack<Integer> indents = new java.util.Stack<Integer>();
-
- // Extra tokens queue (see the NEWLINE rule).
- private java.util.Queue<Token> tokens = new java.util.LinkedList<Token>();
-
- // Number of opened braces, brackets and parenthesis.
- private int opened = 0;
-
- // This is only used to set the line number for dedent
- private Token lastToken = null;
-
-
- @Override
- public void emit(Token t) {
- if(debugIndentRules)
- System.out.println("Emitted token:" + t);
-
- super.setToken(t);
- tokens.offer(t);
- }
-
-
- @Override
- public Token nextToken() {
- if (_input.LA(1) == EOF && !this.indents.isEmpty()) {
- if(debugIndentRules)
- System.out.println("EOF reached and expecting some DEDENTS, so emitting them");
-
- tokens.poll();
- this.emit(commonToken(PydmlParser.NEWLINE, "\n"));
-
- // Now emit as much DEDENT tokens as needed.
- while (!indents.isEmpty()) {
- if(debugIndentRules)
- System.out.println("Emitting (inserted) DEDENTS");
-
- this.emit(createDedent());
- indents.pop();
- }
- // Put the EOF back on the token stream.
- this.emit(commonToken(PydmlParser.EOF, "<EOF>"));
- }
- Token next = super.nextToken();
- if (next.getChannel() == Token.DEFAULT_CHANNEL) {
- // Keep track of the last token on the default channel.
- this.lastToken = next;
- }
- Token retVal = tokens.isEmpty() ? next : tokens.poll();
-
- if(debugIndentRules)
- System.out.println("Returning nextToken: [" + retVal + "]<<" + tokens.isEmpty());
-
- return retVal;
- }
-
- private Token createDedent() {
- CommonToken dedent = commonToken(PydmlParser.DEDENT, "");
- dedent.setLine(this.lastToken.getLine());
- return dedent;
- }
-
- private CommonToken commonToken(int type, String text) {
- // Nike: Main change: This logic was screwed up and was emitting additional 3 characters, so commenting it for now.
- // int start = this.getCharIndex();
- // int stop = start + text.length();
- // return new CommonToken(this._tokenFactorySourcePair, type, DEFAULT_TOKEN_CHANNEL, start, stop);
- return new CommonToken(type, text); // Main change
- }
-
- // Calculates the indentation level from the spaces:
- // "Tabs are replaced (from left to right) by one to eight spaces
- // such that the total number of characters up to and including
- // the replacement is a multiple of eight [...]"
- // https://docs.python.org/3.1/reference/lexical_analysis.html#indentation
- static int getIndentationCount(String spaces) {
- int count = 0;
- for (char ch : spaces.toCharArray()) {
- switch (ch) {
- case '\t':
- count += 8 - (count % 8);
- break;
- default:
- // A normal space char.
- count++;
- }
- }
- return count;
- }
-}
-
-
-// 2. Modify this g4 by comparing it with Java:
-// - https://pythonconquerstheuniverse.wordpress.com/2009/10/03/python-java-a-side-by-side-comparison/
-// - http://www.cs.gordon.edu/courses/cps122/handouts-2014/From%20Python%20to%20Java%20Lecture/A%20Comparison%20of%20the%20Syntax%20of%20Python%20and%20Java.pdf
-// - http://cs.joensuu.fi/~pviktor/python/slides/cheatsheet.pdf
-// - http://www.interfaceware.com/manual/chameleon/scripts/quickreference.pdf
-
-// DML Program is a list of expression
-// For now, we only allow global function definitions (not nested or inside a while block)
-pmlprogram: (blocks+=statement | functionBlocks+=functionStatement)* NEWLINE* EOF;
-
-
-
-statement returns [ StatementInfo info ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new StatementInfo();
-} :
- // ------------------------------------------
- // ImportStatement
- 'source' OPEN_PAREN filePath = STRING CLOSE_PAREN 'as' namespace=ID NEWLINE # ImportStatement
- | 'setwd' OPEN_PAREN pathValue = STRING CLOSE_PAREN NEWLINE # PathStatement
- // ------------------------------------------
- // AssignmentStatement
- | targetList+=dataIdentifier '=' 'ifdef' OPEN_PAREN commandLineParam=dataIdentifier ',' source=expression CLOSE_PAREN NEWLINE # IfdefAssignmentStatement
- // ------------------------------------------
- // Treat function call as AssignmentStatement or MultiAssignmentStatement
- // For backward compatibility and also since the behavior of foo() * A + foo() ... where foo returns A
- // Convert FunctionCallIdentifier(paramExprs, ..) -> source
- | // TODO: Throw an informative error if user doesnot provide the optional assignment
- ( targetList+=dataIdentifier '=' )? name=ID OPEN_PAREN (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? CLOSE_PAREN NEWLINE # FunctionCallAssignmentStatement
- | OPEN_BRACK targetList+=dataIdentifier (',' targetList+=dataIdentifier)* CLOSE_BRACK '=' name=ID OPEN_PAREN (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? CLOSE_PAREN NEWLINE # FunctionCallMultiAssignmentStatement
- // {notifyErrorListeners("Too many parentheses");}
- // We don't support block statement
- // | '{' body+=expression ';'* ( body+=expression ';'* )* '}' # BlockStatement
- // ------------------------------------------
- | targetList+=dataIdentifier '=' source=expression NEWLINE # AssignmentStatement
- // IfStatement
- // | 'if' OPEN_PAREN predicate=expression CLOSE_PAREN (ifBody+=statement ';'* | NEWLINE INDENT (ifBody+=statement)+ DEDENT ) ('else' (elseBody+=statement ';'* | '{' (elseBody+=statement ';'*)* '}'))? # IfStatement
- | 'if' (OPEN_PAREN predicate=expression CLOSE_PAREN | predicate=expression) ':' NEWLINE INDENT (ifBody+=statement)+ DEDENT ('else' ':' NEWLINE INDENT (elseBody+=statement)+ DEDENT )? # IfStatement
- // ------------------------------------------
- // ForStatement & ParForStatement
- | 'for' (OPEN_PAREN iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* CLOSE_PAREN | iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* ) ':' NEWLINE INDENT (body+=statement)+ DEDENT # ForStatement
- // Convert strictParameterizedExpression to HashMap<String, String> for parForParams
- | 'parfor' (OPEN_PAREN iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* CLOSE_PAREN | iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* ) ':' NEWLINE INDENT (body+=statement)+ DEDENT # ParForStatement
- | 'while' ( OPEN_PAREN predicate=expression CLOSE_PAREN | predicate=expression ) ':' NEWLINE INDENT (body+=statement)+ DEDENT # WhileStatement
- // ------------------------------------------
- | NEWLINE #IgnoreNewLine
-;
-
-iterablePredicate returns [ ExpressionInfo info ]
- @init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new ExpressionInfo();
- } :
- from=expression ':' to=expression #IterablePredicateColonExpression
- | ID OPEN_PAREN from=expression ',' to=expression ',' increment=expression CLOSE_PAREN #IterablePredicateSeqExpression
- ;
-
-functionStatement returns [ StatementInfo info ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new StatementInfo();
-} :
- // ------------------------------------------
- // FunctionStatement & ExternalFunctionStatement
- // small change: only allow typed arguments here ... instead of data identifier
- 'def' name=ID OPEN_PAREN ( inputParams+=typedArgNoAssign (',' inputParams+=typedArgNoAssign)* )? CLOSE_PAREN ( '->' OPEN_PAREN ( outputParams+=typedArgNoAssign (',' outputParams+=typedArgNoAssign)* )? CLOSE_PAREN )? ':' NEWLINE INDENT (body+=statement)+ DEDENT # InternalFunctionDefExpression
- | 'defExternal' name=ID OPEN_PAREN ( inputParams+=typedArgNoAssign (',' inputParams+=typedArgNoAssign)* )? CLOSE_PAREN ( '->' OPEN_PAREN ( outputParams+=typedArgNoAssign (',' outputParams+=typedArgNoAssign)* )? CLOSE_PAREN )? 'implemented' 'in' OPEN_PAREN ( otherParams+=strictParameterizedKeyValueString (',' otherParams+=strictParameterizedKeyValueString)* )? CLOSE_PAREN NEWLINE # ExternalFunctionDefExpression
- // ------------------------------------------
-;
-
-
-// Other data identifiers are typedArgNoAssign, parameterizedExpression and strictParameterizedExpression
-dataIdentifier returns [ ExpressionInfo dataInfo ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $dataInfo = new ExpressionInfo();
- // $dataInfo.expr = new org.apache.sysml.parser.DataIdentifier();
-} :
- // ------------------------------------------
- // IndexedIdentifier
- name=ID OPEN_BRACK (rowLower=expression (':' rowUpper=expression)?)? ',' (colLower=expression (':' colUpper=expression)?)? CLOSE_BRACK # IndexedExpression
- // ------------------------------------------
- | ID # SimpleDataIdentifierExpression
- | COMMANDLINE_NAMED_ID # CommandlineParamExpression
- | COMMANDLINE_POSITION_ID # CommandlinePositionExpression
-;
-expression returns [ ExpressionInfo info ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new ExpressionInfo();
- // $info.expr = new org.apache.sysml.parser.BinaryExpression(org.apache.sysml.parser.Expression.BinaryOp.INVALID);
-} :
- // ------------------------------------------
- // BinaryExpression
- // power
- <assoc=right> left=expression op='**' right=expression # PowerExpression
- // unary plus and minus
- | op=('-'|'+') left=expression # UnaryExpression
- // sequence - since we are only using this into for loop => Array not supported
- //| left=expression op=':' right=expression # SequenceExpression
- // matrix multiply
- // | left=expression op='*' right=expression # MatrixMulExpression
- // modulus and integer division
- | left=expression op=('//' | '%' ) right=expression # ModIntDivExpression
- // arithmetic multiply and divide
- | left=expression op=('*'|'/') right=expression # MultDivExpression
- // arithmetic addition and subtraction
- | left=expression op=('+'|'-') right=expression # AddSubExpression
- // ------------------------------------------
- // RelationalExpression
- | left=expression op=('>'|'>='|'<'|'<='|'=='|'!=') right=expression # RelationalExpression
- // ------------------------------------------
- // BooleanExpression
- // boolean not
- | op='!' left=expression # BooleanNotExpression
- // boolean and
- | left=expression op=('&'|'and') right=expression # BooleanAndExpression
- // boolean or
- | left=expression op=('|'|'or') right=expression # BooleanOrExpression
-
- // ---------------------------------
- // only applicable for builtin function expressions
- // Add following additional functions and check number of parameters:
- // power, full, matrix, reshape, dot
- // Also take care whether there is y.transpose() => which sometinamespace
- | name=ID OPEN_PAREN (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? CLOSE_PAREN ';'* # BuiltinFunctionExpression
-
- // 4. Atomic
- | OPEN_PAREN left=expression CLOSE_PAREN # AtomicExpression
-
- // Should you allow indexed expression here ?
- // | OPEN_BRACK targetList+=expression (',' targetList+=expression)* CLOSE_BRACK # MultiIdExpression
-
- // | BOOLEAN # ConstBooleanIdExpression
- | 'True' # ConstTrueExpression
- | 'False' # ConstFalseExpression
- | INT # ConstIntIdExpression
- | DOUBLE # ConstDoubleIdExpression
- | STRING # ConstStringIdExpression
- | dataIdentifier # DataIdExpression
- // Special
- // | 'NULL' | 'NA' | 'Inf' | 'NaN'
-;
-
-typedArgNoAssign : paramName=ID ':' paramType=ml_type ;
-parameterizedExpression : (paramName=ID '=')? paramVal=expression;
-strictParameterizedExpression : paramName=ID '=' paramVal=expression ;
-strictParameterizedKeyValueString : paramName=ID '=' paramVal=STRING ;
-// sometimes this is matrix object and sometimes its namespace
-ID : (ALPHABET (ALPHABET|DIGIT|'_')* '.')? ALPHABET (ALPHABET|DIGIT|'_')*
- // Special ID cases:
- // | 'matrix' // --> This is a special case which causes lot of headache
- // | 'scalar' | 'float' | 'int' | 'bool' // corresponds to as.scalar, as.double, as.integer and as.logical
- | 'index.return'
-;
-// Unfortunately, we have datatype name clashing with builtin function name: matrix :(
-// Therefore, ugly work around for checking datatype
-ml_type : valueType | dataType OPEN_BRACK valueType CLOSE_BRACK;
-// Note to reduce number of keywords, these are case-sensitive,
-// To allow case-insenstive, 'int' becomes: ('i' | 'I') ('n' | 'N') ('t' | 'T')
-valueType:
- ID # ValueDataTypeCheck
- // 'int' | 'str' | 'bool' | 'float'
-;
-dataType:
- // 'scalar' # ScalarDataTypeDummyCheck
- // |
- ID # MatrixDataTypeCheck //{ if($ID.text.compareTo("matrix") != 0) { notifyErrorListeners("incorrect datatype"); } }
- //| 'matrix' //---> See ID, this causes lot of headache
- ;
-INT : DIGIT+ [Ll]?;
-// BOOLEAN : 'TRUE' | 'FALSE';
-DOUBLE: DIGIT+ '.' DIGIT* EXP? [Ll]?
-| DIGIT+ EXP? [Ll]?
-| '.' DIGIT+ EXP? [Ll]?
-;
-DIGIT: '0'..'9';
-ALPHABET : [a-zA-Z] ;
-fragment EXP : ('E' | 'e') ('+' | '-')? INT ;
-COMMANDLINE_NAMED_ID: '$' ALPHABET (ALPHABET|DIGIT|'_')*;
-COMMANDLINE_POSITION_ID: '$' DIGIT+;
-
-// supports single and double quoted string with escape characters
-STRING: '"' ( ESC | ~[\\"] )*? '"' | '\'' ( ESC | ~[\\'] )*? '\'';
-fragment ESC : '\\' [abtnfrv"'\\] ;
-// Comments, whitespaces and new line
-// LINE_COMMENT : '#' .*? '\r'? '\n' -> skip ;
-// MULTILINE_BLOCK_COMMENT : '/*' .*? '*/' -> skip ;
-// WHITESPACE : (' ' | '\r' | '\n')+ -> skip ;
-
-OPEN_BRACK : '[' {opened++;};
-CLOSE_BRACK : ']' {opened--;};
-OPEN_PAREN : '(' {opened++;};
-CLOSE_PAREN : ')' {opened--;};
-// OPEN_BRACE : '{' {opened++;};
-// CLOSE_BRACE : '}' {opened--;};
-
-fragment SPACES : [ \t]+ ;
-fragment COMMENT : '#' ~[\r\n]* ;
-fragment LINE_JOINING : '\\' SPACES? ( '\r'? '\n' | '\r' ) ;
-
-NEWLINE : ( '\r'? '\n' | '\r' ) SPACES?
-{
- String newLine = getText().replaceAll("[^\r\n]+", "");
- String spaces = getText().replaceAll("[\r\n]+", "");
- int next = _input.LA(1);
- if (opened > 0 || next == '\r' || next == '\n' || next == '#') {
- // If we're inside a list or on a blank line, ignore all indents,
- // dedents and line breaks.
- skip();
- if(debugIndentRules) {
- if(next == '\r' || next == '\n') {
- System.out.println("4.1 Skipping (blank lines)");
- }
- else if(next == '#') {
- System.out.println("4.2 Skipping (comment)");
- }
- else {
- System.out.println("4.2 Skipping something else");
- }
- }
- }
- else {
- emit(commonToken(NEWLINE, newLine));
-
- int indent = getIndentationCount(spaces);
- int previous = indents.isEmpty() ? 0 : indents.peek();
- if (indent == previous) {
- if(debugIndentRules)
- System.out.println("3. Skipping identation as of same size:" + next);
-
- // skip indents of the same size as the present indent-size
- skip();
- }
- else if (indent > previous) {
- if(debugIndentRules)
- System.out.println("1. Indent:" + next);
-
- indents.push(indent);
- emit(commonToken(PydmlParser.INDENT, spaces));
- }
- else {
- // Possibly emit more than 1 DEDENT token.
- while(!indents.isEmpty() && indents.peek() > indent) {
- if(debugIndentRules)
- System.out.println("2. Dedent:" + next);
-
- this.emit(createDedent());
- indents.pop();
- }
- }
- }
-}
-;
-
-SKIP : ( SPACES | COMMENT | LINE_JOINING ) -> skip ;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java
deleted file mode 100644
index 9ae63a9..0000000
--- a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.parser.python;
-
-import java.util.Stack;
-
-import org.antlr.v4.runtime.BaseErrorListener;
-import org.antlr.v4.runtime.RecognitionException;
-import org.antlr.v4.runtime.Recognizer;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.sysml.api.DMLScript;
-
-public class PydmlSyntacticErrorListener
-{
- private static final Log LOG = LogFactory.getLog(DMLScript.class.getName());
-
- public static class CustomDmlErrorListener extends BaseErrorListener {
-
- private boolean atleastOneError = false;
- private Stack<String> currentFileName = new Stack<String>();
-
- public void pushCurrentFileName(String currentFilePath) {
- currentFileName.push(currentFilePath);
- }
-
- public String peekFileName() {
- return currentFileName.peek();
- }
-
- public String popFileName() {
- return currentFileName.pop();
- }
-
- public void validationError(int line, int charPositionInLine, String msg) {
- try {
- setAtleastOneError(true);
- // Print error messages with file name
- if(currentFileName == null || currentFileName.empty()) {
- LOG.error("line "+line+":"+charPositionInLine+" "+msg);
- }
- else {
- String fileName = currentFileName.peek();
- LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg);
- }
- }
- catch(Exception e1) {
- LOG.error("ERROR: while customizing error message:" + e1);
- }
- }
-
- public void validationWarning(int line, int charPositionInLine, String msg) {
- try {
- //atleastOneError = true; ---> not an error, just warning
- // Print error messages with file name
- if(currentFileName == null || currentFileName.empty())
- LOG.warn("line "+line+":"+charPositionInLine+" "+msg);
- else {
- String fileName = currentFileName.peek();
- LOG.warn(fileName + " line "+line+":"+charPositionInLine+" "+msg);
- }
- }
- catch(Exception e1) {
- LOG.warn("ERROR: while customizing error message:" + e1);
- }
- }
-
- @Override
- public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol,
- int line, int charPositionInLine,
- String msg, RecognitionException e)
- {
- try {
- setAtleastOneError(true);
- // Print error messages with file name
- if(currentFileName == null || currentFileName.empty())
- LOG.error("line "+line+":"+charPositionInLine+" "+msg);
- else {
- String fileName = currentFileName.peek();
- LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg);
- }
- }
- catch(Exception e1) {
- LOG.error("ERROR: while customizing error message:" + e1);
- }
- }
-
- public boolean isAtleastOneError() {
- return atleastOneError;
- }
-
- public void setAtleastOneError(boolean atleastOneError) {
- this.atleastOneError = atleastOneError;
- }
- }
-}
[6/8] incubator-systemml git commit: [SYSTEMML-148] Refactored
.parser.{antlr4, python} to .parser.{dml, pydml}.
Posted by du...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/dml/Dml.g4
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/dml/Dml.g4 b/src/main/java/org/apache/sysml/parser/dml/Dml.g4
new file mode 100644
index 0000000..9d07dc9
--- /dev/null
+++ b/src/main/java/org/apache/sysml/parser/dml/Dml.g4
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grammar Dml;
+
+@header
+{
+ // Commenting the package name and explicitly passing it in build.xml to maintain compatibility with maven plugin
+ // package org.apache.sysml.parser.dml;
+}
+
+// DML Program is a list of expression
+// For now, we only allow global function definitions (not nested or inside a while block)
+dmlprogram: (blocks+=statement | functionBlocks+=functionStatement)* EOF;
+
+statement returns [ StatementInfo info ]
+@init {
+ // This actions occurs regardless of how many alternatives in this rule
+ $info = new StatementInfo();
+} :
+ // ------------------------------------------
+ // ImportStatement
+ 'source' '(' filePath = STRING ')' 'as' namespace=ID ';'* # ImportStatement
+ | 'setwd' '(' pathValue = STRING ')' ';'* # PathStatement
+ // ------------------------------------------
+ // Treat function call as AssignmentStatement or MultiAssignmentStatement
+ // For backward compatibility and also since the behavior of foo() * A + foo() ... where foo returns A
+ // Convert FunctionCallIdentifier(paramExprs, ..) -> source
+ | // TODO: Throw an informative error if user doesnot provide the optional assignment
+ ( targetList+=dataIdentifier ('='|'<-') )? name=ID '(' (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? ')' ';'* # FunctionCallAssignmentStatement
+ | '[' targetList+=dataIdentifier (',' targetList+=dataIdentifier)* ']' ('='|'<-') name=ID '(' (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? ')' ';'* # FunctionCallMultiAssignmentStatement
+ // {notifyErrorListeners("Too many parentheses");}
+ // ------------------------------------------
+ // AssignmentStatement
+ | targetList+=dataIdentifier op=('<-'|'=') 'ifdef' '(' commandLineParam=dataIdentifier ',' source=expression ')' ';'* # IfdefAssignmentStatement
+ | targetList+=dataIdentifier op=('<-'|'=') source=expression ';'* # AssignmentStatement
+ // ------------------------------------------
+ // We don't support block statement
+ // | '{' body+=expression ';'* ( body+=expression ';'* )* '}' # BlockStatement
+ // ------------------------------------------
+ // IfStatement
+ | 'if' '(' predicate=expression ')' (ifBody+=statement ';'* | '{' (ifBody+=statement ';'*)* '}') ('else' (elseBody+=statement ';'* | '{' (elseBody+=statement ';'*)* '}'))? # IfStatement
+ // ------------------------------------------
+ // ForStatement & ParForStatement
+ | 'for' '(' iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* ')' (body+=statement ';'* | '{' (body+=statement ';'* )* '}') # ForStatement
+ // Convert strictParameterizedExpression to HashMap<String, String> for parForParams
+ | 'parfor' '(' iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* ')' (body+=statement ';'* | '{' (body+=statement ';'*)* '}') # ParForStatement
+ | 'while' '(' predicate=expression ')' (body+=statement ';'* | '{' (body+=statement ';'*)* '}') # WhileStatement
+ // ------------------------------------------
+;
+
+iterablePredicate returns [ ExpressionInfo info ]
+ @init {
+ // This actions occurs regardless of how many alternatives in this rule
+ $info = new ExpressionInfo();
+ } :
+ from=expression ':' to=expression #IterablePredicateColonExpression
+ | ID '(' from=expression ',' to=expression ',' increment=expression ')' #IterablePredicateSeqExpression
+ ;
+
+functionStatement returns [ StatementInfo info ]
+@init {
+ // This actions occurs regardless of how many alternatives in this rule
+ $info = new StatementInfo();
+} :
+ // ------------------------------------------
+ // FunctionStatement & ExternalFunctionStatement
+ // small change: only allow typed arguments here ... instead of data identifier
+ name=ID ('<-'|'=') 'function' '(' ( inputParams+=typedArgNoAssign (',' inputParams+=typedArgNoAssign)* )? ')' ( 'return' '(' ( outputParams+=typedArgNoAssign (',' outputParams+=typedArgNoAssign)* )? ')' )? '{' (body+=statement ';'*)* '}' # InternalFunctionDefExpression
+ | name=ID ('<-'|'=') 'externalFunction' '(' ( inputParams+=typedArgNoAssign (',' inputParams+=typedArgNoAssign)* )? ')' ( 'return' '(' ( outputParams+=typedArgNoAssign (',' outputParams+=typedArgNoAssign)* )? ')' )? 'implemented' 'in' '(' ( otherParams+=strictParameterizedKeyValueString (',' otherParams+=strictParameterizedKeyValueString)* )? ')' ';'* # ExternalFunctionDefExpression
+ // ------------------------------------------
+;
+
+
+// Other data identifiers are typedArgNoAssign, parameterizedExpression and strictParameterizedExpression
+dataIdentifier returns [ ExpressionInfo dataInfo ]
+@init {
+ // This actions occurs regardless of how many alternatives in this rule
+ $dataInfo = new ExpressionInfo();
+ // $dataInfo.expr = new org.apache.sysml.parser.DataIdentifier();
+} :
+ // ------------------------------------------
+ // IndexedIdentifier
+ name=ID '[' (rowLower=expression (':' rowUpper=expression)?)? ',' (colLower=expression (':' colUpper=expression)?)? ']' # IndexedExpression
+ // ------------------------------------------
+ | ID # SimpleDataIdentifierExpression
+ | COMMANDLINE_NAMED_ID # CommandlineParamExpression
+ | COMMANDLINE_POSITION_ID # CommandlinePositionExpression
+;
+expression returns [ ExpressionInfo info ]
+@init {
+ // This actions occurs regardless of how many alternatives in this rule
+ $info = new ExpressionInfo();
+ // $info.expr = new org.apache.sysml.parser.BinaryExpression(org.apache.sysml.parser.Expression.BinaryOp.INVALID);
+} :
+ // ------------------------------------------
+ // BinaryExpression
+ // power
+ <assoc=right> left=expression op='^' right=expression # PowerExpression
+ // unary plus and minus
+ | op=('-'|'+') left=expression # UnaryExpression
+ // sequence - since we are only using this into for
+ //| left=expression op=':' right=expression # SequenceExpression
+ // matrix multiply
+ | left=expression op='%*%' right=expression # MatrixMulExpression
+ // modulus and integer division
+ | left=expression op=('%/%' | '%%' ) right=expression # ModIntDivExpression
+ // arithmetic multiply and divide
+ | left=expression op=('*'|'/') right=expression # MultDivExpression
+ // arithmetic addition and subtraction
+ | left=expression op=('+'|'-') right=expression # AddSubExpression
+ // ------------------------------------------
+ // RelationalExpression
+ | left=expression op=('>'|'>='|'<'|'<='|'=='|'!=') right=expression # RelationalExpression
+ // ------------------------------------------
+ // BooleanExpression
+ // boolean not
+ | op='!' left=expression # BooleanNotExpression
+ // boolean and
+ | left=expression op=('&'|'&&') right=expression # BooleanAndExpression
+ // boolean or
+ | left=expression op=('|'|'||') right=expression # BooleanOrExpression
+
+ // ---------------------------------
+ // only applicable for builtin function expressions
+ | name=ID '(' (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? ')' ';'* # BuiltinFunctionExpression
+
+ // 4. Atomic
+ | '(' left=expression ')' # AtomicExpression
+
+ // Should you allow indexed expression here ?
+ // | '[' targetList+=expression (',' targetList+=expression)* ']' # MultiIdExpression
+
+ // | BOOLEAN # ConstBooleanIdExpression
+ | 'TRUE' # ConstTrueExpression
+ | 'FALSE' # ConstFalseExpression
+ | INT # ConstIntIdExpression
+ | DOUBLE # ConstDoubleIdExpression
+ | STRING # ConstStringIdExpression
+ | dataIdentifier # DataIdExpression
+ // Special
+ // | 'NULL' | 'NA' | 'Inf' | 'NaN'
+;
+
+typedArgNoAssign : paramType=ml_type paramName=ID;
+parameterizedExpression : (paramName=ID '=')? paramVal=expression;
+strictParameterizedExpression : paramName=ID '=' paramVal=expression ;
+strictParameterizedKeyValueString : paramName=ID '=' paramVal=STRING ;
+ID : (ALPHABET (ALPHABET|DIGIT|'_')* '::')? ALPHABET (ALPHABET|DIGIT|'_')*
+ // Special ID cases:
+ // | 'matrix' // --> This is a special case which causes lot of headache
+ | 'as.scalar' | 'as.matrix' | 'as.double' | 'as.integer' | 'as.logical' | 'index.return' | 'lower.tail'
+;
+// Unfortunately, we have datatype name clashing with builtin function name: matrix :(
+// Therefore, ugly work around for checking datatype
+ml_type : valueType | dataType '[' valueType ']';
+// Note to reduce number of keywords, these are case-sensitive,
+// To allow case-insenstive, 'int' becomes: ('i' | 'I') ('n' | 'N') ('t' | 'T')
+valueType: 'int' | 'integer' | 'string' | 'boolean' | 'double'
+ | 'Int' | 'Integer' | 'String' | 'Boolean' | 'Double';
+dataType:
+ // 'scalar' # ScalarDataTypeDummyCheck
+ // |
+ ID # MatrixDataTypeCheck //{ if($ID.text.compareTo("matrix") != 0) { notifyErrorListeners("incorrect datatype"); } }
+ //| 'matrix' //---> See ID, this causes lot of headache
+ ;
+INT : DIGIT+ [Ll]?;
+// BOOLEAN : 'TRUE' | 'FALSE';
+DOUBLE: DIGIT+ '.' DIGIT* EXP? [Ll]?
+| DIGIT+ EXP? [Ll]?
+| '.' DIGIT+ EXP? [Ll]?
+;
+DIGIT: '0'..'9';
+ALPHABET : [a-zA-Z] ;
+fragment EXP : ('E' | 'e') ('+' | '-')? INT ;
+COMMANDLINE_NAMED_ID: '$' ALPHABET (ALPHABET|DIGIT|'_')*;
+COMMANDLINE_POSITION_ID: '$' DIGIT+;
+
+// supports single and double quoted string with escape characters
+STRING: '"' ( ESC | ~[\\"] )*? '"' | '\'' ( ESC | ~[\\'] )*? '\'';
+fragment ESC : '\\' [abtnfrv"'\\] ;
+// Comments, whitespaces and new line
+LINE_COMMENT : '#' .*? '\r'? '\n' -> skip ;
+MULTILINE_BLOCK_COMMENT : '/*' .*? '*/' -> skip ;
+WHITESPACE : (' ' | '\t' | '\r' | '\n')+ -> skip ;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticErrorListener.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticErrorListener.java b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticErrorListener.java
new file mode 100644
index 0000000..027ea08
--- /dev/null
+++ b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticErrorListener.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.parser.dml;
+
+import org.antlr.v4.runtime.BaseErrorListener;
+import org.antlr.v4.runtime.RecognitionException;
+import org.antlr.v4.runtime.Recognizer;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.sysml.api.DMLScript;
+
+import java.util.Stack;
+
+public class DmlSyntacticErrorListener {
+
+
+ private static final Log LOG = LogFactory.getLog(DMLScript.class.getName());
+
+ public static class CustomDmlErrorListener extends BaseErrorListener {
+
+ private boolean atleastOneError = false;
+ private Stack<String> currentFileName = new Stack<String>();
+
+ public void pushCurrentFileName(String currentFilePath) {
+ currentFileName.push(currentFilePath);
+ }
+
+ public String peekFileName() {
+ return currentFileName.peek();
+ }
+
+ public String popFileName() {
+ return currentFileName.pop();
+ }
+
+ public void validationError(int line, int charPositionInLine, String msg) {
+ try {
+ setAtleastOneError(true);
+ // Print error messages with file name
+ if(currentFileName == null || currentFileName.empty()) {
+ LOG.error("line "+line+":"+charPositionInLine+" "+msg);
+ }
+ else {
+ String fileName = currentFileName.peek();
+ LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg);
+ }
+ }
+ catch(Exception e1) {
+ LOG.error("ERROR: while customizing error message:" + e1);
+ }
+ }
+
+ public void validationWarning(int line, int charPositionInLine, String msg) {
+ try {
+ //atleastOneError = true; ---> not an error, just warning
+ // Print error messages with file name
+ if(currentFileName == null || currentFileName.empty())
+ LOG.warn("line "+line+":"+charPositionInLine+" "+msg);
+ else {
+ String fileName = currentFileName.peek();
+ LOG.warn(fileName + " line "+line+":"+charPositionInLine+" "+msg);
+ }
+ }
+ catch(Exception e1) {
+ LOG.warn("ERROR: while customizing error message:" + e1);
+ }
+ }
+
+ @Override
+ public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol,
+ int line, int charPositionInLine,
+ String msg, RecognitionException e)
+ {
+ try {
+ setAtleastOneError(true);
+ // Print error messages with file name
+ if(currentFileName == null || currentFileName.empty())
+ LOG.error("line "+line+":"+charPositionInLine+" "+msg);
+ else {
+ String fileName = currentFileName.peek();
+ LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg);
+ }
+ }
+ catch(Exception e1) {
+ LOG.error("ERROR: while customizing error message:" + e1);
+ }
+ }
+
+ public boolean isAtleastOneError() {
+ return atleastOneError;
+ }
+
+ public void setAtleastOneError(boolean atleastOneError) {
+ this.atleastOneError = atleastOneError;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java
new file mode 100644
index 0000000..798ab40
--- /dev/null
+++ b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java
@@ -0,0 +1,1533 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.parser.dml;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.tree.ErrorNode;
+import org.antlr.v4.runtime.tree.TerminalNode;
+import org.apache.sysml.parser.ConditionalPredicate;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.parser.DoubleIdentifier;
+import org.apache.sysml.parser.Expression;
+import org.apache.sysml.parser.Expression.DataOp;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.dml.DmlParser.AddSubExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.AssignmentStatementContext;
+import org.apache.sysml.parser.dml.DmlParser.AtomicExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.BooleanAndExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.BooleanNotExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.BooleanOrExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.BuiltinFunctionExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.CommandlineParamExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.CommandlinePositionExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.ConstDoubleIdExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.ConstFalseExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.ConstIntIdExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.ConstStringIdExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.ConstTrueExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.DataIdExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.DataIdentifierContext;
+import org.apache.sysml.parser.dml.DmlParser.DmlprogramContext;
+import org.apache.sysml.parser.dml.DmlParser.ExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.ExternalFunctionDefExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.ForStatementContext;
+import org.apache.sysml.parser.dml.DmlParser.FunctionCallAssignmentStatementContext;
+import org.apache.sysml.parser.dml.DmlParser.FunctionCallMultiAssignmentStatementContext;
+import org.apache.sysml.parser.dml.DmlParser.FunctionStatementContext;
+import org.apache.sysml.parser.dml.DmlParser.IfStatementContext;
+import org.apache.sysml.parser.dml.DmlParser.IfdefAssignmentStatementContext;
+import org.apache.sysml.parser.dml.DmlParser.ImportStatementContext;
+import org.apache.sysml.parser.dml.DmlParser.IndexedExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.InternalFunctionDefExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.IterablePredicateColonExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.IterablePredicateSeqExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.MatrixDataTypeCheckContext;
+import org.apache.sysml.parser.dml.DmlParser.MatrixMulExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.Ml_typeContext;
+import org.apache.sysml.parser.dml.DmlParser.ModIntDivExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.MultDivExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.ParForStatementContext;
+import org.apache.sysml.parser.dml.DmlParser.ParameterizedExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.PathStatementContext;
+import org.apache.sysml.parser.dml.DmlParser.PowerExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.RelationalExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.SimpleDataIdentifierExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.StatementContext;
+import org.apache.sysml.parser.dml.DmlParser.StrictParameterizedExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.StrictParameterizedKeyValueStringContext;
+import org.apache.sysml.parser.dml.DmlParser.TypedArgNoAssignContext;
+import org.apache.sysml.parser.dml.DmlParser.UnaryExpressionContext;
+import org.apache.sysml.parser.dml.DmlParser.ValueTypeContext;
+import org.apache.sysml.parser.dml.DmlParser.WhileStatementContext;
+import org.apache.sysml.parser.AParserWrapper;
+import org.apache.sysml.parser.AssignmentStatement;
+import org.apache.sysml.parser.BinaryExpression;
+import org.apache.sysml.parser.BooleanExpression;
+import org.apache.sysml.parser.BooleanIdentifier;
+import org.apache.sysml.parser.BuiltinFunctionExpression;
+import org.apache.sysml.parser.ConstIdentifier;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.ExternalFunctionStatement;
+import org.apache.sysml.parser.ForStatement;
+import org.apache.sysml.parser.FunctionCallIdentifier;
+import org.apache.sysml.parser.FunctionStatement;
+import org.apache.sysml.parser.IfStatement;
+import org.apache.sysml.parser.ImportStatement;
+import org.apache.sysml.parser.IndexedIdentifier;
+import org.apache.sysml.parser.IntIdentifier;
+import org.apache.sysml.parser.IterablePredicate;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.parser.MultiAssignmentStatement;
+import org.apache.sysml.parser.OutputStatement;
+import org.apache.sysml.parser.ParForStatement;
+import org.apache.sysml.parser.ParameterExpression;
+import org.apache.sysml.parser.ParameterizedBuiltinFunctionExpression;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.parser.PathStatement;
+import org.apache.sysml.parser.PrintStatement;
+import org.apache.sysml.parser.RelationalExpression;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.StringIdentifier;
+import org.apache.sysml.parser.WhileStatement;
+
+/**
+ * TODO: Refactor duplicated parser code dml/pydml (entire package).
+ *
+ */
+public class DmlSyntacticValidator implements DmlListener
+{
+ private DmlSyntacticValidatorHelper helper = null;
+
+ private String _workingDir = "."; //current working directory
+ private String _currentPath = null; //current file path
+ private HashMap<String,String> argVals = null;
+
+ public DmlSyntacticValidator(DmlSyntacticValidatorHelper helper, String currentPath, HashMap<String,String> argVals) {
+ this.helper = helper;
+ this.argVals = argVals;
+
+ _currentPath = currentPath;
+ }
+
+ // Functions we have to implement but don't really need it
+ @Override
+ public void enterAddSubExpression(AddSubExpressionContext ctx) { }
+ @Override
+ public void enterAssignmentStatement(AssignmentStatementContext ctx) {}
+ @Override
+ public void enterAtomicExpression(AtomicExpressionContext ctx) { }
+ @Override
+ public void enterBooleanAndExpression(BooleanAndExpressionContext ctx) { }
+ @Override
+ public void enterBooleanNotExpression(BooleanNotExpressionContext ctx) { }
+ @Override
+ public void enterBooleanOrExpression(BooleanOrExpressionContext ctx) { }
+ @Override
+ public void enterCommandlineParamExpression(CommandlineParamExpressionContext ctx) { }
+ @Override
+ public void enterCommandlinePositionExpression(CommandlinePositionExpressionContext ctx) { }
+ @Override
+ public void enterConstDoubleIdExpression(ConstDoubleIdExpressionContext ctx) { }
+ @Override
+ public void enterConstIntIdExpression(ConstIntIdExpressionContext ctx) { }
+ @Override
+ public void enterConstStringIdExpression(ConstStringIdExpressionContext ctx) { }
+ @Override
+ public void enterDataIdExpression(DataIdExpressionContext ctx) { }
+
+ @Override
+ public void enterDmlprogram(DmlprogramContext ctx) { }
+ @Override
+ public void enterEveryRule(ParserRuleContext arg0) {
+ if(arg0 instanceof StatementContext) {
+ if(((StatementContext) arg0).info == null) {
+ ((StatementContext) arg0).info = new StatementInfo();
+ }
+ }
+ if(arg0 instanceof FunctionStatementContext) {
+ if(((FunctionStatementContext) arg0).info == null) {
+ ((FunctionStatementContext) arg0).info = new StatementInfo();
+ }
+ }
+ if(arg0 instanceof ExpressionContext) {
+ if(((ExpressionContext) arg0).info == null) {
+ ((ExpressionContext) arg0).info = new ExpressionInfo();
+ }
+ }
+ if(arg0 instanceof DataIdentifierContext) {
+ if(((DataIdentifierContext) arg0).dataInfo == null) {
+ ((DataIdentifierContext) arg0).dataInfo = new ExpressionInfo();
+ }
+ }
+ }
+ @Override
+ public void enterExternalFunctionDefExpression(ExternalFunctionDefExpressionContext ctx) { }
+ @Override
+ public void enterForStatement(ForStatementContext ctx) {}
+ @Override
+ public void enterFunctionCallAssignmentStatement(FunctionCallAssignmentStatementContext ctx) { }
+ @Override
+ public void enterFunctionCallMultiAssignmentStatement(FunctionCallMultiAssignmentStatementContext ctx) { }
+ @Override
+ public void enterIfStatement(IfStatementContext ctx) { }
+ @Override
+ public void enterImportStatement(ImportStatementContext ctx) { }
+ @Override
+ public void enterIndexedExpression(IndexedExpressionContext ctx) { }
+ @Override
+ public void enterInternalFunctionDefExpression(InternalFunctionDefExpressionContext ctx) { }
+ public void enterMatrixMulExpression(MatrixMulExpressionContext ctx) { }
+ @Override
+ public void enterMl_type(Ml_typeContext ctx) { }
+ @Override
+ public void enterModIntDivExpression(ModIntDivExpressionContext ctx) { }
+ @Override
+ public void enterMultDivExpression(MultDivExpressionContext ctx) { }
+ @Override
+ public void enterParameterizedExpression(ParameterizedExpressionContext ctx) { }
+ @Override
+ public void enterParForStatement(ParForStatementContext ctx) { }
+ @Override
+ public void enterPathStatement(PathStatementContext ctx) { }
+ @Override
+ public void enterPowerExpression(PowerExpressionContext ctx) { }
+ @Override
+ public void enterRelationalExpression(RelationalExpressionContext ctx) { }
+ @Override
+ public void enterSimpleDataIdentifierExpression(SimpleDataIdentifierExpressionContext ctx) { }
+ @Override
+ public void enterStrictParameterizedExpression(StrictParameterizedExpressionContext ctx) { }
+ @Override
+ public void enterTypedArgNoAssign(TypedArgNoAssignContext ctx) { }
+ @Override
+ public void enterUnaryExpression(UnaryExpressionContext ctx) { }
+ @Override
+ public void enterValueType(ValueTypeContext ctx) { }
+ @Override
+ public void enterWhileStatement(WhileStatementContext ctx) { }
+
+ @Override
+ public void visitErrorNode(ErrorNode arg0) { }
+ @Override
+ public void visitTerminal(TerminalNode arg0) { }
+ @Override
+ public void exitEveryRule(ParserRuleContext arg0) {}
+ // --------------------------------------------------------------------
+ private void setFileLineColumn(Expression expr, ParserRuleContext ctx) {
+ // expr.setFilename(helper.getCurrentFileName());
+ String txt = ctx.getText();
+ expr.setFilename(_currentPath);
+ expr.setBeginLine(ctx.start.getLine());
+ expr.setBeginColumn(ctx.start.getCharPositionInLine());
+ expr.setEndLine(ctx.stop.getLine());
+ expr.setEndColumn(ctx.stop.getCharPositionInLine());
+ if(expr.getBeginColumn() == expr.getEndColumn() && expr.getBeginLine() == expr.getEndLine() && txt.length() > 1) {
+ expr.setEndColumn(expr.getBeginColumn() + txt.length() - 1);
+ }
+ }
+
+ private void setFileLineColumn(Statement stmt, ParserRuleContext ctx) {
+ String txt = ctx.getText();
+ stmt.setFilename(helper.getCurrentFileName());
+ stmt.setBeginLine(ctx.start.getLine());
+ stmt.setBeginColumn(ctx.start.getCharPositionInLine());
+ stmt.setEndLine(ctx.stop.getLine());
+ stmt.setEndColumn(ctx.stop.getCharPositionInLine());
+ if(stmt.getBeginColumn() == stmt.getEndColumn() && stmt.getBeginLine() == stmt.getEndLine() && txt.length() > 1) {
+ stmt.setEndColumn(stmt.getBeginColumn() + txt.length() - 1);
+ }
+ }
+
+ // For now do no type checking, let validation handle it.
+ // This way parser doesn't have to open metadata file
+ @Override
+ public void exitAddSubExpression(AddSubExpressionContext ctx) {
+ if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
+ Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
+ ctx.info.expr = new BinaryExpression(bop);
+ ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ }
+ @Override
+ public void exitModIntDivExpression(ModIntDivExpressionContext ctx) {
+ if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
+ Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
+ ctx.info.expr = new BinaryExpression(bop);
+ ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ }
+
+ @Override
+ public void exitUnaryExpression(UnaryExpressionContext ctx) {
+ if(ctx.left.info.expr != null) {
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+
+ if(ctx.left.info.expr instanceof IntIdentifier) {
+ if(ctx.op.getText().compareTo("-") == 0) {
+ ((IntIdentifier) ctx.left.info.expr).multiplyByMinusOne();
+ }
+ ctx.info.expr = ctx.left.info.expr;
+ }
+ else if(ctx.left.info.expr instanceof DoubleIdentifier) {
+ if(ctx.op.getText().compareTo("-") == 0) {
+ ((DoubleIdentifier) ctx.left.info.expr).multiplyByMinusOne();
+ }
+ ctx.info.expr = ctx.left.info.expr;
+ }
+ else {
+ Expression right = new IntIdentifier(1, fileName, line, col, line, col);
+ if(ctx.op.getText().compareTo("-") == 0) {
+ right = new IntIdentifier(-1, fileName, line, col, line, col);
+ }
+
+ Expression.BinaryOp bop = Expression.getBinaryOp("*");
+ ctx.info.expr = new BinaryExpression(bop);
+ ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ ((BinaryExpression)ctx.info.expr).setRight(right);
+ }
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ }
+
+ @Override
+ public void exitMultDivExpression(MultDivExpressionContext ctx) {
+ Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
+ ctx.info.expr = new BinaryExpression(bop);
+ ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ @Override
+ public void exitPowerExpression(PowerExpressionContext ctx) {
+ Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
+ ctx.info.expr = new BinaryExpression(bop);
+ ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+
+ @Override
+ public void exitMatrixMulExpression(MatrixMulExpressionContext ctx) {
+ Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
+ ctx.info.expr = new BinaryExpression(bop);
+ ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+
+ // --------------------------------------------------------------------
+
+ @Override
+ public void exitRelationalExpression(RelationalExpressionContext ctx) {
+ if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
+// String fileName = helper.getCurrentFileName();
+// int line = ctx.start.getLine();
+// int col = ctx.start.getCharPositionInLine();
+// ArrayList<ParameterExpression> paramExpression = new ArrayList<ParameterExpression>();
+// paramExpression.add(new ParameterExpression(null, ctx.left.info.expr));
+// paramExpression.add(new ParameterExpression(null, ctx.right.info.expr));
+// ParameterExpression operator = new ParameterExpression(null, new StringIdentifier(ctx.op.getText(), fileName, line, col, line, col));
+// paramExpression.add(operator);
+//
+// try {
+// BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression("ppred", paramExpression, fileName, line, col, line, col);
+// if (bife != null){
+// // It is a builtin function
+// ctx.info.expr = bife;
+// return;
+// }
+// }
+// catch(Exception e) {}
+// helper.notifyErrorListeners("Cannot parse relational expression", ctx.getStart());
+
+ Expression.RelationalOp rop = Expression.getRelationalOp(ctx.op.getText());
+ ctx.info.expr = new RelationalExpression(rop);
+ ((RelationalExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ ((RelationalExpression)ctx.info.expr).setRight(ctx.right.info.expr);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ }
+
+ // --------------------------------------------------------------------
+
+ @Override
+ public void exitBooleanAndExpression(BooleanAndExpressionContext ctx) {
+ if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
+ Expression.BooleanOp bop = Expression.getBooleanOp(ctx.op.getText());
+ ctx.info.expr = new BooleanExpression(bop);
+ ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ ((BooleanExpression)ctx.info.expr).setRight(ctx.right.info.expr);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ }
+
+ @Override
+ public void exitBooleanOrExpression(BooleanOrExpressionContext ctx) {
+ if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
+ Expression.BooleanOp bop = Expression.getBooleanOp(ctx.op.getText());
+ ctx.info.expr = new BooleanExpression(bop);
+ ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ ((BooleanExpression)ctx.info.expr).setRight(ctx.right.info.expr);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ }
+
+ @Override
+ public void exitBooleanNotExpression(BooleanNotExpressionContext ctx) {
+ if(ctx.left.info.expr != null) {
+ Expression.BooleanOp bop = Expression.getBooleanOp(ctx.op.getText());
+ ctx.info.expr = new BooleanExpression(bop);
+ ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ }
+
+ // --------------------------------------------------------------------
+
+ @Override
+ public void exitAtomicExpression(AtomicExpressionContext ctx) {
+ ctx.info.expr = ctx.left.info.expr;
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+
+// @Override
+// public void exitConstBooleanIdExpression(ConstBooleanIdExpressionContext ctx) {
+// boolean val = false;
+// if(ctx.getText().compareTo("TRUE") == 0) {
+// val = true;
+// }
+// else if(ctx.getText().compareTo("FALSE") == 0) {
+// val = false;
+// }
+// else {
+// helper.notifyErrorListeners("cannot parse the boolean value: \'" + ctx.getText() + "\'", ctx.getStart());
+// return;
+// }
+// int linePosition = ctx.start.getLine();
+// int charPosition = ctx.start.getCharPositionInLine();
+// ctx.info.expr = new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+// setFileLineColumn(ctx.info.expr, ctx);
+// }
+
+ @Override
+ public void exitConstDoubleIdExpression(ConstDoubleIdExpressionContext ctx) {
+ try {
+ double val = Double.parseDouble(ctx.getText());
+ int linePosition = ctx.start.getLine();
+ int charPosition = ctx.start.getCharPositionInLine();
+ ctx.info.expr = new DoubleIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ catch(Exception e) {
+ helper.notifyErrorListeners("cannot parse the double value: \'" + ctx.getText() + "\'", ctx.getStart());
+ return;
+ }
+ }
+
+ @Override
+ public void exitConstIntIdExpression(ConstIntIdExpressionContext ctx) {
+ try {
+ long val = Long.parseLong(ctx.getText());
+ int linePosition = ctx.start.getLine();
+ int charPosition = ctx.start.getCharPositionInLine();
+ ctx.info.expr = new IntIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ catch(Exception e) {
+ helper.notifyErrorListeners("cannot parse the integer value: \'" + ctx.getText() + "\'", ctx.getStart());
+ return;
+ }
+ }
+
+ @Override
+ public void exitConstStringIdExpression(ConstStringIdExpressionContext ctx) {
+ String val = "";
+ String text = ctx.getText();
+ if( (text.startsWith("\"") && text.endsWith("\"")) ||
+ (text.startsWith("\'") && text.endsWith("\'"))) {
+ if(text.length() > 2) {
+ val = text.substring(1, text.length()-1);
+ }
+ }
+ else {
+ helper.notifyErrorListeners("something wrong while parsing string ... strange", ctx.start);
+ return;
+ }
+
+ int linePosition = ctx.start.getLine();
+ int charPosition = ctx.start.getCharPositionInLine();
+ ctx.info.expr = new StringIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+
+ // --------------------------------------------------------------------
+
+ @Override
+ public void exitDataIdExpression(DataIdExpressionContext ctx) {
+ ctx.info.expr = ctx.dataIdentifier().dataInfo.expr;
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+ ctx.info.expr.setAllPositions(helper.getCurrentFileName(), line, col, line, col);
+ setFileLineColumn(ctx.info.expr, ctx);
+// if(ctx.getChild(0) instanceof DataIdentifierContext) {
+// ctx.info.expr = ctx.dataIdentifier().dataInfo.expr;
+// }
+// else {
+// String msg = "cannot evaluate data expression ... strange";
+// helper.notifyErrorListeners(msg, ctx.start);
+// }
+ }
+
+ @Override
+ public void exitSimpleDataIdentifierExpression(SimpleDataIdentifierExpressionContext ctx) {
+ // This is either a function, or variable with namespace
+ // By default, it assigns to a data type
+ ctx.dataInfo.expr = new DataIdentifier(ctx.getText());
+ setFileLineColumn(ctx.dataInfo.expr, ctx);
+ }
+
+ @Override
+ public void exitIndexedExpression(IndexedExpressionContext ctx) {
+ ctx.dataInfo.expr = new IndexedIdentifier(ctx.name.getText(), false, false);
+ setFileLineColumn(ctx.dataInfo.expr, ctx);
+ try {
+ ArrayList< ArrayList<Expression> > exprList = new ArrayList< ArrayList<Expression> >();
+
+ ArrayList<Expression> rowIndices = new ArrayList<Expression>();
+ ArrayList<Expression> colIndices = new ArrayList<Expression>();
+
+ boolean isRowLower = (ctx.rowLower != null && !ctx.rowLower.isEmpty() && (ctx.rowLower.info.expr != null));
+ boolean isRowUpper = (ctx.rowUpper != null && !ctx.rowUpper.isEmpty() && (ctx.rowUpper.info.expr != null));
+ boolean isColLower = (ctx.colLower != null && !ctx.colLower.isEmpty() && (ctx.colLower.info.expr != null));
+ boolean isColUpper = (ctx.colUpper != null && !ctx.colUpper.isEmpty() && (ctx.colUpper.info.expr != null));
+
+ if(!isRowLower && !isRowUpper) {
+ // both not set
+ rowIndices.add(null); rowIndices.add(null);
+ }
+ else if(isRowLower && isRowUpper) {
+ // both set
+ rowIndices.add(ctx.rowLower.info.expr);
+ rowIndices.add(ctx.rowUpper.info.expr);
+ }
+ else if(isRowLower && !isRowUpper) {
+ // only row set
+ rowIndices.add(ctx.rowLower.info.expr);
+ }
+ else {
+ helper.notifyErrorListeners("incorrect index expression for row", ctx.start);
+ return;
+ }
+
+ if(!isColLower && !isColUpper) {
+ // both not set
+ colIndices.add(null); colIndices.add(null);
+ }
+ else if(isColLower && isColUpper) {
+ colIndices.add(ctx.colLower.info.expr);
+ colIndices.add(ctx.colUpper.info.expr);
+ }
+ else if(isColLower && !isColUpper) {
+ colIndices.add(ctx.colLower.info.expr);
+ }
+ else {
+ helper.notifyErrorListeners("incorrect index expression for column", ctx.start);
+ return;
+ }
+
+
+// boolean rowIndexLowerSet = false;
+// boolean colIndexLowerSet = false;
+//
+// if(ctx.rowLower != null && !ctx.rowLower.isEmpty() && (ctx.rowLower.info.expr != null)) {
+// rowIndices.add(ctx.rowLower.info.expr);
+// rowIndexLowerSet = true;
+// }
+// else {
+// rowIndices.add(null);
+// }
+// if(ctx.rowUpper != null && !ctx.rowUpper.isEmpty() && (ctx.rowUpper.info.expr != null)) {
+// rowIndices.add(ctx.rowUpper.info.expr);
+// if(!rowIndexLowerSet) {
+// helper.notifyErrorListeners("incorrect index expression for row", ctx.start);
+// return;
+// }
+// }
+// if(ctx.colLower != null && !ctx.colLower.isEmpty() && (ctx.colLower.info.expr != null)) {
+// colIndices.add(ctx.colLower.info.expr);
+// colIndexLowerSet = true;
+// }
+// else {
+// colIndices.add(null);
+// }
+// if(ctx.colUpper != null && !ctx.colUpper.isEmpty() && (ctx.colUpper.info.expr != null)) {
+// colIndices.add(ctx.colUpper.info.expr);
+// if(!colIndexLowerSet) {
+// helper.notifyErrorListeners("incorrect index expression for column", ctx.start);
+// return;
+// }
+// }
+ exprList.add(rowIndices);
+ exprList.add(colIndices);
+ ((IndexedIdentifier) ctx.dataInfo.expr).setIndices(exprList);
+ }
+ catch(Exception e) {
+ helper.notifyErrorListeners("cannot set the indices", ctx.start);
+ return;
+ }
+ }
+
+ private ConstIdentifier getConstIdFromString(String varValue, Token start) {
+ // Both varName and varValue are correct
+ int linePosition = start.getLine();
+ int charPosition = start.getCharPositionInLine();
+ try {
+ long val = Long.parseLong(varValue);
+ return new IntIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ }
+ catch(Exception e) {
+ try {
+ double val = Double.parseDouble(varValue);
+ return new DoubleIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ }
+ catch(Exception e1) {
+ try {
+ if(varValue.compareTo("TRUE") == 0 || varValue.compareTo("FALSE") == 0) {
+ boolean val = false;
+ if(varValue.compareTo("TRUE") == 0) {
+ val = true;
+ }
+ return new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ }
+ else {
+ String val = "";
+ String text = varValue;
+ if( (text.startsWith("\"") && text.endsWith("\"")) ||
+ (text.startsWith("\'") && text.endsWith("\'"))) {
+ if(text.length() > 2) {
+ val = text.substring(1, text.length()-1);
+ }
+ }
+ else {
+ val = text;
+ // the commandline parameters can be passed without any quotes
+// helper.notifyErrorListeners("something wrong while parsing string ... strange", start);
+// return null;
+ }
+ return new StringIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ }
+ }
+ catch(Exception e3) {
+ helper.notifyErrorListeners("unable to cast the commandline parameter into int/double/boolean/string", start);
+ return null;
+ }
+ }
+ }
+
+ }
+
+ private void fillExpressionInfoCommandLineParameters(String varName, ExpressionInfo dataInfo, Token start) {
+
+ if(!varName.startsWith("$")) {
+ helper.notifyErrorListeners("commandline param doesnot start with $ ... strange", start);
+ return;
+ }
+
+ String varValue = null;
+ for(Map.Entry<String, String> arg : this.argVals.entrySet()) {
+ if(arg.getKey().trim().compareTo(varName) == 0) {
+ if(varValue != null) {
+ helper.notifyErrorListeners("multiple values passed for the parameter " + varName + " via commandline", start);
+ return;
+ }
+ else {
+ varValue = arg.getValue().trim();
+ }
+ }
+ }
+
+ if(varValue == null) {
+ // helper.notifyErrorListeners("the parameter " + varName + " either needs to be passed through commandline or initialized to default value", start);
+ return;
+ }
+
+ // Command line param cannot be empty string
+ // If you want to pass space, please quote it
+ if(varValue.trim().compareTo("") == 0)
+ return;
+
+ dataInfo.expr = getConstIdFromString(varValue, start);
+ }
+
+ @Override
+ public void exitCommandlineParamExpression(CommandlineParamExpressionContext ctx) {
+ handleCommandlineArgumentExpression(ctx);
+ }
+
+ @Override
+ public void exitCommandlinePositionExpression(CommandlinePositionExpressionContext ctx) {
+ handleCommandlineArgumentExpression(ctx);
+ }
+
+ /**
+ *
+ * @param ctx
+ */
+ private void handleCommandlineArgumentExpression(DataIdentifierContext ctx)
+ {
+ String varName = ctx.getText().trim();
+ fillExpressionInfoCommandLineParameters(varName, ctx.dataInfo, ctx.start);
+
+ if(ctx.dataInfo.expr == null) {
+ if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) {
+ String msg = "The parameter " + varName + " either needs to be passed "
+ + "through commandline or initialized to default value.";
+ if( AParserWrapper.IGNORE_UNSPECIFIED_ARGS ) {
+ ctx.dataInfo.expr = getConstIdFromString(" ", ctx.start);
+ helper.raiseWarning(msg, ctx.start);
+ }
+ else {
+ helper.notifyErrorListeners(msg, ctx.start);
+ }
+ }
+ }
+ }
+
+ // --------------------------------------------------------------------
+
+ @Override
+ public void exitImportStatement(ImportStatementContext ctx)
+ {
+ //prepare import filepath
+ String filePath = ctx.filePath.getText();
+ String namespace = DMLProgram.DEFAULT_NAMESPACE;
+ if(ctx.namespace != null && ctx.namespace.getText() != null && !ctx.namespace.getText().isEmpty()) {
+ namespace = ctx.namespace.getText();
+ }
+ if((filePath.startsWith("\"") && filePath.endsWith("\"")) ||
+ filePath.startsWith("'") && filePath.endsWith("'")) {
+ filePath = filePath.substring(1, filePath.length()-1);
+ }
+
+ //concatenate working directory to filepath
+ filePath = _workingDir + File.separator + filePath;
+
+ DMLProgram prog = null;
+ try {
+ prog = (new DMLParserWrapper()).doParse(filePath, null, argVals);
+ } catch (ParseException e) {
+ helper.notifyErrorListeners("Exception found during importing a program from file " + filePath, ctx.start);
+ return;
+ }
+ // Custom logic whether to proceed ahead or not. Better than the current exception handling mechanism
+ if(prog == null) {
+ helper.notifyErrorListeners("One or more errors found during importing a program from file " + filePath, ctx.start);
+ return;
+ }
+ else {
+ ctx.info.namespaces = new HashMap<String, DMLProgram>();
+ ctx.info.namespaces.put(namespace, prog);
+ ctx.info.stmt = new ImportStatement();
+ ((ImportStatement) ctx.info.stmt).setCompletePath(filePath);
+ ((ImportStatement) ctx.info.stmt).setFilePath(ctx.filePath.getText());
+ ((ImportStatement) ctx.info.stmt).setNamespace(namespace);
+ }
+ }
+
+ @Override
+ public void exitAssignmentStatement(AssignmentStatementContext ctx) {
+ if(ctx.targetList == null || ctx.targetList.size() != 1) {
+ helper.notifyErrorListeners("incorrect parsing for assignment", ctx.start);
+ return;
+ }
+ String targetListText = ctx.targetList.get(0).getText();
+ if(targetListText.startsWith("$")) {
+ helper.notifyErrorListeners("assignment of commandline parameters is not allowed. (Quickfix: try using someLocalVariable=ifdef(" + targetListText + ", default value))", ctx.start);
+ return;
+ }
+
+ DataIdentifier target = null;
+ if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
+ target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
+ Expression source = ctx.source.info.expr;
+
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+ try {
+ ctx.info.stmt = new AssignmentStatement(target, source, line, col, line, col);
+ setFileLineColumn(ctx.info.stmt, ctx);
+ } catch (LanguageException e) {
+ // TODO: extract more meaningful info from this exception.
+ helper.notifyErrorListeners("invalid assignment", ctx.targetList.get(0).start);
+ return;
+ }
+ }
+ else {
+ helper.notifyErrorListeners("incorrect lvalue ... strange", ctx.targetList.get(0).start);
+ return;
+ }
+
+ }
+
+
+ private void setAssignmentStatement(DataIdentifier target, Expression expression, StatementContext ctx) {
+ try {
+ ctx.info.stmt = new AssignmentStatement(target, expression, ctx.start.getLine(), ctx.start.getCharPositionInLine(), ctx.start.getLine(), ctx.start.getCharPositionInLine());
+ setFileLineColumn(ctx.info.stmt, ctx);
+ } catch (LanguageException e) {
+ // TODO: extract more meaningful info from this exception.
+ helper.notifyErrorListeners("invalid function call", ctx.start);
+ return;
+ }
+ }
+
+ private void setPrintStatement(FunctionCallAssignmentStatementContext ctx, String functionName) {
+ ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
+ if(paramExpression.size() != 1) {
+ helper.notifyErrorListeners(functionName + "() has only one parameter", ctx.start);
+ return;
+ }
+ Expression expr = paramExpression.get(0).getExpr();
+ if(expr == null) {
+ helper.notifyErrorListeners("cannot process " + functionName + "() function", ctx.start);
+ return;
+ }
+ try {
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+ ctx.info.stmt = new PrintStatement(functionName, expr, line, col, line, col);
+ setFileLineColumn(ctx.info.stmt, ctx);
+ } catch (LanguageException e) {
+ helper.notifyErrorListeners("cannot process " + functionName + "() function", ctx.start);
+ return;
+ }
+ }
+
+ private void setOutputStatement(FunctionCallAssignmentStatementContext ctx) {
+ ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
+ if(paramExpression.size() < 2){
+ helper.notifyErrorListeners("incorrect usage of write function (atleast 2 arguments required)", ctx.start);
+ return;
+ }
+ if(paramExpression.get(0).getExpr() instanceof DataIdentifier) {
+ // && paramExpression.get(0).getName() == null
+ // correct usage of identifier
+ // if(paramExpression.get(1).getName() == null) {
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+ HashMap<String, Expression> varParams = new HashMap<String, Expression>();
+ varParams.put(DataExpression.IO_FILENAME, paramExpression.get(1).getExpr());
+ for(int i = 2; i < paramExpression.size(); i++) {
+ // DataExpression.FORMAT_TYPE, DataExpression.DELIM_DELIMITER, DataExpression.DELIM_HAS_HEADER_ROW, DataExpression.DELIM_SPARSE
+ varParams.put(paramExpression.get(i).getName(), paramExpression.get(i).getExpr());
+ }
+
+ DataExpression dataExpression = new DataExpression(DataOp.WRITE, varParams, fileName, line, col, line, col);
+ ctx.info.stmt = new OutputStatement((DataIdentifier) paramExpression.get(0).getExpr(), DataOp.WRITE, fileName, line, col, line, col);
+ setFileLineColumn(ctx.info.stmt, ctx);
+ ((OutputStatement)ctx.info.stmt).setExprParams(dataExpression);
+ return;
+ //}
+ }
+
+ helper.notifyErrorListeners("incorrect usage of write function", ctx.start);
+ return;
+
+ }
+
+ @Override
+ public void exitFunctionCallAssignmentStatement(FunctionCallAssignmentStatementContext ctx) {
+ String fullyQualifiedFunctionName = ctx.name.getText();
+ String [] fnNames = fullyQualifiedFunctionName.split("::");
+ String functionName = "";
+ String namespace = "";
+ if(fnNames.length == 1) {
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ functionName = fnNames[0].trim();
+ }
+ else if(fnNames.length == 2) {
+ namespace = fnNames[0].trim();
+ functionName = fnNames[1].trim();
+ }
+ else {
+ helper.notifyErrorListeners("incorrect function name", ctx.name);
+ return;
+ }
+
+ if((functionName.compareTo("print") == 0 || functionName.compareTo("stop") == 0 ) && namespace.compareTo(DMLProgram.DEFAULT_NAMESPACE) == 0) {
+ setPrintStatement(ctx, functionName);
+ return;
+ }
+ else if(functionName.compareTo("write") == 0
+ && namespace.compareTo(DMLProgram.DEFAULT_NAMESPACE) == 0) {
+ setOutputStatement(ctx);
+ return;
+ }
+
+ boolean ignoreLValue = false;
+ if(ctx.targetList == null || ctx.targetList.size() == 0 || ctx.targetList.get(0).isEmpty()) {
+ helper.notifyErrorListeners("function call needs to have lvalue (Quickfix: change it to \'tmpVar = " + functionName + "(...)\')", ctx.name);
+ return;
+ }
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+
+ ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
+// if(functionName.compareTo("read") == 0 && paramExpression.size() > 0 && paramExpression.get(0).getName() == null) {
+// paramExpression.get(0).setName(DataExpression.IO_FILENAME);
+// }
+
+ FunctionCallIdentifier functCall = new FunctionCallIdentifier(paramExpression);
+ try {
+ functCall.setFunctionName(functionName);
+ functCall.setFunctionNamespace(namespace);
+ } catch (ParseException e1) {
+ helper.notifyErrorListeners("unable to process function " + functionName, ctx.start);
+ return;
+ }
+
+ DataIdentifier target = null;
+ if(!ignoreLValue) {
+ if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
+ target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
+ }
+ else {
+ helper.notifyErrorListeners("incorrect lvalue ... strange", ctx.targetList.get(0).start);
+ //target = new DataIdentifier(); // so as not to avoid null pointer
+ return;
+ }
+ }
+
+ if(!functionName.contains("::") || functionName.startsWith(DMLProgram.DEFAULT_NAMESPACE)) {
+ // In global namespace, so it can be a builtin function
+ if(!helper.validateBuiltinFunctions(ctx)) {
+ return; // it is a built-in function and validation failed, so donot proceed ahead.
+ }
+ // Double verification: verify passed function name is a (non-parameterized) built-in function.
+ try {
+ BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
+ if (bife != null){
+ // It is a builtin function
+ setAssignmentStatement(target, bife, ctx);
+ return;
+ }
+
+ ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
+ if (pbife != null){
+ // It is a parameterized builtin function
+ setAssignmentStatement(target, pbife, ctx);
+ return;
+ }
+
+ // built-in read, rand ...
+ DataExpression dbife = DataExpression.getDataExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
+ if (dbife != null){
+ setAssignmentStatement(target, dbife, ctx);
+ return;
+ }
+ } catch(Exception e) {
+ helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
+ return ;
+ }
+ }
+
+ setAssignmentStatement(target, functCall, ctx);
+ }
+
+ @Override
+ public void exitBuiltinFunctionExpression(BuiltinFunctionExpressionContext ctx) {
+// if(!helper.validateBuiltinFunctions(ctx)) {
+// return; // it is a built-in function and validation failed, so donot proceed ahead.
+// }
+ // Double verification: verify passed function name is a (non-parameterized) built-in function.
+ String functionName = ctx.name.getText();
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+ ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
+// if(functionName.compareTo("read") == 0 && paramExpression.size() > 0 && paramExpression.get(0).getName() == null) {
+// paramExpression.get(0).setName(DataExpression.IO_FILENAME);
+// }
+
+ try {
+ BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, paramExpression, fileName, line, col, line, col);
+ if (bife != null){
+ // It is a builtin function
+ ctx.info.expr = bife;
+ setFileLineColumn(ctx.info.expr, ctx);
+ return;
+ }
+
+ ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, paramExpression, fileName, line, col, line, col);
+ if (pbife != null){
+ // It is a parameterized builtin function
+ ctx.info.expr = pbife;
+ setFileLineColumn(ctx.info.expr, ctx);
+ return;
+ }
+
+ // built-in read, rand ...
+ DataExpression dbife = DataExpression.getDataExpression(functionName, paramExpression, fileName, line, col, line, col);
+ if (dbife != null){
+ ctx.info.expr = dbife;
+ setFileLineColumn(ctx.info.expr, ctx);
+ return;
+ }
+
+ } catch(Exception e) {
+ helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
+ return ;
+ }
+ helper.notifyErrorListeners("only builtin functions allowed as part of expression", ctx.start);
+ }
+
+ private void setMultiAssignmentStatement(ArrayList<DataIdentifier> target, Expression expression, StatementContext ctx) {
+ ctx.info.stmt = new MultiAssignmentStatement(target, expression);
+ ctx.info.stmt.setAllPositions(helper.getCurrentFileName(), ctx.start.getLine(), ctx.start.getCharPositionInLine(), ctx.start.getLine(), ctx.start.getCharPositionInLine());
+ setFileLineColumn(ctx.info.stmt, ctx);
+ }
+
+ @Override
+ public void exitFunctionCallMultiAssignmentStatement(
+ FunctionCallMultiAssignmentStatementContext ctx) {
+ String fullyQualifiedFunctionName = ctx.name.getText();
+ String [] fnNames = fullyQualifiedFunctionName.split("::");
+ String functionName = "";
+ String namespace = "";
+ if(fnNames.length == 1) {
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ functionName = fnNames[0].trim();
+ }
+ else if(fnNames.length == 2) {
+ namespace = fnNames[0].trim();
+ functionName = fnNames[1].trim();
+ }
+ else {
+ helper.notifyErrorListeners("incorrect function name", ctx.name);
+ return;
+ }
+
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+
+ ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
+// if(functionName.compareTo("read") == 0 && paramExpression.size() > 0 && paramExpression.get(0).getName() == null) {
+// paramExpression.get(0).setName(DataExpression.IO_FILENAME);
+// }
+
+ FunctionCallIdentifier functCall = new FunctionCallIdentifier(paramExpression);
+ try {
+ functCall.setFunctionName(functionName);
+ functCall.setFunctionNamespace(namespace);
+ } catch (ParseException e1) {
+ helper.notifyErrorListeners("unable to process function " + functionName, ctx.start);
+ return;
+ }
+
+ ArrayList<DataIdentifier> targetList = new ArrayList<DataIdentifier>();
+ for(DataIdentifierContext dataCtx : ctx.targetList) {
+ if(dataCtx.dataInfo.expr instanceof DataIdentifier) {
+ targetList.add((DataIdentifier) dataCtx.dataInfo.expr);
+ }
+ else {
+ helper.notifyErrorListeners("incorrect lvalue ... strange", dataCtx.start);
+ //target = new DataIdentifier(); // so as not to avoid null pointer
+ return;
+ }
+ }
+
+ if(!functionName.contains("::") || functionName.startsWith(DMLProgram.DEFAULT_NAMESPACE)) {
+ // In global namespace, so it can be a builtin function
+// if(!helper.validateBuiltinFunctions(ctx)) {
+// return; // it is a built-in function and validation failed, so donot proceed ahead.
+// }
+ // Double verification: verify passed function name is a (non-parameterized) built-in function.
+ try {
+ BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
+ if (bife != null){
+ // It is a builtin function
+ setMultiAssignmentStatement(targetList, bife, ctx);
+ return;
+ }
+
+ ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
+ if (pbife != null){
+ // It is a parameterized builtin function
+ setMultiAssignmentStatement(targetList, pbife, ctx);
+ return;
+ }
+
+ // built-in read, rand ...
+ DataExpression dbife = DataExpression.getDataExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
+ if (dbife != null){
+ setMultiAssignmentStatement(targetList, dbife, ctx);
+ return;
+ }
+ } catch(Exception e) {
+ helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
+ return;
+ }
+ }
+
+ setMultiAssignmentStatement(targetList, functCall, ctx);
+ }
+
+ private StatementBlock getStatementBlock(Statement current) {
+ return DMLParserWrapper.getStatementBlock(current);
+ }
+
+ @Override
+ public void exitIfStatement(IfStatementContext ctx) {
+ IfStatement ifStmt = new IfStatement();
+ ConditionalPredicate predicate = new ConditionalPredicate(ctx.predicate.info.expr);
+ ifStmt.setConditionalPredicate(predicate);
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+ ifStmt.setAllPositions(fileName, line, col, line, col);
+
+ if(ctx.ifBody.size() > 0) {
+ for(StatementContext stmtCtx : ctx.ifBody) {
+ ifStmt.addStatementBlockIfBody(getStatementBlock(stmtCtx.info.stmt));
+ }
+ ifStmt.mergeStatementBlocksIfBody();
+ }
+
+ if(ctx.elseBody.size() > 0) {
+ for(StatementContext stmtCtx : ctx.elseBody) {
+ ifStmt.addStatementBlockElseBody(getStatementBlock(stmtCtx.info.stmt));
+ }
+ ifStmt.mergeStatementBlocksElseBody();
+ }
+
+ ctx.info.stmt = ifStmt;
+ setFileLineColumn(ctx.info.stmt, ctx);
+ }
+
+ @Override
+ public void exitWhileStatement(WhileStatementContext ctx) {
+ WhileStatement whileStmt = new WhileStatement();
+ ConditionalPredicate predicate = new ConditionalPredicate(ctx.predicate.info.expr);
+ whileStmt.setPredicate(predicate);
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+ whileStmt.setAllPositions(fileName, line, col, line, col);
+
+ if(ctx.body.size() > 0) {
+ for(StatementContext stmtCtx : ctx.body) {
+ whileStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
+ }
+ whileStmt.mergeStatementBlocks();
+ }
+
+ ctx.info.stmt = whileStmt;
+ setFileLineColumn(ctx.info.stmt, ctx);
+ }
+
+ @Override
+ public void exitForStatement(ForStatementContext ctx) {
+ ForStatement forStmt = new ForStatement();
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+
+ DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText());
+ HashMap<String, String> parForParamValues = null;
+ Expression incrementExpr = new IntIdentifier(1, fileName, line, col, line, col);
+ if(ctx.iterPred.info.increment != null) {
+ incrementExpr = ctx.iterPred.info.increment;
+ }
+ IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, fileName, line, col, line, col);
+ forStmt.setPredicate(predicate);
+
+ if(ctx.body.size() > 0) {
+ for(StatementContext stmtCtx : ctx.body) {
+ forStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
+ }
+ forStmt.mergeStatementBlocks();
+ }
+ ctx.info.stmt = forStmt;
+ setFileLineColumn(ctx.info.stmt, ctx);
+ }
+
+ @Override
+ public void exitParForStatement(ParForStatementContext ctx) {
+ ParForStatement parForStmt = new ParForStatement();
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+
+ DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText());
+ HashMap<String, String> parForParamValues = new HashMap<String, String>();
+ if(ctx.parForParams != null && ctx.parForParams.size() > 0) {
+ for(StrictParameterizedExpressionContext parForParamCtx : ctx.parForParams) {
+ parForParamValues.put(parForParamCtx.paramName.getText(), parForParamCtx.paramVal.getText());
+ }
+ }
+
+ Expression incrementExpr = new IntIdentifier(1, fileName, line, col, line, col);
+
+ if( ctx.iterPred.info.increment != null ) {
+ incrementExpr = ctx.iterPred.info.increment;
+ }
+ IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, fileName, line, col, line, col);
+ parForStmt.setPredicate(predicate);
+ if(ctx.body.size() > 0) {
+ for(StatementContext stmtCtx : ctx.body) {
+ parForStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
+ }
+ parForStmt.mergeStatementBlocks();
+ }
+ ctx.info.stmt = parForStmt;
+ setFileLineColumn(ctx.info.stmt, ctx);
+ }
+
+
+ @Override
+ public void exitDmlprogram(DmlprogramContext ctx) { }
+
+
+ // ----------------------------------------------------------------------
+ @Override
+ public void exitValueType(ValueTypeContext ctx) { }
+ @Override
+ public void exitMl_type(Ml_typeContext ctx) { }
+
+ private ArrayList<DataIdentifier> getFunctionParameters(List<TypedArgNoAssignContext> ctx) {
+ ArrayList<DataIdentifier> retVal = new ArrayList<DataIdentifier>();
+ for(TypedArgNoAssignContext paramCtx : ctx) {
+ DataIdentifier dataId = new DataIdentifier(paramCtx.paramName.getText());
+ String dataType = null;
+ String valueType = null;
+
+ if(paramCtx.paramType == null || paramCtx.paramType.dataType() == null
+ || paramCtx.paramType.dataType().getText() == null || paramCtx.paramType.dataType().getText().isEmpty()) {
+ dataType = "scalar";
+ }
+ else {
+ dataType = paramCtx.paramType.dataType().getText();
+ }
+
+ if(dataType.compareTo("matrix") == 0 || dataType.compareTo("Matrix") == 0) {
+ // matrix
+ dataId.setDataType(DataType.MATRIX);
+ }
+ else if(dataType.compareTo("scalar") == 0 || dataType.compareTo("Scalar") == 0) {
+ // scalar
+ dataId.setDataType(DataType.SCALAR);
+ }
+ else {
+ helper.notifyErrorListeners("invalid datatype " + dataType, paramCtx.start);
+ return null;
+ }
+
+ valueType = paramCtx.paramType.valueType().getText();
+ if(valueType.compareTo("int") == 0 || valueType.compareTo("integer") == 0
+ || valueType.compareTo("Int") == 0 || valueType.compareTo("Integer") == 0) {
+ dataId.setValueType(ValueType.INT);
+ }
+ else if(valueType.compareTo("string") == 0 || valueType.compareTo("String") == 0) {
+ dataId.setValueType(ValueType.STRING);
+ }
+ else if(valueType.compareTo("boolean") == 0 || valueType.compareTo("Boolean") == 0) {
+ dataId.setValueType(ValueType.BOOLEAN);
+ }
+ else if(valueType.compareTo("double") == 0 || valueType.compareTo("Double") == 0) {
+ dataId.setValueType(ValueType.DOUBLE);
+ }
+ else if(valueType.compareTo("bool") == 0) {
+ helper.notifyErrorListeners("invalid valuetype " + valueType + " (Quickfix: use \'boolean\' instead)", paramCtx.start);
+ return null;
+ }
+ else {
+ helper.notifyErrorListeners("invalid valuetype " + valueType, paramCtx.start);
+ return null;
+ }
+ retVal.add(dataId);
+ }
+ return retVal;
+ }
+
+ @Override
+ public void exitInternalFunctionDefExpression(InternalFunctionDefExpressionContext ctx) {
+ FunctionStatement functionStmt = new FunctionStatement();
+
+ ArrayList<DataIdentifier> functionInputs = getFunctionParameters(ctx.inputParams);
+ functionStmt.setInputParams(functionInputs);
+
+ // set function outputs
+ ArrayList<DataIdentifier> functionOutputs = getFunctionParameters(ctx.outputParams);
+ functionStmt.setOutputParams(functionOutputs);
+
+ // set function name
+ functionStmt.setName(ctx.name.getText());
+
+
+ if(ctx.body.size() > 0) {
+ // handle function body
+ // Create arraylist of one statement block
+ ArrayList<StatementBlock> body = new ArrayList<StatementBlock>();
+ for(StatementContext stmtCtx : ctx.body) {
+ body.add(getStatementBlock(stmtCtx.info.stmt));
+ }
+ functionStmt.setBody(body);
+ functionStmt.mergeStatementBlocks();
+ }
+ else {
+ helper.notifyErrorListeners("functions with no statements are not allowed", ctx.start);
+ return;
+ }
+
+ ctx.info.stmt = functionStmt;
+ setFileLineColumn(ctx.info.stmt, ctx);
+ ctx.info.functionName = ctx.name.getText();
+ }
+
+ @Override
+ public void exitExternalFunctionDefExpression(ExternalFunctionDefExpressionContext ctx) {
+ ExternalFunctionStatement functionStmt = new ExternalFunctionStatement();
+
+ ArrayList<DataIdentifier> functionInputs = getFunctionParameters(ctx.inputParams);
+ functionStmt.setInputParams(functionInputs);
+
+ // set function outputs
+ ArrayList<DataIdentifier> functionOutputs = getFunctionParameters(ctx.outputParams);
+ functionStmt.setOutputParams(functionOutputs);
+
+ // set function name
+ functionStmt.setName(ctx.name.getText());
+
+ // set other parameters
+ HashMap<String, String> otherParams = new HashMap<String,String>();
+ boolean atleastOneClassName = false;
+ for(StrictParameterizedKeyValueStringContext otherParamCtx : ctx.otherParams){
+ String paramName = otherParamCtx.paramName.getText();
+ String val = "";
+ String text = otherParamCtx.paramVal.getText();
+ // First unquote the string
+ if( (text.startsWith("\"") && text.endsWith("\"")) ||
+ (text.startsWith("\'") && text.endsWith("\'"))) {
+ if(text.length() > 2) {
+ val = text.substring(1, text.length()-1);
+ }
+ // Empty value allowed
+ }
+ else {
+ helper.notifyErrorListeners("the value of user parameter for external function should be of type string", ctx.start);
+ return;
+ }
+ otherParams.put(paramName, val);
+ if(paramName.compareTo("classname") == 0) {
+ atleastOneClassName = true;
+ }
+ }
+ functionStmt.setOtherParams(otherParams);
+ if(!atleastOneClassName) {
+ helper.notifyErrorListeners("the parameter \'className\' needs to be passed for externalFunction", ctx.start);
+ return;
+ }
+
+// if(ctx.body.size() > 0) {
+// // handle function body
+// // Create arraylist of one statement block
+// ArrayList<StatementBlock> body = new ArrayList<StatementBlock>();
+// for(StatementContext stmtCtx : ctx.body) {
+// body.add(getStatementBlock(stmtCtx.info.stmt));
+// }
+// ((ExternalFunctionStatement) functionStmt).setBody(body);
+// ((ExternalFunctionStatement) functionStmt).mergeStatementBlocks();
+// }
+// else {
+// helper.notifyErrorListeners("functions with no statements are not allowed", ctx.start);
+// return;
+// }
+
+ ctx.info.stmt = functionStmt;
+ setFileLineColumn(ctx.info.stmt, ctx);
+ ctx.info.functionName = ctx.name.getText();
+ }
+
+
+ @Override
+ public void exitPathStatement(PathStatementContext ctx) {
+ PathStatement stmt = new PathStatement(ctx.pathValue.getText());
+ String filePath = ctx.pathValue.getText();
+ if((filePath.startsWith("\"") && filePath.endsWith("\"")) ||
+ filePath.startsWith("'") && filePath.endsWith("'")) {
+ filePath = filePath.substring(1, filePath.length()-1);
+ }
+
+ _workingDir = filePath;
+ ctx.info.stmt = stmt;
+ }
+
+ @Override
+ public void exitIfdefAssignmentStatement(IfdefAssignmentStatementContext ctx) {
+ if(!ctx.commandLineParam.getText().startsWith("$")) {
+ helper.notifyErrorListeners("the first argument of ifdef function should be a commandline argument parameter (which starts with $)", ctx.commandLineParam.start);
+ return;
+ }
+
+ if(ctx.targetList == null || ctx.targetList.size() != 1) {
+ helper.notifyErrorListeners("incorrect parsing for ifdef function", ctx.start);
+ return;
+ }
+ String targetListText = ctx.targetList.get(0).getText();
+ if(targetListText.startsWith("$")) {
+ helper.notifyErrorListeners("lhs of ifdef function cannot be a commandline parameters. Use local variable instead", ctx.start);
+ return;
+ }
+
+ DataIdentifier target = null;
+ if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
+ target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
+ Expression source = null;
+ if(ctx.commandLineParam.dataInfo.expr != null) {
+ // Since commandline parameter is set
+ // The check of following is done in fillExpressionInfoCommandLineParameters:
+ // Command line param cannot be empty string
+ // If you want to pass space, please quote it
+ source = ctx.commandLineParam.dataInfo.expr;
+ }
+ else {
+ source = ctx.source.info.expr;
+ }
+
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+ try {
+ ctx.info.stmt = new AssignmentStatement(target, source, line, col, line, col);
+ setFileLineColumn(ctx.info.stmt, ctx);
+ } catch (LanguageException e) {
+ helper.notifyErrorListeners("invalid assignment for ifdef function", ctx.targetList.get(0).start);
+ return;
+ }
+
+ }
+ else {
+ helper.notifyErrorListeners("incorrect lvalue in ifdef function... strange", ctx.targetList.get(0).start);
+ return;
+ }
+
+ }
+
+ // ----------------------------------------------------------------------
+ @Override
+ public void exitParameterizedExpression(ParameterizedExpressionContext ctx) { }
+
+
+ @Override
+ public void exitStrictParameterizedExpression(StrictParameterizedExpressionContext ctx) { }
+
+ @Override
+ public void exitTypedArgNoAssign(TypedArgNoAssignContext ctx) { }
+ @Override
+ public void enterIfdefAssignmentStatement(IfdefAssignmentStatementContext ctx) { }
+ @Override
+ public void enterMatrixDataTypeCheck(MatrixDataTypeCheckContext ctx) { }
+ @Override
+ public void exitMatrixDataTypeCheck(MatrixDataTypeCheckContext ctx) {
+ if( ctx.ID().getText().compareTo("matrix") == 0
+ || ctx.ID().getText().compareTo("Matrix") == 0
+ || ctx.ID().getText().compareTo("Scalar") == 0
+ || ctx.ID().getText().compareTo("scalar") == 0
+ ) {
+ // Do nothing
+ }
+ else {
+ helper.notifyErrorListeners("incorrect datatype (expected matrix or scalar)", ctx.start);
+ }
+ }
+
+ @Override
+ public void enterBuiltinFunctionExpression(BuiltinFunctionExpressionContext ctx) {}
+ @Override
+ public void enterStrictParameterizedKeyValueString(StrictParameterizedKeyValueStringContext ctx) { }
+ @Override
+ public void exitStrictParameterizedKeyValueString(StrictParameterizedKeyValueStringContext ctx) {}
+ @Override
+ public void enterIterablePredicateColonExpression(IterablePredicateColonExpressionContext ctx) {}
+ @Override
+ public void enterIterablePredicateSeqExpression(IterablePredicateSeqExpressionContext ctx) { }
+
+ @Override
+ public void exitIterablePredicateColonExpression(IterablePredicateColonExpressionContext ctx) {
+ ctx.info.from = ctx.from.info.expr;
+ ctx.info.to = ctx.to.info.expr;
+ ctx.info.increment = null;
+ }
+
+ @Override
+ public void exitIterablePredicateSeqExpression(IterablePredicateSeqExpressionContext ctx) {
+ if(ctx.ID().getText().compareTo("seq") != 0) {
+ helper.notifyErrorListeners("incorrect function:\'" + ctx.ID().getText() + "\'. expected \'seq\'", ctx.start);
+ return;
+ }
+ ctx.info.from = ctx.from.info.expr;
+ ctx.info.to = ctx.to.info.expr;
+ ctx.info.increment = ctx.increment.info.expr;
+ }
+
+ @Override
+ public void enterConstFalseExpression(ConstFalseExpressionContext ctx) { }
+ @Override
+ public void enterConstTrueExpression(ConstTrueExpressionContext ctx) { }
+
+ @Override
+ public void exitConstFalseExpression(ConstFalseExpressionContext ctx) {
+ boolean val = false;
+ int linePosition = ctx.start.getLine();
+ int charPosition = ctx.start.getCharPositionInLine();
+ ctx.info.expr = new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+
+
+ @Override
+ public void exitConstTrueExpression(ConstTrueExpressionContext ctx) {
+ boolean val = true;
+ int linePosition = ctx.start.getLine();
+ int charPosition = ctx.start.getCharPositionInLine();
+ ctx.info.expr = new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidatorHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidatorHelper.java b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidatorHelper.java
new file mode 100644
index 0000000..a1bd06f
--- /dev/null
+++ b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidatorHelper.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.parser.dml;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.antlr.v4.runtime.Token;
+
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.dml.DmlParser.FunctionCallAssignmentStatementContext;
+import org.apache.sysml.parser.dml.DmlParser.ParameterizedExpressionContext;
+import org.apache.sysml.parser.dml.DmlSyntacticErrorListener.CustomDmlErrorListener;
+
+public class DmlSyntacticValidatorHelper {
+
+ private CustomDmlErrorListener _errorListener = null;
+
+ public DmlSyntacticValidatorHelper(CustomDmlErrorListener errorListener) {
+ this._errorListener = errorListener;
+ }
+
+ public void notifyErrorListeners(String message, int line, int charPositionInLine) {
+ this._errorListener.validationError(line, charPositionInLine, message);
+ }
+
+ public void notifyErrorListeners(String message, Token op) {
+ this._errorListener.validationError(op.getLine(), op.getCharPositionInLine(), message);
+ }
+
+ public void raiseWarning(String message, Token op) {
+ this._errorListener.validationWarning(op.getLine(), op.getCharPositionInLine(), message);
+ }
+
+ public String getCurrentFileName() {
+ return _errorListener.peekFileName();
+ }
+
+// public static void setInfoForArithmeticOp(org.apache.sysml.parser.Expression current,
+// org.apache.sysml.parser.Expression left,
+// org.apache.sysml.parser.Expression right, String opStr) {
+// try {
+// // PLUS, MINUS, MULT, DIV, MODULUS, INTDIV, MATMULT, POW, INVALID
+// org.apache.sysml.parser.Expression.BinaryOp bop = org.apache.sysml.parser.Expression.getBinaryOp(opStr);
+// current = new org.apache.sysml.parser.BinaryExpression(bop);
+// ((org.apache.sysml.parser.BinaryExpression)current).setLeft(left);
+// ((org.apache.sysml.parser.BinaryExpression)current).setRight(right);
+// ((org.apache.sysml.parser.BinaryExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
+// }
+// catch(Exception e) {
+// System.out.println("In setInfoForArithmeticOp>>");
+// e.printStackTrace();
+// }
+// }
+
+// public static void setInfoForBooleanOp(org.apache.sysml.parser.Expression current,
+// org.apache.sysml.parser.Expression left,
+// org.apache.sysml.parser.Expression right, String opStr) {
+// org.apache.sysml.parser.Expression.BooleanOp bop = org.apache.sysml.parser.Expression.getBooleanOp(opStr);
+// current = new org.apache.sysml.parser.BooleanExpression(bop);
+// ((org.apache.sysml.parser.BooleanExpression)current).setLeft(left);
+// ((org.apache.sysml.parser.BooleanExpression)current).setRight(right);
+// ((org.apache.sysml.parser.BooleanExpression)current).setFilename(DmlSyntacticErrorListener.currentFileName.peek());
+// }
+
+ public boolean validateBuiltinFunctions(FunctionCallAssignmentStatementContext ctx) {
+ String functionName = ctx.name.getText().replaceAll(" ", "").trim();
+ if(functionName.compareTo("write") == 0 || functionName.compareTo(DMLProgram.DEFAULT_NAMESPACE + "::write") == 0) {
+ return validateBuiltinWriteFunction(ctx);
+ }
+ return true;
+ }
+
+ private boolean validateBuiltinWriteFunction(FunctionCallAssignmentStatementContext ctx) {
+
+ return true;
+ }
+
+ public ArrayList<org.apache.sysml.parser.ParameterExpression> getParameterExpressionList(List<ParameterizedExpressionContext> paramExprs) {
+ ArrayList<org.apache.sysml.parser.ParameterExpression> retVal = new ArrayList<org.apache.sysml.parser.ParameterExpression>();
+ for(ParameterizedExpressionContext ctx : paramExprs) {
+ String paramName = null;
+ if(ctx.paramName != null && ctx.paramName.getText() != null && !ctx.paramName.getText().isEmpty()) {
+ paramName = ctx.paramName.getText();
+ }
+ org.apache.sysml.parser.ParameterExpression myArg = new org.apache.sysml.parser.ParameterExpression(paramName, ctx.paramVal.info.expr);
+ retVal.add(myArg);
+ }
+ return retVal;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/dml/ExpressionInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/dml/ExpressionInfo.java b/src/main/java/org/apache/sysml/parser/dml/ExpressionInfo.java
new file mode 100644
index 0000000..436c409
--- /dev/null
+++ b/src/main/java/org/apache/sysml/parser/dml/ExpressionInfo.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.parser.dml;
+
+
+public class ExpressionInfo {
+
+ public org.apache.sysml.parser.Expression expr = null;
+
+ // For parfor and for
+ public org.apache.sysml.parser.Expression from = null;
+ public org.apache.sysml.parser.Expression to = null;
+ public org.apache.sysml.parser.Expression increment = null;
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/dml/StatementInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/dml/StatementInfo.java b/src/main/java/org/apache/sysml/parser/dml/StatementInfo.java
new file mode 100644
index 0000000..fd4c50e
--- /dev/null
+++ b/src/main/java/org/apache/sysml/parser/dml/StatementInfo.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.parser.dml;
+
+import java.util.HashMap;
+import org.apache.sysml.parser.DMLProgram;
+
+public class StatementInfo {
+
+ public org.apache.sysml.parser.Statement stmt = null;
+
+ // Valid only for import statements
+ public HashMap<String,DMLProgram> namespaces = null;
+
+ // Valid only for function statement
+ //public String namespace = DMLProgram.DEFAULT_NAMESPACE;
+ public String functionName = "";
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/pydml/ExpressionInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/pydml/ExpressionInfo.java b/src/main/java/org/apache/sysml/parser/pydml/ExpressionInfo.java
new file mode 100644
index 0000000..6cf5773
--- /dev/null
+++ b/src/main/java/org/apache/sysml/parser/pydml/ExpressionInfo.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.parser.pydml;
+
+/**
+ * This class exists solely to prevent compiler warnings.
+ *
+ * <p>
+ * The ExpressionInfo and StatementInfo classes are shared among both parsers
+ * (R-like and Python-like dialects), and Antlr-generated code assumes that
+ * these classes are present in the parser's namespace.
+ */
+class ExpressionInfo extends org.apache.sysml.parser.dml.ExpressionInfo {
+
+}
[4/8] incubator-systemml git commit: [SYSTEMML-148] Refactored
.parser.{antlr4, python} to .parser.{dml, pydml}.
Posted by du...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java
new file mode 100644
index 0000000..58dce00
--- /dev/null
+++ b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java
@@ -0,0 +1,2040 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.parser.pydml;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.tree.ErrorNode;
+import org.antlr.v4.runtime.tree.TerminalNode;
+import org.apache.commons.lang.StringUtils;
+import org.apache.sysml.parser.AssignmentStatement;
+import org.apache.sysml.parser.BinaryExpression;
+import org.apache.sysml.parser.BooleanExpression;
+import org.apache.sysml.parser.BooleanIdentifier;
+import org.apache.sysml.parser.BuiltinFunctionExpression;
+import org.apache.sysml.parser.ConditionalPredicate;
+import org.apache.sysml.parser.ConstIdentifier;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.DataIdentifier;
+import org.apache.sysml.parser.DoubleIdentifier;
+import org.apache.sysml.parser.Expression;
+import org.apache.sysml.parser.Expression.DataOp;
+import org.apache.sysml.parser.Expression.DataType;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.parser.pydml.PydmlParser.AddSubExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.AssignmentStatementContext;
+import org.apache.sysml.parser.pydml.PydmlParser.AtomicExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.BooleanAndExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.BooleanNotExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.BooleanOrExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.BuiltinFunctionExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.CommandlineParamExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.CommandlinePositionExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.ConstDoubleIdExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.ConstFalseExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.ConstIntIdExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.ConstStringIdExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.ConstTrueExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.DataIdExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.DataIdentifierContext;
+import org.apache.sysml.parser.pydml.PydmlParser.ExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.ExternalFunctionDefExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.ForStatementContext;
+import org.apache.sysml.parser.pydml.PydmlParser.FunctionCallAssignmentStatementContext;
+import org.apache.sysml.parser.pydml.PydmlParser.FunctionCallMultiAssignmentStatementContext;
+import org.apache.sysml.parser.pydml.PydmlParser.FunctionStatementContext;
+import org.apache.sysml.parser.pydml.PydmlParser.IfStatementContext;
+import org.apache.sysml.parser.pydml.PydmlParser.IfdefAssignmentStatementContext;
+import org.apache.sysml.parser.pydml.PydmlParser.IgnoreNewLineContext;
+import org.apache.sysml.parser.pydml.PydmlParser.ImportStatementContext;
+import org.apache.sysml.parser.pydml.PydmlParser.IndexedExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.InternalFunctionDefExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.IterablePredicateColonExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.IterablePredicateSeqExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.MatrixDataTypeCheckContext;
+import org.apache.sysml.parser.pydml.PydmlParser.Ml_typeContext;
+import org.apache.sysml.parser.pydml.PydmlParser.ModIntDivExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.MultDivExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.ParForStatementContext;
+import org.apache.sysml.parser.pydml.PydmlParser.ParameterizedExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.PathStatementContext;
+import org.apache.sysml.parser.pydml.PydmlParser.PmlprogramContext;
+import org.apache.sysml.parser.pydml.PydmlParser.PowerExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.RelationalExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.SimpleDataIdentifierExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.StatementContext;
+import org.apache.sysml.parser.pydml.PydmlParser.StrictParameterizedExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.StrictParameterizedKeyValueStringContext;
+import org.apache.sysml.parser.pydml.PydmlParser.TypedArgNoAssignContext;
+import org.apache.sysml.parser.pydml.PydmlParser.UnaryExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlParser.ValueDataTypeCheckContext;
+import org.apache.sysml.parser.pydml.PydmlParser.WhileStatementContext;
+import org.apache.sysml.parser.AParserWrapper;
+import org.apache.sysml.parser.ExternalFunctionStatement;
+import org.apache.sysml.parser.ForStatement;
+import org.apache.sysml.parser.FunctionCallIdentifier;
+import org.apache.sysml.parser.FunctionStatement;
+import org.apache.sysml.parser.IfStatement;
+import org.apache.sysml.parser.ImportStatement;
+import org.apache.sysml.parser.IndexedIdentifier;
+import org.apache.sysml.parser.IntIdentifier;
+import org.apache.sysml.parser.IterablePredicate;
+import org.apache.sysml.parser.LanguageException;
+import org.apache.sysml.parser.MultiAssignmentStatement;
+import org.apache.sysml.parser.OutputStatement;
+import org.apache.sysml.parser.ParForStatement;
+import org.apache.sysml.parser.ParameterExpression;
+import org.apache.sysml.parser.ParameterizedBuiltinFunctionExpression;
+import org.apache.sysml.parser.ParseException;
+import org.apache.sysml.parser.PathStatement;
+import org.apache.sysml.parser.PrintStatement;
+import org.apache.sysml.parser.RelationalExpression;
+import org.apache.sysml.parser.Statement;
+import org.apache.sysml.parser.StatementBlock;
+import org.apache.sysml.parser.StringIdentifier;
+import org.apache.sysml.parser.WhileStatement;
+
+/**
+ * TODO: Refactor duplicated parser code dml/pydml (entire package).
+ *
+ */
+public class PydmlSyntacticValidator implements PydmlListener
+{
+ private PydmlSyntacticValidatorHelper helper = null;
+
+ private String _workingDir = "."; //current working directory
+ private String _currentPath = null; //current file path
+ private HashMap<String,String> argVals = null;
+
+ public PydmlSyntacticValidator(PydmlSyntacticValidatorHelper helper, String currentPath, HashMap<String,String> argVals) {
+ this.helper = helper;
+ this.argVals = argVals;
+
+ _currentPath = currentPath;
+ }
+
+ // Functions we have to implement but don't really need it
+ @Override
+ public void enterAddSubExpression(AddSubExpressionContext ctx) { }
+ @Override
+ public void enterAssignmentStatement(AssignmentStatementContext ctx) {}
+ @Override
+ public void enterAtomicExpression(AtomicExpressionContext ctx) { }
+ @Override
+ public void enterBooleanAndExpression(BooleanAndExpressionContext ctx) { }
+ @Override
+ public void enterBooleanNotExpression(BooleanNotExpressionContext ctx) { }
+ @Override
+ public void enterBooleanOrExpression(BooleanOrExpressionContext ctx) { }
+ @Override
+ public void enterCommandlineParamExpression(CommandlineParamExpressionContext ctx) { }
+ @Override
+ public void enterCommandlinePositionExpression(CommandlinePositionExpressionContext ctx) { }
+ @Override
+ public void enterConstDoubleIdExpression(ConstDoubleIdExpressionContext ctx) { }
+ @Override
+ public void enterConstIntIdExpression(ConstIntIdExpressionContext ctx) { }
+ @Override
+ public void enterConstStringIdExpression(ConstStringIdExpressionContext ctx) { }
+ @Override
+ public void enterDataIdExpression(DataIdExpressionContext ctx) { }
+
+ @Override
+ public void enterIgnoreNewLine(IgnoreNewLineContext ctx) { }
+ @Override
+ public void enterPmlprogram(PmlprogramContext ctx) { }
+ @Override
+ public void exitPmlprogram(PmlprogramContext ctx) { }
+
+ @Override
+ public void enterEveryRule(ParserRuleContext arg0) {
+ if(arg0 instanceof StatementContext) {
+ if(((StatementContext) arg0).info == null) {
+ ((StatementContext) arg0).info = new StatementInfo();
+ }
+ }
+ if(arg0 instanceof FunctionStatementContext) {
+ if(((FunctionStatementContext) arg0).info == null) {
+ ((FunctionStatementContext) arg0).info = new StatementInfo();
+ }
+ }
+ if(arg0 instanceof ExpressionContext) {
+ if(((ExpressionContext) arg0).info == null) {
+ ((ExpressionContext) arg0).info = new ExpressionInfo();
+ }
+ }
+ if(arg0 instanceof DataIdentifierContext) {
+ if(((DataIdentifierContext) arg0).dataInfo == null) {
+ ((DataIdentifierContext) arg0).dataInfo = new ExpressionInfo();
+ }
+ }
+ }
+ @Override
+ public void enterExternalFunctionDefExpression(ExternalFunctionDefExpressionContext ctx) { }
+ @Override
+ public void enterForStatement(ForStatementContext ctx) {}
+ @Override
+ public void enterFunctionCallAssignmentStatement(FunctionCallAssignmentStatementContext ctx) { }
+ @Override
+ public void enterFunctionCallMultiAssignmentStatement(FunctionCallMultiAssignmentStatementContext ctx) { }
+ @Override
+ public void enterIfStatement(IfStatementContext ctx) { }
+ @Override
+ public void enterImportStatement(ImportStatementContext ctx) { }
+ @Override
+ public void enterIndexedExpression(IndexedExpressionContext ctx) { }
+ @Override
+ public void enterInternalFunctionDefExpression(InternalFunctionDefExpressionContext ctx) { }
+ @Override
+ public void enterMl_type(Ml_typeContext ctx) { }
+ @Override
+ public void enterModIntDivExpression(ModIntDivExpressionContext ctx) { }
+ @Override
+ public void enterMultDivExpression(MultDivExpressionContext ctx) { }
+ @Override
+ public void enterParameterizedExpression(ParameterizedExpressionContext ctx) { }
+ @Override
+ public void enterParForStatement(ParForStatementContext ctx) { }
+ @Override
+ public void enterPathStatement(PathStatementContext ctx) { }
+ @Override
+ public void enterPowerExpression(PowerExpressionContext ctx) { }
+ @Override
+ public void enterRelationalExpression(RelationalExpressionContext ctx) { }
+ @Override
+ public void enterSimpleDataIdentifierExpression(SimpleDataIdentifierExpressionContext ctx) { }
+ @Override
+ public void enterStrictParameterizedExpression(StrictParameterizedExpressionContext ctx) { }
+ @Override
+ public void enterTypedArgNoAssign(TypedArgNoAssignContext ctx) { }
+ @Override
+ public void enterUnaryExpression(UnaryExpressionContext ctx) { }
+ @Override
+ public void enterWhileStatement(WhileStatementContext ctx) { }
+
+ @Override
+ public void visitErrorNode(ErrorNode arg0) { }
+ @Override
+ public void visitTerminal(TerminalNode arg0) { }
+ @Override
+ public void exitEveryRule(ParserRuleContext arg0) {}
+ // --------------------------------------------------------------------
+ private void setFileLineColumn(Expression expr, ParserRuleContext ctx) {
+ // expr.setFilename(helper.getCurrentFileName());
+ String txt = ctx.getText();
+ expr.setFilename(_currentPath);
+ expr.setBeginLine(ctx.start.getLine());
+ expr.setBeginColumn(ctx.start.getCharPositionInLine());
+ expr.setEndLine(ctx.stop.getLine());
+ expr.setEndColumn(ctx.stop.getCharPositionInLine());
+ if(expr.getBeginColumn() == expr.getEndColumn() && expr.getBeginLine() == expr.getEndLine() && txt.length() > 1) {
+ expr.setEndColumn(expr.getBeginColumn() + txt.length() - 1);
+ }
+ }
+
+ private void setFileLineColumn(Statement stmt, ParserRuleContext ctx) {
+ String txt = ctx.getText();
+ stmt.setFilename(helper.getCurrentFileName());
+ stmt.setBeginLine(ctx.start.getLine());
+ stmt.setBeginColumn(ctx.start.getCharPositionInLine());
+ stmt.setEndLine(ctx.stop.getLine());
+ stmt.setEndColumn(ctx.stop.getCharPositionInLine());
+ if(stmt.getBeginColumn() == stmt.getEndColumn() && stmt.getBeginLine() == stmt.getEndLine() && txt.length() > 1) {
+ stmt.setEndColumn(stmt.getBeginColumn() + txt.length() - 1);
+ }
+ }
+
+ // For now do no type checking, let validation handle it.
+ // This way parser doesn't have to open metadata file
+ @Override
+ public void exitAddSubExpression(AddSubExpressionContext ctx) {
+ if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
+ // Addition and subtraction operator same as DML
+ Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
+ ctx.info.expr = new BinaryExpression(bop);
+ ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ }
+
+
+
+ @Override
+ public void exitModIntDivExpression(ModIntDivExpressionContext ctx) {
+ if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
+ String dmlOperator = "";
+ if(ctx.op.getText().compareTo("//") == 0) {
+ dmlOperator = "%/%";
+ }
+ else if(ctx.op.getText().compareTo("%") == 0) {
+ dmlOperator = "%%";
+ }
+ else {
+ helper.notifyErrorListeners("Incorrect operator (expected // or %)", ctx.op);
+ return;
+ }
+ Expression.BinaryOp bop = Expression.getBinaryOp(dmlOperator);
+ ctx.info.expr = new BinaryExpression(bop);
+ ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ }
+
+ @Override
+ public void exitUnaryExpression(UnaryExpressionContext ctx) {
+ if(ctx.left.info.expr != null) {
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+
+ if(ctx.left.info.expr instanceof IntIdentifier) {
+ if(ctx.op.getText().compareTo("-") == 0) {
+ ((IntIdentifier) ctx.left.info.expr).multiplyByMinusOne();
+ }
+ ctx.info.expr = ctx.left.info.expr;
+ }
+ else if(ctx.left.info.expr instanceof DoubleIdentifier) {
+ if(ctx.op.getText().compareTo("-") == 0) {
+ ((DoubleIdentifier) ctx.left.info.expr).multiplyByMinusOne();
+ }
+ ctx.info.expr = ctx.left.info.expr;
+ }
+ else {
+ Expression right = new IntIdentifier(1, fileName, line, col, line, col);
+ if(ctx.op.getText().compareTo("-") == 0) {
+ right = new IntIdentifier(-1, fileName, line, col, line, col);
+ }
+
+ Expression.BinaryOp bop = Expression.getBinaryOp("*");
+ ctx.info.expr = new BinaryExpression(bop);
+ ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ ((BinaryExpression)ctx.info.expr).setRight(right);
+ }
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ }
+
+ @Override
+ public void exitMultDivExpression(MultDivExpressionContext ctx) {
+ Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
+ ctx.info.expr = new BinaryExpression(bop);
+ ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+
+ @Override
+ public void exitPowerExpression(PowerExpressionContext ctx) {
+ String dmlOperator = "";
+ if(ctx.op.getText().compareTo("**") == 0) {
+ dmlOperator = "^";
+ }
+ else {
+ helper.notifyErrorListeners("Incorrect operator (expected **)", ctx.op);
+ return;
+ }
+
+ Expression.BinaryOp bop = Expression.getBinaryOp(dmlOperator);
+ ctx.info.expr = new BinaryExpression(bop);
+ ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+
+ // TODO:
+// @Override
+// public void exitMatrixMulExpression(MatrixMulExpressionContext ctx) {
+// Expression.BinaryOp bop = Expression.getBinaryOp(ctx.op.getText());
+// ctx.info.expr = new BinaryExpression(bop);
+// ((BinaryExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+// ((BinaryExpression)ctx.info.expr).setRight(ctx.right.info.expr);
+// setFileLineColumn(ctx.info.expr, ctx);
+// }
+
+ // --------------------------------------------------------------------
+
+ @Override
+ public void exitRelationalExpression(RelationalExpressionContext ctx) {
+ if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
+ Expression.RelationalOp rop = Expression.getRelationalOp(ctx.op.getText());
+ ctx.info.expr = new RelationalExpression(rop);
+ ((RelationalExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ ((RelationalExpression)ctx.info.expr).setRight(ctx.right.info.expr);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ }
+
+ // --------------------------------------------------------------------
+
+ @Override
+ public void exitBooleanAndExpression(BooleanAndExpressionContext ctx) {
+ if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
+ String dmlOperator = "";
+ if(ctx.op.getText().compareTo("&") == 0 || ctx.op.getText().compareTo("and") == 0) {
+ dmlOperator = "&";
+ }
+ else {
+ helper.notifyErrorListeners("Incorrect operator (expected &)", ctx.op);
+ return;
+ }
+
+ Expression.BooleanOp bop = Expression.getBooleanOp(dmlOperator);
+ ctx.info.expr = new BooleanExpression(bop);
+ ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ ((BooleanExpression)ctx.info.expr).setRight(ctx.right.info.expr);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ }
+
+ @Override
+ public void exitBooleanOrExpression(BooleanOrExpressionContext ctx) {
+ if(ctx.left.info.expr != null && ctx.right.info.expr != null) {
+ String dmlOperator = "";
+ if(ctx.op.getText().compareTo("|") == 0 || ctx.op.getText().compareTo("or") == 0) {
+ dmlOperator = "|";
+ }
+ else {
+ helper.notifyErrorListeners("Incorrect operator (expected |)", ctx.op);
+ return;
+ }
+
+ Expression.BooleanOp bop = Expression.getBooleanOp(dmlOperator);
+ ctx.info.expr = new BooleanExpression(bop);
+ ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ ((BooleanExpression)ctx.info.expr).setRight(ctx.right.info.expr);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ }
+
+ @Override
+ public void exitBooleanNotExpression(BooleanNotExpressionContext ctx) {
+ if(ctx.left.info.expr != null) {
+ Expression.BooleanOp bop = Expression.getBooleanOp(ctx.op.getText());
+ ctx.info.expr = new BooleanExpression(bop);
+ ((BooleanExpression)ctx.info.expr).setLeft(ctx.left.info.expr);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ }
+
+ // --------------------------------------------------------------------
+
+ @Override
+ public void exitAtomicExpression(AtomicExpressionContext ctx) {
+ ctx.info.expr = ctx.left.info.expr;
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+
+ @Override
+ public void exitConstDoubleIdExpression(ConstDoubleIdExpressionContext ctx) {
+ try {
+ double val = Double.parseDouble(ctx.getText());
+ int linePosition = ctx.start.getLine();
+ int charPosition = ctx.start.getCharPositionInLine();
+ ctx.info.expr = new DoubleIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ catch(Exception e) {
+ helper.notifyErrorListeners("cannot parse the float value: \'" + ctx.getText() + "\'", ctx.getStart());
+ return;
+ }
+ }
+
+ @Override
+ public void exitConstIntIdExpression(ConstIntIdExpressionContext ctx) {
+ try {
+ long val = Long.parseLong(ctx.getText());
+ int linePosition = ctx.start.getLine();
+ int charPosition = ctx.start.getCharPositionInLine();
+ ctx.info.expr = new IntIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+ catch(Exception e) {
+ helper.notifyErrorListeners("cannot parse the int value: \'" + ctx.getText() + "\'", ctx.getStart());
+ return;
+ }
+ }
+
+ @Override
+ public void exitConstStringIdExpression(ConstStringIdExpressionContext ctx) {
+ String val = "";
+ String text = ctx.getText();
+ if( (text.startsWith("\"") && text.endsWith("\"")) ||
+ (text.startsWith("\'") && text.endsWith("\'"))) {
+ if(text.length() > 2) {
+ val = text.substring(1, text.length()-1);
+ }
+ }
+ else {
+ helper.notifyErrorListeners("something wrong while parsing string ... strange", ctx.start);
+ return;
+ }
+
+ int linePosition = ctx.start.getLine();
+ int charPosition = ctx.start.getCharPositionInLine();
+ ctx.info.expr = new StringIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+
+ // --------------------------------------------------------------------
+
+ @Override
+ public void exitDataIdExpression(DataIdExpressionContext ctx) {
+ ctx.info.expr = ctx.dataIdentifier().dataInfo.expr;
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+ ctx.info.expr.setAllPositions(helper.getCurrentFileName(), line, col, line, col);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+
+ @Override
+ public void exitSimpleDataIdentifierExpression(SimpleDataIdentifierExpressionContext ctx) {
+ // This is either a function, or variable with namespace
+ // By default, it assigns to a data type
+ ctx.dataInfo.expr = new DataIdentifier(ctx.getText());
+ setFileLineColumn(ctx.dataInfo.expr, ctx);
+ }
+
+
+ private Expression incrementByOne(Expression expr, ParserRuleContext ctx) {
+ // For maintaining semantic consistency, we have decided to keep 1-based indexing
+ // If in future, PyDML becomes more popular than DML, this can be switched.
+ return expr;
+ }
+
+ @Override
+ public void exitIndexedExpression(IndexedExpressionContext ctx) {
+ ctx.dataInfo.expr = new IndexedIdentifier(ctx.name.getText(), false, false);
+ setFileLineColumn(ctx.dataInfo.expr, ctx);
+ try {
+ ArrayList< ArrayList<Expression> > exprList = new ArrayList< ArrayList<Expression> >();
+
+ ArrayList<Expression> rowIndices = new ArrayList<Expression>();
+ ArrayList<Expression> colIndices = new ArrayList<Expression>();
+
+ boolean isRowLower = (ctx.rowLower != null && !ctx.rowLower.isEmpty() && (ctx.rowLower.info.expr != null));
+ boolean isRowUpper = (ctx.rowUpper != null && !ctx.rowUpper.isEmpty() && (ctx.rowUpper.info.expr != null));
+ boolean isColLower = (ctx.colLower != null && !ctx.colLower.isEmpty() && (ctx.colLower.info.expr != null));
+ boolean isColUpper = (ctx.colUpper != null && !ctx.colUpper.isEmpty() && (ctx.colUpper.info.expr != null));
+
+ if(!isRowLower && !isRowUpper) {
+ // both not set
+ rowIndices.add(null); rowIndices.add(null);
+ }
+ else if(isRowLower && isRowUpper) {
+ // both set
+ rowIndices.add(incrementByOne(ctx.rowLower.info.expr, ctx));
+ rowIndices.add(ctx.rowUpper.info.expr);
+ }
+ else if(isRowLower && !isRowUpper) {
+ // only row set
+ rowIndices.add(incrementByOne(ctx.rowLower.info.expr, ctx));
+ }
+ else {
+ helper.notifyErrorListeners("incorrect index expression for row", ctx.start);
+ return;
+ }
+
+ if(!isColLower && !isColUpper) {
+ // both not set
+ colIndices.add(null); colIndices.add(null);
+ }
+ else if(isColLower && isColUpper) {
+ colIndices.add(incrementByOne(ctx.colLower.info.expr, ctx));
+ colIndices.add(ctx.colUpper.info.expr);
+ }
+ else if(isColLower && !isColUpper) {
+ colIndices.add(incrementByOne(ctx.colLower.info.expr, ctx));
+ }
+ else {
+ helper.notifyErrorListeners("incorrect index expression for column", ctx.start);
+ return;
+ }
+ exprList.add(rowIndices);
+ exprList.add(colIndices);
+ ((IndexedIdentifier) ctx.dataInfo.expr).setIndices(exprList);
+ }
+ catch(Exception e) {
+ helper.notifyErrorListeners("cannot set the indices", ctx.start);
+ return;
+ }
+ }
+
+ private ConstIdentifier getConstIdFromString(String varValue, Token start) {
+ // Both varName and varValue are correct
+ int linePosition = start.getLine();
+ int charPosition = start.getCharPositionInLine();
+ try {
+ long val = Long.parseLong(varValue);
+ return new IntIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ }
+ catch(Exception e) {
+ try {
+ double val = Double.parseDouble(varValue);
+ return new DoubleIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ }
+ catch(Exception e1) {
+ try {
+ if(varValue.compareTo("True") == 0 || varValue.compareTo("False") == 0) {
+ boolean val = false;
+ if(varValue.compareTo("True") == 0) {
+ val = true;
+ }
+ return new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ }
+ else {
+ String val = "";
+ String text = varValue;
+ if( (text.startsWith("\"") && text.endsWith("\"")) ||
+ (text.startsWith("\'") && text.endsWith("\'"))) {
+ if(text.length() > 2) {
+ val = text.substring(1, text.length()-1);
+ }
+ }
+ else {
+ // the commandline parameters can be passed without any quotes
+ val = text;
+ }
+ return new StringIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ }
+ }
+ catch(Exception e3) {
+ helper.notifyErrorListeners("unable to cast the commandline parameter into int/float/bool/str", start);
+ return null;
+ }
+ }
+ }
+ }
+
+ private void fillExpressionInfoCommandLineParameters(String varName, ExpressionInfo dataInfo, Token start) {
+
+ if(!varName.startsWith("$")) {
+ helper.notifyErrorListeners("commandline param doesnot start with $ ... strange", start);
+ return;
+ }
+
+ String varValue = null;
+ for(Map.Entry<String, String> arg : this.argVals.entrySet()) {
+ if(arg.getKey().trim().compareTo(varName) == 0) {
+ if(varValue != null) {
+ helper.notifyErrorListeners("multiple values passed for the parameter " + varName + " via commandline", start);
+ return;
+ }
+ else {
+ varValue = arg.getValue().trim();
+ }
+ }
+ }
+
+ if(varValue == null) {
+ // helper.notifyErrorListeners("the parameter " + varName + " either needs to be passed through commandline or initialized to default value", start);
+ return;
+ }
+
+ // Command line param cannot be empty string
+ // If you want to pass space, please quote it
+ if(varValue.trim().compareTo("") == 0)
+ return;
+
+ dataInfo.expr = getConstIdFromString(varValue, start);
+ }
+
+ @Override
+ public void exitCommandlineParamExpression(CommandlineParamExpressionContext ctx) {
+ handleCommandlineArgumentExpression(ctx);
+ }
+
+ @Override
+ public void exitCommandlinePositionExpression(CommandlinePositionExpressionContext ctx) {
+ handleCommandlineArgumentExpression(ctx);
+ }
+
+ /**
+ *
+ * @param ctx
+ */
+ private void handleCommandlineArgumentExpression(DataIdentifierContext ctx)
+ {
+ String varName = ctx.getText().trim();
+ fillExpressionInfoCommandLineParameters(varName, ctx.dataInfo, ctx.start);
+
+ if(ctx.dataInfo.expr == null) {
+ if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) {
+ String msg = "The parameter " + varName + " either needs to be passed "
+ + "through commandline or initialized to default value.";
+ if( AParserWrapper.IGNORE_UNSPECIFIED_ARGS ) {
+ ctx.dataInfo.expr = getConstIdFromString(" ", ctx.start);
+ helper.raiseWarning(msg, ctx.start);
+ }
+ else {
+ helper.notifyErrorListeners(msg, ctx.start);
+ }
+ }
+ }
+ }
+
+ // --------------------------------------------------------------------
+
+ @Override
+ public void exitImportStatement(ImportStatementContext ctx)
+ {
+ //prepare import filepath
+ String filePath = ctx.filePath.getText();
+ String namespace = DMLProgram.DEFAULT_NAMESPACE;
+ if(ctx.namespace != null && ctx.namespace.getText() != null && !ctx.namespace.getText().isEmpty()) {
+ namespace = ctx.namespace.getText();
+ }
+ if((filePath.startsWith("\"") && filePath.endsWith("\"")) ||
+ filePath.startsWith("'") && filePath.endsWith("'")) {
+ filePath = filePath.substring(1, filePath.length()-1);
+ }
+
+ //concatenate working directory to filepath
+ filePath = _workingDir + File.separator + filePath;
+
+ DMLProgram prog = null;
+ try {
+ prog = (new PyDMLParserWrapper()).doParse(filePath, null, argVals);
+ } catch (ParseException e) {
+ helper.notifyErrorListeners("Exception found during importing a program from file " + filePath, ctx.start);
+ return;
+ }
+ // Custom logic whether to proceed ahead or not. Better than the current exception handling mechanism
+ if(prog == null) {
+ helper.notifyErrorListeners("One or more errors found during importing a program from file " + filePath, ctx.start);
+ return;
+ }
+ else {
+ ctx.info.namespaces = new HashMap<String, DMLProgram>();
+ ctx.info.namespaces.put(namespace, prog);
+ ctx.info.stmt = new ImportStatement();
+ ((ImportStatement) ctx.info.stmt).setCompletePath(filePath);
+ ((ImportStatement) ctx.info.stmt).setFilePath(ctx.filePath.getText());
+ ((ImportStatement) ctx.info.stmt).setNamespace(namespace);
+ }
+ }
+
+ @Override
+ public void exitAssignmentStatement(AssignmentStatementContext ctx) {
+ if(ctx.targetList == null || ctx.targetList.size() != 1) {
+ helper.notifyErrorListeners("incorrect parsing for assignment", ctx.start);
+ return;
+ }
+ String targetListText = ctx.targetList.get(0).getText();
+ if(targetListText.startsWith("$")) {
+ helper.notifyErrorListeners("assignment of commandline parameters is not allowed. (Quickfix: try using someLocalVariable=ifdef(" + targetListText + ", default value))", ctx.start);
+ return;
+ }
+
+ DataIdentifier target = null;
+ if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
+ target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
+ Expression source = ctx.source.info.expr;
+
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+ try {
+ ctx.info.stmt = new AssignmentStatement(target, source, line, col, line, col);
+ setFileLineColumn(ctx.info.stmt, ctx);
+ } catch (LanguageException e) {
+ // TODO: extract more meaningful info from this exception.
+ helper.notifyErrorListeners("invalid assignment", ctx.targetList.get(0).start);
+ return;
+ }
+ }
+ else {
+ helper.notifyErrorListeners("incorrect lvalue ... strange", ctx.targetList.get(0).start);
+ return;
+ }
+
+ }
+
+
+ private void setAssignmentStatement(DataIdentifier target, Expression expression, StatementContext ctx) {
+ try {
+ ctx.info.stmt = new AssignmentStatement(target, expression, ctx.start.getLine(), ctx.start.getCharPositionInLine(), ctx.start.getLine(), ctx.start.getCharPositionInLine());
+ setFileLineColumn(ctx.info.stmt, ctx);
+ } catch (LanguageException e) {
+ // TODO: extract more meaningful info from this exception.
+ helper.notifyErrorListeners("invalid function call", ctx.start);
+ return;
+ }
+ }
+
+ private void setPrintStatement(FunctionCallAssignmentStatementContext ctx, String functionName) {
+ ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
+ if(paramExpression.size() != 1) {
+ helper.notifyErrorListeners(functionName + "() has only one parameter", ctx.start);
+ return;
+ }
+ Expression expr = paramExpression.get(0).getExpr();
+ if(expr == null) {
+ helper.notifyErrorListeners("cannot process " + functionName + "() function", ctx.start);
+ return;
+ }
+ try {
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+ ctx.info.stmt = new PrintStatement(functionName, expr, line, col, line, col);
+ } catch (LanguageException e) {
+ helper.notifyErrorListeners("cannot process " + functionName + "() function", ctx.start);
+ return;
+ }
+ }
+
+ private void setOutputStatement(FunctionCallAssignmentStatementContext ctx) {
+ ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
+ if(paramExpression.size() < 2){
+ helper.notifyErrorListeners("incorrect usage of load function (atleast 2 arguments required)", ctx.start);
+ return;
+ }
+ if(paramExpression.get(0).getExpr() instanceof DataIdentifier) {
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+ HashMap<String, Expression> varParams = new HashMap<String, Expression>();
+ varParams.put(DataExpression.IO_FILENAME, paramExpression.get(1).getExpr());
+ for(int i = 2; i < paramExpression.size(); i++) {
+ // DataExpression.FORMAT_TYPE, DataExpression.DELIM_DELIMITER, DataExpression.DELIM_HAS_HEADER_ROW, DataExpression.DELIM_SPARSE
+ varParams.put(paramExpression.get(i).getName(), paramExpression.get(i).getExpr());
+ }
+
+ DataExpression dataExpression = new DataExpression(DataOp.WRITE, varParams, fileName, line, col, line, col);
+ ctx.info.stmt = new OutputStatement((DataIdentifier) paramExpression.get(0).getExpr(), DataOp.WRITE, fileName, line, col, line, col);
+ setFileLineColumn(ctx.info.stmt, ctx);
+ ((OutputStatement)ctx.info.stmt).setExprParams(dataExpression);
+ return;
+ }
+
+ helper.notifyErrorListeners("incorrect usage of write function", ctx.start);
+ return;
+
+ }
+
+ private boolean inDefaultNamespace(String namespace) {
+ return namespace.compareTo(DMLProgram.DEFAULT_NAMESPACE) == 0;
+ }
+
+ // Returns 0, 1 or -1 (for error)
+ private int getAxis(ParameterizedExpressionContext ctx) {
+ if(ctx.paramName != null && ctx.paramName.getText() != null && !ctx.paramName.getText().isEmpty()) {
+ if(ctx.paramName.getText().compareTo("axis") != 0) {
+ return -1;
+ }
+ }
+
+ String val = ctx.paramVal.getText();
+ if(val != null && val.compareTo("0") == 0) {
+ return 0;
+ }
+ else if(val != null && val.compareTo("1") == 0) {
+ return 1;
+ }
+
+ return -1;
+ }
+
+ private String getPythonAggFunctionNames(String functionName, int axis) {
+ if(axis != 0 && axis != 1) {
+ return functionName;
+ }
+ // axis=0 maps to column-wise computation and axis=1 maps to row-wise computation
+
+ if(functionName.compareTo("sum") == 0) {
+ return axis == 0 ? "colSums" : "rowSums";
+ }
+ else if(functionName.compareTo("mean") == 0) {
+ return axis == 0 ? "colMeans" : "rowMeans";
+ }
+ else if(functionName.compareTo("avg") == 0) {
+ return axis == 0 ? "colMeans" : "rowMeans";
+ }
+ else if(functionName.compareTo("max") == 0) {
+ return axis == 0 ? "colMaxs" : "rowMaxs";
+ }
+ else if(functionName.compareTo("min") == 0) {
+ return axis == 0 ? "colMins" : "rowMins";
+ }
+ else if(functionName.compareTo("argmin") == 0) {
+ return axis == 0 ? "Not Supported" : "rowIndexMin";
+ }
+ else if(functionName.compareTo("argmax") == 0) {
+ return axis == 0 ? "Not Supported" : "rowIndexMax";
+ }
+ else if(functionName.compareTo("cumsum") == 0) {
+ return axis == 0 ? "cumsum" : "Not Supported";
+ }
+ else if(functionName.compareTo("transpose") == 0) {
+ return axis == 0 ? "Not Supported" : "Not Supported";
+ }
+ else if(functionName.compareTo("trace") == 0) {
+ return axis == 0 ? "Not Supported" : "Not Supported";
+ }
+ else {
+ return functionName;
+ }
+ }
+
+ private ConvertedDMLSyntax convertPythonBuiltinFunctionToDMLSyntax(String namespace, String functionName, ArrayList<ParameterExpression> paramExpression,
+ List<ParameterizedExpressionContext> paramCtx, Token fnName, String fileName, int line, int col) {
+ // ===========================================================================================
+ // Check function name, namespace, parameters (#params & possible values) and throw useful hints
+ if(inDefaultNamespace(namespace) && functionName.compareTo("len") == 0) {
+ if(paramExpression.size() != 1) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts 1 arguments", fnName);
+ return null;
+ }
+ functionName = "length";
+ }
+ else if(functionName.compareTo("sum") == 0 || functionName.compareTo("mean") == 0 || functionName.compareTo("avg") == 0 ||
+ functionName.compareTo("min") == 0 || functionName.compareTo("max") == 0 ||
+ functionName.compareTo("argmax") == 0 || functionName.compareTo("argmin") == 0 ||
+ functionName.compareTo("cumsum") == 0 || functionName.compareTo("transpose") == 0 || functionName.compareTo("trace") == 0) {
+ // 0 maps row-wise computation and 1 maps to column-wise computation
+
+ // can mean sum of all cells or row-wise or columnwise sum
+ if(inDefaultNamespace(namespace) && paramExpression.size() == 1) {
+ // sum(x) => sum(x)
+ // otherwise same function name
+ if(functionName.compareTo("avg") == 0) {
+ functionName = "mean";
+ }
+ else if(functionName.compareTo("transpose") == 0) {
+ functionName = "t";
+ }
+ else if(functionName.compareTo("argmax") == 0 || functionName.compareTo("argmin") == 0 || functionName.compareTo("cumsum") == 0) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for entire matrix is not supported", fnName);
+ return null;
+ }
+ }
+ else if(!inDefaultNamespace(namespace) && paramExpression.size() == 0) {
+ // x.sum() => sum(x)
+ paramExpression = new ArrayList<ParameterExpression>();
+ paramExpression.add(new ParameterExpression(null, new DataIdentifier(namespace)));
+ // otherwise same function name
+ if(functionName.compareTo("avg") == 0) {
+ functionName = "mean";
+ }
+ else if(functionName.compareTo("transpose") == 0) {
+ functionName = "t";
+ }
+ else if(functionName.compareTo("argmax") == 0 || functionName.compareTo("argmin") == 0 || functionName.compareTo("cumsum") == 0) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for entire matrix is not supported", fnName);
+ return null;
+ }
+ }
+ else if(inDefaultNamespace(namespace) && paramExpression.size() == 2) {
+ // sum(x, axis=1) => rowSums(x)
+ int axis = getAxis(paramCtx.get(1));
+ if(axis == -1 && (functionName.compareTo("min") == 0 || functionName.compareTo("max") == 0 )) {
+ // Do nothing
+ // min(2, 3)
+ }
+ else if(axis == -1) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for given arguments is not supported", fnName);
+ return null;
+ }
+ else {
+ ArrayList<ParameterExpression> temp = new ArrayList<ParameterExpression>();
+ temp.add(paramExpression.get(0));
+ paramExpression = temp;
+ functionName = getPythonAggFunctionNames(functionName, axis);
+ if(functionName.compareTo("Not Supported") == 0) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for given arguments is not supported", fnName);
+ return null;
+ }
+ }
+ }
+ else if(!inDefaultNamespace(namespace) && paramExpression.size() == 1) {
+ // x.sum(axis=1) => rowSums(x)
+ int axis = getAxis(paramCtx.get(0));
+ if(axis == -1) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for given arguments is not supported", fnName);
+ return null;
+ }
+ else {
+ paramExpression = new ArrayList<ParameterExpression>();
+ paramExpression.add(new ParameterExpression(null, new DataIdentifier(namespace)));
+ functionName = getPythonAggFunctionNames(functionName, axis);
+ if(functionName.compareTo("Not Supported") == 0) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' for given arguments is not supported", fnName);
+ return null;
+ }
+ }
+ }
+ else {
+ helper.notifyErrorListeners("Incorrect number of arguments for the builtin function \'" + functionName + "\'.", fnName);
+ return null;
+ }
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("concatenate") == 0) {
+ if(paramExpression.size() != 2) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts 2 arguments (Note: concatenate append columns of two matrices)", fnName);
+ return null;
+ }
+ functionName = "append";
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("minimum") == 0) {
+ if(paramExpression.size() != 2) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts 2 arguments", fnName);
+ return null;
+ }
+ functionName = "min";
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("maximum") == 0) {
+ if(paramExpression.size() != 2) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts 2 arguments", fnName);
+ return null;
+ }
+ functionName = "max";
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(!inDefaultNamespace(namespace) && functionName.compareTo("shape") == 0) {
+ if(paramExpression.size() != 1) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts only 1 argument (0 or 1)", fnName);
+ return null;
+ }
+
+ int axis = getAxis(paramCtx.get(0));
+ if(axis == -1) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts only 1 argument (0 or 1)", fnName);
+ return null;
+ }
+ paramExpression = new ArrayList<ParameterExpression>();
+ paramExpression.add(new ParameterExpression(null, new DataIdentifier(namespace)));
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ if(axis == 0) {
+ functionName = "nrow";
+ }
+ else if(axis == 1) {
+ functionName = "ncol";
+ }
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("random.normal") == 0) {
+ if(paramExpression.size() != 3) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (number of rows, number of columns, sparsity)", fnName);
+ return null;
+ }
+ paramExpression.get(0).setName("rows");
+ paramExpression.get(1).setName("cols");
+ paramExpression.get(2).setName("sparsity");
+ paramExpression.add(new org.apache.sysml.parser.ParameterExpression("pdf", new StringIdentifier("normal", fileName, line, col, line, col)));
+ functionName = "rand";
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("random.uniform") == 0) {
+ if(paramExpression.size() != 5) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 5 arguments (number of rows, number of columns, sparsity, min, max)", fnName);
+ return null;
+ }
+ paramExpression.get(0).setName("rows");
+ paramExpression.get(1).setName("cols");
+ paramExpression.get(2).setName("sparsity");
+ paramExpression.get(3).setName("min");
+ paramExpression.get(4).setName("max");
+ paramExpression.add(new org.apache.sysml.parser.ParameterExpression("pdf", new StringIdentifier("uniform", fileName, line, col, line, col)));
+ functionName = "rand";
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("full") == 0) {
+ if(paramExpression.size() != 3) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (constant float value, number of rows, number of columns)", fnName);
+ return null;
+ }
+ paramExpression.get(1).setName("rows");
+ paramExpression.get(2).setName("cols");
+ functionName = "matrix";
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("matrix") == 0) {
+ // This can either be string initializer or as.matrix function
+ if(paramExpression.size() != 1) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 1 argument (either str or float value)", fnName);
+ return null;
+ }
+
+ if(paramExpression.get(0).getExpr() instanceof StringIdentifier) {
+ String initializerString = ((StringIdentifier)paramExpression.get(0).getExpr()).getValue().trim();
+ if(!initializerString.startsWith("[") || !initializerString.endsWith("]")) {
+ helper.notifyErrorListeners("Incorrect initializer string for builtin function \'" + functionName + "\' (Eg: matrix(\"[1 2 3; 4 5 6]\"))", fnName);
+ return null;
+ }
+ int rows = StringUtils.countMatches(initializerString, ";") + 1;
+
+ // Make sure user doesnot have pretty string
+ initializerString = initializerString.replaceAll("; ", ";");
+ initializerString = initializerString.replaceAll(" ;", ";");
+ initializerString = initializerString.replaceAll("\\[ ", "\\[");
+ initializerString = initializerString.replaceAll(" \\]", "\\]");
+
+ // Each row has ncol-1 spaces
+ // #spaces = nrow * (ncol-1)
+ // ncol = (#spaces / nrow) + 1
+ int cols = (StringUtils.countMatches(initializerString, " ") / rows) + 1;
+
+ initializerString = initializerString.replaceAll(";", " ");
+ initializerString = initializerString.replaceAll("\\[", "");
+ initializerString = initializerString.replaceAll("\\]", "");
+ paramExpression = new ArrayList<ParameterExpression>();
+ paramExpression.add(new ParameterExpression(null, new StringIdentifier(initializerString, fileName, line, col, line, col)));
+ paramExpression.add(new ParameterExpression("rows", new IntIdentifier(rows, fileName, line, col, line, col)));
+ paramExpression.add(new ParameterExpression("cols", new IntIdentifier(cols, fileName, line, col, line, col)));
+ }
+ else {
+ functionName = "as.matrix";
+ }
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("scalar") == 0) {
+ if(paramExpression.size() != 1) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 1 argument", fnName);
+ return null;
+ }
+ functionName = "as.scalar";
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("float") == 0) {
+ if(paramExpression.size() != 1) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 1 argument", fnName);
+ return null;
+ }
+ functionName = "as.double";
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("int") == 0) {
+ if(paramExpression.size() != 1) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 1 argument", fnName);
+ return null;
+ }
+ functionName = "as.integer";
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("bool") == 0) {
+ if(paramExpression.size() != 1) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 1 argument", fnName);
+ return null;
+ }
+ functionName = "as.logical";
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(!inDefaultNamespace(namespace) && functionName.compareTo("reshape") == 0) {
+ if(paramExpression.size() != 2) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (number of rows, number of columns)", fnName);
+ return null;
+ }
+ paramExpression.get(0).setName("rows");
+ paramExpression.get(1).setName("cols");
+
+ ArrayList<ParameterExpression> temp = new ArrayList<ParameterExpression>();
+ temp.add(new ParameterExpression(null, new DataIdentifier(namespace)));
+ temp.add(paramExpression.get(0));
+ temp.add(paramExpression.get(1));
+ paramExpression = temp;
+
+ functionName = "matrix";
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("removeEmpty") == 0) {
+ if(paramExpression.size() != 2) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (matrix, axis=0 or 1)", fnName);
+ return null;
+ }
+ int axis = getAxis(paramCtx.get(1));
+ if(axis == -1) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (matrix, axis=0 or 1)", fnName);
+ return null;
+ }
+ StringIdentifier marginVal = null;
+ if(axis == 0) {
+ marginVal = new StringIdentifier("rows", fileName, line, col, line, col);
+ }
+ else {
+ marginVal = new StringIdentifier("cols", fileName, line, col, line, col);
+ }
+ paramExpression.get(0).setName("target");
+ paramExpression.get(1).setName("margin");
+ paramExpression.get(1).setExpr(marginVal);
+ functionName = "removeEmpty";
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("replace") == 0) {
+ if(paramExpression.size() != 3) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (matrix, scalar value that should be replaced (pattern), scalar value (replacement))", fnName);
+ return null;
+ }
+ paramExpression.get(0).setName("target");
+ paramExpression.get(1).setName("pattern");
+ paramExpression.get(2).setName("replacement");
+ functionName = "replace";
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("range") == 0) {
+ if(paramExpression.size() != 3) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (matrix, scalar value that should be replaced (pattern), scalar value (replacement))", fnName);
+ return null;
+ }
+ functionName = "seq";
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("norm.cdf") == 0) {
+ if(paramExpression.size() != 3) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (target, mean, sd)", fnName);
+ return null;
+ }
+ functionName = "cumulativeProbability";
+ paramExpression.get(0).setName("target");
+ paramExpression.get(1).setName("mean");
+ paramExpression.get(2).setName("sd");
+ paramExpression.add(new ParameterExpression("dist", new StringIdentifier("normal", fileName, line, col, line, col)));
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("expon.cdf") == 0) {
+ if(paramExpression.size() != 2) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (target, mean)", fnName);
+ return null;
+ }
+ functionName = "cumulativeProbability";
+ paramExpression.get(0).setName("target");
+ paramExpression.get(1).setName("mean");
+ paramExpression.add(new ParameterExpression("dist", new StringIdentifier("exp", fileName, line, col, line, col)));
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("chi.cdf") == 0) {
+ if(paramExpression.size() != 2) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (target, df)", fnName);
+ return null;
+ }
+ functionName = "cumulativeProbability";
+ paramExpression.get(0).setName("target");
+ paramExpression.get(1).setName("df");
+ paramExpression.add(new ParameterExpression("dist", new StringIdentifier("chisq", fileName, line, col, line, col)));
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("f.cdf") == 0) {
+ if(paramExpression.size() != 3) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (target, df1, df2)", fnName);
+ return null;
+ }
+ functionName = "cumulativeProbability";
+ paramExpression.get(0).setName("target");
+ paramExpression.get(1).setName("df1");
+ paramExpression.get(2).setName("df2");
+ paramExpression.add(new ParameterExpression("dist", new StringIdentifier("f", fileName, line, col, line, col)));
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("t.cdf") == 0) {
+ if(paramExpression.size() != 2) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (target, df)", fnName);
+ return null;
+ }
+ functionName = "cumulativeProbability";
+ paramExpression.get(0).setName("target");
+ paramExpression.get(1).setName("df");
+ paramExpression.add(new ParameterExpression("dist", new StringIdentifier("t", fileName, line, col, line, col)));
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("percentile") == 0) {
+ if(paramExpression.size() != 2 && paramExpression.size() != 3) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts either 2 or 3 arguments", fnName);
+ return null;
+ }
+ functionName = "quantile";
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("arcsin") == 0) {
+ functionName = "asin";
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("arccos") == 0) {
+ functionName = "acos";
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("arctan") == 0) {
+ functionName = "atan";
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("load") == 0) {
+ functionName = "read";
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("eigen") == 0) {
+ functionName = "eig";
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("power") == 0) {
+ if(paramExpression.size() != 2) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments", fnName);
+ return null;
+ }
+ }
+ else if(inDefaultNamespace(namespace) && functionName.compareTo("dot") == 0) {
+ if(paramExpression.size() != 2) {
+ helper.notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments", fnName);
+ return null;
+ }
+ }
+
+ ConvertedDMLSyntax retVal = new ConvertedDMLSyntax();
+ retVal.namespace = namespace;
+ retVal.functionName = functionName;
+ retVal.paramExpression = paramExpression;
+ return retVal;
+ }
+
+ class ConvertedDMLSyntax {
+ public String namespace;
+ public String functionName;
+ public ArrayList<ParameterExpression> paramExpression;
+ };
+
+ private Expression getOperatorExpression(String namespace, String functionName, ArrayList<ParameterExpression> paramExpression) {
+ String dmlOperator = null;
+
+ if(inDefaultNamespace(namespace) && functionName.compareTo("dot") == 0) {
+ if(paramExpression.size() == 2) {
+ dmlOperator = "%*%";
+ }
+ }
+
+ if(dmlOperator != null) {
+ Expression.BinaryOp bop = Expression.getBinaryOp(dmlOperator);
+ Expression expr = new BinaryExpression(bop);
+ ((BinaryExpression)expr).setLeft(paramExpression.get(0).getExpr());
+ ((BinaryExpression)expr).setRight(paramExpression.get(1).getExpr());
+ return expr;
+ }
+
+ return null;
+ }
+
+ @Override
+ public void exitFunctionCallAssignmentStatement(FunctionCallAssignmentStatementContext ctx) {
+ ArrayList<String> names = helper.getQualifiedNames(ctx.name.getText());
+ if(names == null) {
+ helper.notifyErrorListeners("incorrect function name (only namespace.functionName allowed. Hint: If you are trying to use builtin functions, you can skip the namespace)", ctx.name);
+ return;
+ }
+ String namespace = names.get(0);
+ String functionName = names.get(1);
+
+ if((functionName.compareTo("print") == 0 || functionName.compareTo("stop") == 0 ) && namespace.compareTo(DMLProgram.DEFAULT_NAMESPACE) == 0) {
+ setPrintStatement(ctx, functionName);
+ return;
+ }
+ else if(functionName.compareTo("save") == 0 && namespace.compareTo(DMLProgram.DEFAULT_NAMESPACE) == 0) {
+ setOutputStatement(ctx);
+ return;
+ }
+
+ boolean ignoreLValue = false;
+ if(ctx.targetList == null || ctx.targetList.size() == 0 || ctx.targetList.get(0).isEmpty()) {
+ helper.notifyErrorListeners("function call needs to have lvalue (Quickfix: change it to \'tmpVar = " + functionName + "(...)\')", ctx.name);
+ return;
+ }
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+
+ ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
+
+ ConvertedDMLSyntax convertedSyntax = convertPythonBuiltinFunctionToDMLSyntax(namespace, functionName, paramExpression, ctx.paramExprs, ctx.name, fileName, line, col);
+ if(convertedSyntax == null) {
+ return;
+ }
+ else {
+ namespace = convertedSyntax.namespace;
+ functionName = convertedSyntax.functionName;
+ paramExpression = convertedSyntax.paramExpression;
+ }
+
+ // ===========================================================================================
+ FunctionCallIdentifier functCall = new FunctionCallIdentifier(paramExpression);
+ try {
+ functCall.setFunctionName(functionName);
+ functCall.setFunctionNamespace(namespace);
+ } catch (ParseException e1) {
+ helper.notifyErrorListeners("unable to process function " + functionName, ctx.start);
+ return;
+ }
+
+ DataIdentifier target = null;
+ if(!ignoreLValue) {
+ if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
+ target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
+ }
+ else {
+ helper.notifyErrorListeners("incorrect lvalue ... strange", ctx.targetList.get(0).start);
+ //target = new DataIdentifier(); // so as not to avoid null pointer
+ return;
+ }
+ }
+
+ Expression operatorExpr = getOperatorExpression(namespace, functionName, paramExpression);
+ if(operatorExpr != null) {
+ setFileLineColumn(operatorExpr, ctx);
+ setAssignmentStatement(target, operatorExpr, ctx);
+ return;
+ }
+
+ //Note: In contrast to the dml parser, namespace and function names are separated by '.' not '::'.
+ //Hence, we have to include a whitelist of function names to handle builtins like 'as.scalar'.
+ String[] whitelist = new String[]{"as.matrix","as.scalar","as.double","as.integer","as.logical"};
+ boolean isWhitelisted = Arrays.asList(whitelist).contains(functionName);
+
+ if( !functionName.contains(".") || isWhitelisted
+ || functionName.startsWith(DMLProgram.DEFAULT_NAMESPACE) )
+ {
+ // In global namespace, so it can be a builtin function
+ if(!helper.validateBuiltinFunctions(ctx)) {
+ return; // it is a built-in function and validation failed, so donot proceed ahead.
+ }
+ // Double verification: verify passed function name is a (non-parameterized) built-in function.
+ try
+ {
+ // builtin functions
+ BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
+ if (bife != null) {
+ setAssignmentStatement(target, bife, ctx);
+ return;
+ }
+
+ // parameterized builtin functions
+ ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
+ if (pbife != null) {
+ setAssignmentStatement(target, pbife, ctx);
+ return;
+ }
+
+ // built-in data expressions, e.g. read
+ DataExpression dbife = DataExpression.getDataExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
+ if (dbife != null){
+ setAssignmentStatement(target, dbife, ctx);
+ return;
+ }
+ }
+ catch(Exception e) {
+ helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
+ return ;
+ }
+ }
+
+ setAssignmentStatement(target, functCall, ctx);
+ }
+
+
+ @Override
+ public void exitBuiltinFunctionExpression(BuiltinFunctionExpressionContext ctx) {
+ // Double verification: verify passed function name is a (non-parameterized) built-in function.
+ ArrayList<String> names = helper.getQualifiedNames(ctx.name.getText());
+ if(names == null) {
+ helper.notifyErrorListeners("incorrect function name (only namespace.functionName allowed. Hint: If you are trying to use builtin functions, you can skip the namespace)", ctx.name);
+ return;
+ }
+ String namespace = names.get(0);
+ String functionName = names.get(1);
+
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+ ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
+
+ ConvertedDMLSyntax convertedSyntax = convertPythonBuiltinFunctionToDMLSyntax(namespace, functionName, paramExpression, ctx.paramExprs, ctx.name, fileName, line, col);
+ if(convertedSyntax == null) {
+ return;
+ }
+ else {
+ namespace = convertedSyntax.namespace;
+ functionName = convertedSyntax.functionName;
+ paramExpression = convertedSyntax.paramExpression;
+ // System.out.println(ctx.name.getText() + ">>" + namespace + " " + functionName);
+ }
+
+ Expression operatorExpr = getOperatorExpression(namespace, functionName, paramExpression);
+ if(operatorExpr != null) {
+ ctx.info.expr = operatorExpr;
+ setFileLineColumn(operatorExpr, ctx);
+ return;
+ }
+
+ try {
+ BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, paramExpression, fileName, line, col, line, col);
+ if (bife != null){
+ // It is a builtin function
+ ctx.info.expr = bife;
+ return;
+ }
+
+ ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, paramExpression, fileName, line, col, line, col);
+ if (pbife != null){
+ // It is a parameterized builtin function
+ ctx.info.expr = pbife;
+ return;
+ }
+
+ // built-in read, rand ...
+ DataExpression dbife = DataExpression.getDataExpression(functionName, paramExpression, fileName, line, col, line, col);
+ if (dbife != null){
+ ctx.info.expr = dbife;
+ return;
+ }
+ } catch(Exception e) {
+ helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
+ return ;
+ }
+ helper.notifyErrorListeners("only builtin functions allowed as part of expression", ctx.start);
+ }
+
+ private void setMultiAssignmentStatement(ArrayList<DataIdentifier> target, Expression expression, StatementContext ctx) {
+ ctx.info.stmt = new MultiAssignmentStatement(target, expression);
+ ctx.info.stmt.setAllPositions(helper.getCurrentFileName(), ctx.start.getLine(), ctx.start.getCharPositionInLine(), ctx.start.getLine(), ctx.start.getCharPositionInLine());
+ setFileLineColumn(ctx.info.stmt, ctx);
+ }
+
+ @Override
+ public void exitFunctionCallMultiAssignmentStatement(
+ FunctionCallMultiAssignmentStatementContext ctx) {
+ ArrayList<String> names = helper.getQualifiedNames(ctx.name.getText());
+ if(names == null) {
+ helper.notifyErrorListeners("incorrect function name (only namespace.functionName allowed. Hint: If you are trying to use builtin functions, you can skip the namespace)", ctx.name);
+ return;
+ }
+ String namespace = names.get(0);
+ String functionName = names.get(1);
+
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+
+ ArrayList<ParameterExpression> paramExpression = helper.getParameterExpressionList(ctx.paramExprs);
+ ConvertedDMLSyntax convertedSyntax = convertPythonBuiltinFunctionToDMLSyntax(namespace, functionName, paramExpression, ctx.paramExprs, ctx.name, fileName, line, col);
+ if(convertedSyntax == null) {
+ return;
+ }
+ else {
+ namespace = convertedSyntax.namespace;
+ functionName = convertedSyntax.functionName;
+ paramExpression = convertedSyntax.paramExpression;
+ }
+
+ // No need to support dot() function since it will never return multi-assignment function
+
+ FunctionCallIdentifier functCall = new FunctionCallIdentifier(paramExpression);
+ try {
+ functCall.setFunctionName(functionName);
+ functCall.setFunctionNamespace(namespace);
+ } catch (ParseException e1) {
+ helper.notifyErrorListeners("unable to process function " + functionName, ctx.start);
+ return;
+ }
+
+ ArrayList<DataIdentifier> targetList = new ArrayList<DataIdentifier>();
+ for(DataIdentifierContext dataCtx : ctx.targetList) {
+ if(dataCtx.dataInfo.expr instanceof DataIdentifier) {
+ targetList.add((DataIdentifier) dataCtx.dataInfo.expr);
+ }
+ else {
+ helper.notifyErrorListeners("incorrect lvalue ... strange", dataCtx.start);
+ //target = new DataIdentifier(); // so as not to avoid null pointer
+ return;
+ }
+ }
+
+ if(!functionName.contains(".") || functionName.startsWith(DMLProgram.DEFAULT_NAMESPACE)) {
+ // In global namespace, so it can be a builtin function
+ // Double verification: verify passed function name is a (non-parameterized) built-in function.
+ try {
+ BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
+ if (bife != null){
+ // It is a builtin function
+ setMultiAssignmentStatement(targetList, bife, ctx);
+ return;
+ }
+
+ ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
+ if (pbife != null){
+ // It is a parameterized builtin function
+ setMultiAssignmentStatement(targetList, pbife, ctx);
+ return;
+ }
+
+ // built-in read, rand ...
+ DataExpression dbife = DataExpression.getDataExpression(functionName, functCall.getParamExprs(), fileName, line, col, line, col);
+ if (dbife != null){
+ setMultiAssignmentStatement(targetList, dbife, ctx);
+ return;
+ }
+ } catch(Exception e) {
+ helper.notifyErrorListeners("unable to process builtin function expression " + functionName + ":" + e.getMessage(), ctx.start);
+ return;
+ }
+ }
+
+ setMultiAssignmentStatement(targetList, functCall, ctx);
+ }
+
+ private StatementBlock getStatementBlock(Statement current) {
+ return PyDMLParserWrapper.getStatementBlock(current);
+ }
+
+ @Override
+ public void exitIfStatement(IfStatementContext ctx) {
+ IfStatement ifStmt = new IfStatement();
+ ConditionalPredicate predicate = new ConditionalPredicate(ctx.predicate.info.expr);
+ ifStmt.setConditionalPredicate(predicate);
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+ ifStmt.setAllPositions(fileName, line, col, line, col);
+
+ if(ctx.ifBody.size() > 0) {
+ for(StatementContext stmtCtx : ctx.ifBody) {
+ ifStmt.addStatementBlockIfBody(getStatementBlock(stmtCtx.info.stmt));
+ }
+ ifStmt.mergeStatementBlocksIfBody();
+ }
+
+ if(ctx.elseBody.size() > 0) {
+ for(StatementContext stmtCtx : ctx.elseBody) {
+ ifStmt.addStatementBlockElseBody(getStatementBlock(stmtCtx.info.stmt));
+ }
+ ifStmt.mergeStatementBlocksElseBody();
+ }
+
+ ctx.info.stmt = ifStmt;
+ setFileLineColumn(ctx.info.stmt, ctx);
+ }
+
+ @Override
+ public void exitWhileStatement(WhileStatementContext ctx) {
+ WhileStatement whileStmt = new WhileStatement();
+ ConditionalPredicate predicate = new ConditionalPredicate(ctx.predicate.info.expr);
+ whileStmt.setPredicate(predicate);
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+ whileStmt.setAllPositions(fileName, line, col, line, col);
+
+ if(ctx.body.size() > 0) {
+ for(StatementContext stmtCtx : ctx.body) {
+ whileStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
+ }
+ whileStmt.mergeStatementBlocks();
+ }
+
+ ctx.info.stmt = whileStmt;
+ setFileLineColumn(ctx.info.stmt, ctx);
+ }
+
+ @Override
+ public void exitForStatement(ForStatementContext ctx) {
+ ForStatement forStmt = new ForStatement();
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+
+ DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText());
+ HashMap<String, String> parForParamValues = null;
+ Expression incrementExpr = new IntIdentifier(1, fileName, line, col, line, col);
+ if(ctx.iterPred.info.increment != null) {
+ incrementExpr = ctx.iterPred.info.increment;
+ }
+ IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, fileName, line, col, line, col);
+ forStmt.setPredicate(predicate);
+
+ if(ctx.body.size() > 0) {
+ for(StatementContext stmtCtx : ctx.body) {
+ forStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
+ }
+ forStmt.mergeStatementBlocks();
+ }
+ ctx.info.stmt = forStmt;
+ setFileLineColumn(ctx.info.stmt, ctx);
+ }
+
+ @Override
+ public void exitParForStatement(ParForStatementContext ctx) {
+ ParForStatement parForStmt = new ParForStatement();
+ String fileName = helper.getCurrentFileName();
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+
+ DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText());
+ HashMap<String, String> parForParamValues = new HashMap<String, String>();
+ if(ctx.parForParams != null && ctx.parForParams.size() > 0) {
+ for(StrictParameterizedExpressionContext parForParamCtx : ctx.parForParams) {
+ parForParamValues.put(parForParamCtx.paramName.getText(), parForParamCtx.paramVal.getText());
+ }
+ }
+
+ Expression incrementExpr = new IntIdentifier(1, fileName, line, col, line, col);
+
+ if( ctx.iterPred.info.increment != null ) {
+ incrementExpr = ctx.iterPred.info.increment;
+ }
+ IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, fileName, line, col, line, col);
+ parForStmt.setPredicate(predicate);
+ if(ctx.body.size() > 0) {
+ for(StatementContext stmtCtx : ctx.body) {
+ parForStmt.addStatementBlock(getStatementBlock(stmtCtx.info.stmt));
+ }
+ parForStmt.mergeStatementBlocks();
+ }
+ ctx.info.stmt = parForStmt;
+ setFileLineColumn(ctx.info.stmt, ctx);
+ }
+
+
+
+
+ // ----------------------------------------------------------------------
+ @Override
+ public void exitMl_type(Ml_typeContext ctx) { }
+
+ private ArrayList<DataIdentifier> getFunctionParameters(List<TypedArgNoAssignContext> ctx) {
+ ArrayList<DataIdentifier> retVal = new ArrayList<DataIdentifier>();
+ for(TypedArgNoAssignContext paramCtx : ctx) {
+ DataIdentifier dataId = new DataIdentifier(paramCtx.paramName.getText());
+ String dataType = null;
+ String valueType = null;
+
+ if(paramCtx.paramType == null || paramCtx.paramType.dataType() == null
+ || paramCtx.paramType.dataType().getText() == null || paramCtx.paramType.dataType().getText().isEmpty()) {
+ dataType = "scalar";
+ }
+ else {
+ dataType = paramCtx.paramType.dataType().getText();
+ }
+
+ if(dataType.compareTo("matrix") == 0) {
+ // matrix
+ dataId.setDataType(DataType.MATRIX);
+ }
+ else if(dataType.compareTo("scalar") == 0) {
+ // scalar
+ dataId.setDataType(DataType.SCALAR);
+ }
+ else {
+ helper.notifyErrorListeners("invalid datatype " + dataType, paramCtx.start);
+ return null;
+ }
+
+ valueType = paramCtx.paramType.valueType().getText();
+ if(valueType.compareTo("int") == 0) {
+ dataId.setValueType(ValueType.INT);
+ }
+ else if(valueType.compareTo("str") == 0) {
+ dataId.setValueType(ValueType.STRING);
+ }
+ else if(valueType.compareTo("bool") == 0) {
+ dataId.setValueType(ValueType.BOOLEAN);
+ }
+ else if(valueType.compareTo("float") == 0) {
+ dataId.setValueType(ValueType.DOUBLE);
+ }
+ else {
+ helper.notifyErrorListeners("invalid valuetype " + valueType, paramCtx.start);
+ return null;
+ }
+ retVal.add(dataId);
+ }
+ return retVal;
+ }
+
+ @Override
+ public void exitInternalFunctionDefExpression(InternalFunctionDefExpressionContext ctx) {
+ FunctionStatement functionStmt = new FunctionStatement();
+
+ ArrayList<DataIdentifier> functionInputs = getFunctionParameters(ctx.inputParams);
+ functionStmt.setInputParams(functionInputs);
+
+ // set function outputs
+ ArrayList<DataIdentifier> functionOutputs = getFunctionParameters(ctx.outputParams);
+ functionStmt.setOutputParams(functionOutputs);
+
+ // set function name
+ functionStmt.setName(ctx.name.getText());
+
+
+ if(ctx.body.size() > 0) {
+ // handle function body
+ // Create arraylist of one statement block
+ ArrayList<StatementBlock> body = new ArrayList<StatementBlock>();
+ for(StatementContext stmtCtx : ctx.body) {
+ body.add(getStatementBlock(stmtCtx.info.stmt));
+ }
+ functionStmt.setBody(body);
+ functionStmt.mergeStatementBlocks();
+ }
+ else {
+ helper.notifyErrorListeners("functions with no statements are not allowed", ctx.start);
+ return;
+ }
+
+ ctx.info.stmt = functionStmt;
+ setFileLineColumn(ctx.info.stmt, ctx);
+ ctx.info.functionName = ctx.name.getText();
+ }
+
+ @Override
+ public void exitExternalFunctionDefExpression(ExternalFunctionDefExpressionContext ctx) {
+ ExternalFunctionStatement functionStmt = new ExternalFunctionStatement();
+
+ ArrayList<DataIdentifier> functionInputs = getFunctionParameters(ctx.inputParams);
+ functionStmt.setInputParams(functionInputs);
+
+ // set function outputs
+ ArrayList<DataIdentifier> functionOutputs = getFunctionParameters(ctx.outputParams);
+ functionStmt.setOutputParams(functionOutputs);
+
+ // set function name
+ functionStmt.setName(ctx.name.getText());
+
+ // set other parameters
+ HashMap<String, String> otherParams = new HashMap<String,String>();
+ boolean atleastOneClassName = false;
+ for(StrictParameterizedKeyValueStringContext otherParamCtx : ctx.otherParams){
+ String paramName = otherParamCtx.paramName.getText();
+ String val = "";
+ String text = otherParamCtx.paramVal.getText();
+ // First unquote the string
+ if( (text.startsWith("\"") && text.endsWith("\"")) ||
+ (text.startsWith("\'") && text.endsWith("\'"))) {
+ if(text.length() > 2) {
+ val = text.substring(1, text.length()-1);
+ }
+ // Empty value allowed
+ }
+ else {
+ helper.notifyErrorListeners("the value of user parameter for external function should be of type str", ctx.start);
+ return;
+ }
+ otherParams.put(paramName, val);
+ if(paramName.compareTo("classname") == 0) {
+ atleastOneClassName = true;
+ }
+ }
+ functionStmt.setOtherParams(otherParams);
+ if(!atleastOneClassName) {
+ helper.notifyErrorListeners("the parameter \'className\' needs to be passed for defExternal", ctx.start);
+ return;
+ }
+
+ ctx.info.stmt = functionStmt;
+ setFileLineColumn(ctx.info.stmt, ctx);
+ ctx.info.functionName = ctx.name.getText();
+ }
+
+
+ @Override
+ public void exitPathStatement(PathStatementContext ctx) {
+ PathStatement stmt = new PathStatement(ctx.pathValue.getText());
+ String filePath = ctx.pathValue.getText();
+ if((filePath.startsWith("\"") && filePath.endsWith("\"")) ||
+ filePath.startsWith("'") && filePath.endsWith("'")) {
+ filePath = filePath.substring(1, filePath.length()-1);
+ }
+
+ _workingDir = filePath;
+ ctx.info.stmt = stmt;
+ }
+
+ @Override
+ public void exitIfdefAssignmentStatement(IfdefAssignmentStatementContext ctx) {
+ if(!ctx.commandLineParam.getText().startsWith("$")) {
+ helper.notifyErrorListeners("the first argument of ifdef function should be a commandline argument parameter (which starts with $)", ctx.commandLineParam.start);
+ return;
+ }
+
+ if(ctx.targetList == null || ctx.targetList.size() != 1) {
+ helper.notifyErrorListeners("incorrect parsing for ifdef function", ctx.start);
+ return;
+ }
+ String targetListText = ctx.targetList.get(0).getText();
+ if(targetListText.startsWith("$")) {
+ helper.notifyErrorListeners("lhs of ifdef function cannot be a commandline parameters. Use local variable instead", ctx.start);
+ return;
+ }
+
+ DataIdentifier target = null;
+ if(ctx.targetList.get(0).dataInfo.expr instanceof DataIdentifier) {
+ target = (DataIdentifier) ctx.targetList.get(0).dataInfo.expr;
+ Expression source = null;
+ if(ctx.commandLineParam.dataInfo.expr != null) {
+ // Since commandline parameter is set
+ // The check of following is done in fillExpressionInfoCommandLineParameters:
+ // Command line param cannot be empty string
+ // If you want to pass space, please quote it
+ source = ctx.commandLineParam.dataInfo.expr;
+ }
+ else {
+ source = ctx.source.info.expr;
+ }
+
+ int line = ctx.start.getLine();
+ int col = ctx.start.getCharPositionInLine();
+ try {
+ ctx.info.stmt = new AssignmentStatement(target, source, line, col, line, col);
+ setFileLineColumn(ctx.info.stmt, ctx);
+ } catch (LanguageException e) {
+ helper.notifyErrorListeners("invalid assignment for ifdef function", ctx.targetList.get(0).start);
+ return;
+ }
+
+ }
+ else {
+ helper.notifyErrorListeners("incorrect lvalue in ifdef function... strange", ctx.targetList.get(0).start);
+ return;
+ }
+
+ }
+
+ // ----------------------------------------------------------------------
+ @Override
+ public void exitParameterizedExpression(ParameterizedExpressionContext ctx) { }
+
+
+ @Override
+ public void exitStrictParameterizedExpression(StrictParameterizedExpressionContext ctx) { }
+
+ @Override
+ public void exitTypedArgNoAssign(TypedArgNoAssignContext ctx) { }
+ @Override
+ public void enterIfdefAssignmentStatement(IfdefAssignmentStatementContext ctx) { }
+ @Override
+ public void enterMatrixDataTypeCheck(MatrixDataTypeCheckContext ctx) { }
+ @Override
+ public void exitMatrixDataTypeCheck(MatrixDataTypeCheckContext ctx) {
+ if( ctx.ID().getText().compareTo("matrix") == 0
+ || ctx.ID().getText().compareTo("scalar") == 0
+ ) {
+ // Do nothing
+ }
+ else if(ctx.ID().getText().compareTo("Matrix") == 0)
+ helper.notifyErrorListeners("incorrect datatype (Hint: use matrix instead of Matrix)", ctx.start);
+ else if(ctx.ID().getText().compareTo("Scalar") == 0)
+ helper.notifyErrorListeners("incorrect datatype (Hint: use scalar instead of Scalar)", ctx.start);
+ else if( ctx.ID().getText().compareTo("int") == 0
+ || ctx.ID().getText().compareTo("str") == 0
+ || ctx.ID().getText().compareTo("bool") == 0
+ || ctx.ID().getText().compareTo("float") == 0
+ ) {
+ helper.notifyErrorListeners("expected datatype but found a valuetype (Hint: use matrix or scalar instead of " + ctx.ID().getText() + ")", ctx.start);
+ }
+ else {
+ helper.notifyErrorListeners("incorrect datatype (expected matrix or scalar)", ctx.start);
+ }
+ }
+
+ @Override
+ public void enterBuiltinFunctionExpression(BuiltinFunctionExpressionContext ctx) {}
+ @Override
+ public void enterStrictParameterizedKeyValueString(StrictParameterizedKeyValueStringContext ctx) { }
+ @Override
+ public void exitStrictParameterizedKeyValueString(StrictParameterizedKeyValueStringContext ctx) {}
+ @Override
+ public void enterIterablePredicateColonExpression(IterablePredicateColonExpressionContext ctx) {}
+ @Override
+ public void enterIterablePredicateSeqExpression(IterablePredicateSeqExpressionContext ctx) { }
+
+ @Override
+ public void exitIterablePredicateColonExpression(IterablePredicateColonExpressionContext ctx) {
+ ctx.info.from = ctx.from.info.expr;
+ ctx.info.to = ctx.to.info.expr;
+ ctx.info.increment = null;
+ }
+
+ @Override
+ public void exitIterablePredicateSeqExpression(IterablePredicateSeqExpressionContext ctx) {
+ if(ctx.ID().getText().compareTo("range") != 0) {
+ helper.notifyErrorListeners("incorrect function:\'" + ctx.ID().getText() + "\'. expected \'range\'", ctx.start);
+ return;
+ }
+ ctx.info.from = ctx.from.info.expr;
+ ctx.info.to = ctx.to.info.expr;
+ ctx.info.increment = ctx.increment.info.expr;
+ }
+
+ @Override
+ public void enterConstFalseExpression(ConstFalseExpressionContext ctx) { }
+ @Override
+ public void enterConstTrueExpression(ConstTrueExpressionContext ctx) { }
+
+ @Override
+ public void exitConstFalseExpression(ConstFalseExpressionContext ctx) {
+ boolean val = false;
+ int linePosition = ctx.start.getLine();
+ int charPosition = ctx.start.getCharPositionInLine();
+ ctx.info.expr = new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+
+
+ @Override
+ public void exitConstTrueExpression(ConstTrueExpressionContext ctx) {
+ boolean val = true;
+ int linePosition = ctx.start.getLine();
+ int charPosition = ctx.start.getCharPositionInLine();
+ ctx.info.expr = new BooleanIdentifier(val, helper.getCurrentFileName(), linePosition, charPosition, linePosition, charPosition);
+ setFileLineColumn(ctx.info.expr, ctx);
+ }
+
+ @Override
+ public void exitIgnoreNewLine(IgnoreNewLineContext ctx) {
+ // Introduce empty StatementInfo
+ // This is later ignored by PyDMLParserWrapper
+ try {
+ ctx.info.stmt = new AssignmentStatement(null, null, 0, 0, 0, 0);
+ ctx.info.stmt.setEmptyNewLineStatement(true);
+ } catch (LanguageException e) {
+ e.printStackTrace();
+ }
+
+ }
+ @Override
+ public void enterValueDataTypeCheck(ValueDataTypeCheckContext ctx) { }
+ @Override
+ public void exitValueDataTypeCheck(ValueDataTypeCheckContext ctx) {
+ if( ctx.ID().getText().compareTo("int") == 0
+ || ctx.ID().getText().compareTo("str") == 0
+ || ctx.ID().getText().compareTo("bool") == 0
+ || ctx.ID().getText().compareTo("float") == 0
+ ) {
+ // Do nothing
+ }
+ else if(ctx.ID().getText().compareTo("integer") == 0)
+ helper.notifyErrorListeners("incorrect valuetype (Hint: use int instead of integer)", ctx.start);
+ else if(ctx.ID().getText().compareTo("double") == 0)
+ helper.notifyErrorListeners("incorrect valuetype (Hint: use float instead of double)", ctx.start);
+ else if(ctx.ID().getText().compareTo("boolean") == 0)
+ helper.notifyErrorListeners("incorrect valuetype (Hint: use bool instead of boolean)", ctx.start);
+ else if(ctx.ID().getText().compareTo("string") == 0)
+ helper.notifyErrorListeners("incorrect valuetype (Hint: use str instead of string)", ctx.start);
+ else {
+ helper.notifyErrorListeners("incorrect valuetype (expected int, str, bool or float)", ctx.start);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidatorHelper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidatorHelper.java b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidatorHelper.java
new file mode 100644
index 0000000..5b14613
--- /dev/null
+++ b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidatorHelper.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.parser.pydml;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.antlr.v4.runtime.Token;
+import org.apache.sysml.parser.DMLProgram;
+import org.apache.sysml.parser.pydml.PydmlParser.FunctionCallAssignmentStatementContext;
+import org.apache.sysml.parser.pydml.PydmlParser.ParameterizedExpressionContext;
+import org.apache.sysml.parser.pydml.PydmlSyntacticErrorListener.CustomDmlErrorListener;
+
+
+public class PydmlSyntacticValidatorHelper
+{
+ private CustomDmlErrorListener _errorListener = null;
+ public PydmlSyntacticValidatorHelper(CustomDmlErrorListener errorListener) {
+ this._errorListener = errorListener;
+ }
+ public void notifyErrorListeners(String message, int line, int charPositionInLine) {
+ this._errorListener.validationError(line, charPositionInLine, message);
+ }
+
+ public void notifyErrorListeners(String message, Token op) {
+ this._errorListener.validationError(op.getLine(), op.getCharPositionInLine(), message);
+ }
+
+ public void raiseWarning(String message, Token op) {
+ this._errorListener.validationWarning(op.getLine(), op.getCharPositionInLine(), message);
+ }
+
+ public String getCurrentFileName() {
+ return _errorListener.peekFileName();
+ }
+
+ // Returns list of two elements <namespace, function names>, else null
+ public ArrayList<String> getQualifiedNames(String fullyQualifiedFunctionName) {
+ String [] fnNames = fullyQualifiedFunctionName.split("\\."); // instead of ::
+ String functionName = "";
+ String namespace = "";
+ if(fnNames.length == 1) {
+ namespace = DMLProgram.DEFAULT_NAMESPACE;
+ functionName = fnNames[0].trim();
+ }
+ else if(fnNames.length == 2) {
+ namespace = fnNames[0].trim();
+ functionName = fnNames[1].trim();
+ }
+ else
+ return null;
+
+ ArrayList<String> retVal = new ArrayList<String>();
+ retVal.add(namespace);
+ retVal.add(functionName);
+ return retVal;
+ }
+
+ public boolean validateBuiltinFunctions(FunctionCallAssignmentStatementContext ctx) {
+ String functionName = ctx.name.getText().replaceAll(" ", "").trim();
+ if(functionName.compareTo("write") == 0 || functionName.compareTo(DMLProgram.DEFAULT_NAMESPACE + ".write") == 0) {
+ return validateBuiltinWriteFunction(ctx);
+ }
+ return true;
+ }
+
+ private boolean validateBuiltinWriteFunction(FunctionCallAssignmentStatementContext ctx) {
+
+ return true;
+ }
+
+ public ArrayList<org.apache.sysml.parser.ParameterExpression> getParameterExpressionList(List<ParameterizedExpressionContext> paramExprs) {
+ ArrayList<org.apache.sysml.parser.ParameterExpression> retVal = new ArrayList<org.apache.sysml.parser.ParameterExpression>();
+ for(ParameterizedExpressionContext ctx : paramExprs) {
+ String paramName = null;
+ if(ctx.paramName != null && ctx.paramName.getText() != null && !ctx.paramName.getText().isEmpty()) {
+ paramName = ctx.paramName.getText();
+ }
+ org.apache.sysml.parser.ParameterExpression myArg = new org.apache.sysml.parser.ParameterExpression(paramName, ctx.paramVal.info.expr);
+ retVal.add(myArg);
+ }
+ return retVal;
+ }
+}