You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by du...@apache.org on 2016/01/21 02:06:25 UTC
[8/8] incubator-systemml git commit: [SYSTEMML-148] Refactored
.parser.{antlr4, python} to .parser.{dml, pydml}.
[SYSTEMML-148] Refactored .parser.{antlr4,python} to .parser.{dml,pydml}.
Closes #44.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/c04fc99f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/c04fc99f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/c04fc99f
Branch: refs/heads/master
Commit: c04fc99fc1e6d6cc5f5546dd86f51513da63714c
Parents: 8ba0fdc
Author: Nakul Jindal <na...@gmail.com>
Authored: Wed Jan 20 17:02:12 2016 -0800
Committer: Mike Dusenberry <mw...@us.ibm.com>
Committed: Wed Jan 20 17:02:12 2016 -0800
----------------------------------------------------------------------
.gitignore | 16 +-
pom.xml | 16 +-
.../org/apache/sysml/parser/AParserWrapper.java | 4 +-
.../sysml/parser/antlr4/DMLParserWrapper.java | 385 ----
.../java/org/apache/sysml/parser/antlr4/Dml.g4 | 201 --
.../antlr4/DmlSyntacticErrorListener.java | 115 -
.../parser/antlr4/DmlSyntacticValidator.java | 1533 -------------
.../antlr4/DmlSyntacticValidatorHelper.java | 108 -
.../sysml/parser/antlr4/ExpressionInfo.java | 31 -
.../sysml/parser/antlr4/StatementInfo.java | 36 -
.../sysml/parser/dml/DMLParserWrapper.java | 385 ++++
.../java/org/apache/sysml/parser/dml/Dml.g4 | 201 ++
.../parser/dml/DmlSyntacticErrorListener.java | 115 +
.../sysml/parser/dml/DmlSyntacticValidator.java | 1533 +++++++++++++
.../parser/dml/DmlSyntacticValidatorHelper.java | 108 +
.../apache/sysml/parser/dml/ExpressionInfo.java | 31 +
.../apache/sysml/parser/dml/StatementInfo.java | 36 +
.../sysml/parser/pydml/ExpressionInfo.java | 32 +
.../sysml/parser/pydml/PyDMLParserWrapper.java | 283 +++
.../java/org/apache/sysml/parser/pydml/Pydml.g4 | 384 ++++
.../pydml/PydmlSyntacticErrorListener.java | 114 +
.../parser/pydml/PydmlSyntacticValidator.java | 2040 +++++++++++++++++
.../pydml/PydmlSyntacticValidatorHelper.java | 101 +
.../sysml/parser/pydml/StatementInfo.java | 32 +
.../sysml/parser/python/ExpressionInfo.java | 32 -
.../sysml/parser/python/PyDMLParserWrapper.java | 283 ---
.../org/apache/sysml/parser/python/Pydml.g4 | 384 ----
.../python/PydmlSyntacticErrorListener.java | 114 -
.../parser/python/PydmlSyntacticValidator.java | 2042 ------------------
.../python/PydmlSyntacticValidatorHelper.java | 101 -
.../sysml/parser/python/StatementInfo.java | 32 -
31 files changed, 5413 insertions(+), 5415 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 7df2eea..2d11e27 100644
--- a/.gitignore
+++ b/.gitignore
@@ -41,14 +41,14 @@ src/main/java/Dml.tokens
src/main/java/DmlLexer.tokens
src/main/java/Pydml.tokens
src/main/java/PydmlLexer.tokens
-src/main/java/org/apache/sysml/parser/antlr4/DmlBaseListener.java
-src/main/java/org/apache/sysml/parser/antlr4/DmlLexer.java
-src/main/java/org/apache/sysml/parser/antlr4/DmlListener.java
-src/main/java/org/apache/sysml/parser/antlr4/DmlParser.java
-src/main/java/org/apache/sysml/parser/python/PydmlBaseListener.java
-src/main/java/org/apache/sysml/parser/python/PydmlLexer.java
-src/main/java/org/apache/sysml/parser/python/PydmlListener.java
-src/main/java/org/apache/sysml/parser/python/PydmlParser.java
+src/main/java/org/apache/sysml/parser/dml/DmlBaseListener.java
+src/main/java/org/apache/sysml/parser/dml/DmlLexer.java
+src/main/java/org/apache/sysml/parser/dml/DmlListener.java
+src/main/java/org/apache/sysml/parser/dml/DmlParser.java
+src/main/java/org/apache/sysml/parser/pydml/PydmlBaseListener.java
+src/main/java/org/apache/sysml/parser/pydml/PydmlLexer.java
+src/main/java/org/apache/sysml/parser/pydml/PydmlListener.java
+src/main/java/org/apache/sysml/parser/pydml/PydmlParser.java
src/test/scripts/**/in
src/test/scripts/**/out
src/test/scripts/**/expected
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index bba680c..dd4a586 100644
--- a/pom.xml
+++ b/pom.xml
@@ -474,14 +474,14 @@
<exclude>src/main/java/DmlLexer.tokens</exclude>
<exclude>src/main/java/Pydml.tokens</exclude>
<exclude>src/main/java/PydmlLexer.tokens</exclude>
- <exclude>src/main/java/org/apache/sysml/parser/antlr4/DmlBaseListener.java</exclude>
- <exclude>src/main/java/org/apache/sysml/parser/antlr4/DmlLexer.java</exclude>
- <exclude>src/main/java/org/apache/sysml/parser/antlr4/DmlListener.java</exclude>
- <exclude>src/main/java/org/apache/sysml/parser/antlr4/DmlParser.java</exclude>
- <exclude>src/main/java/org/apache/sysml/parser/python/PydmlBaseListener.java</exclude>
- <exclude>src/main/java/org/apache/sysml/parser/python/PydmlLexer.java</exclude>
- <exclude>src/main/java/org/apache/sysml/parser/python/PydmlListener.java</exclude>
- <exclude>src/main/java/org/apache/sysml/parser/python/PydmlParser.java</exclude>
+ <exclude>src/main/java/org/apache/sysml/parser/dml/DmlBaseListener.java</exclude>
+ <exclude>src/main/java/org/apache/sysml/parser/dml/DmlLexer.java</exclude>
+ <exclude>src/main/java/org/apache/sysml/parser/dml/DmlListener.java</exclude>
+ <exclude>src/main/java/org/apache/sysml/parser/dml/DmlParser.java</exclude>
+ <exclude>src/main/java/org/apache/sysml/parser/pydml/PydmlBaseListener.java</exclude>
+ <exclude>src/main/java/org/apache/sysml/parser/pydml/PydmlLexer.java</exclude>
+ <exclude>src/main/java/org/apache/sysml/parser/pydml/PydmlListener.java</exclude>
+ <exclude>src/main/java/org/apache/sysml/parser/pydml/PydmlParser.java</exclude>
<!-- Test Validation files -->
<exclude>src/test/scripts/functions/external/kMeans/kMeansWrapperOutput</exclude>
<exclude>src/test/scripts/functions/external/kMeans2/kMeansWrapperOutput1</exclude>
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/AParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/AParserWrapper.java b/src/main/java/org/apache/sysml/parser/AParserWrapper.java
index fd600f2..6e67954 100644
--- a/src/main/java/org/apache/sysml/parser/AParserWrapper.java
+++ b/src/main/java/org/apache/sysml/parser/AParserWrapper.java
@@ -21,8 +21,8 @@ package org.apache.sysml.parser;
import java.util.HashMap;
-import org.apache.sysml.parser.antlr4.DMLParserWrapper;
-import org.apache.sysml.parser.python.PyDMLParserWrapper;
+import org.apache.sysml.parser.dml.DMLParserWrapper;
+import org.apache.sysml.parser.pydml.PyDMLParserWrapper;
/**
* Base class for all dml parsers in order to make the various compilation chains
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/antlr4/DMLParserWrapper.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/DMLParserWrapper.java b/src/main/java/org/apache/sysml/parser/antlr4/DMLParserWrapper.java
deleted file mode 100644
index 25b7b5e..0000000
--- a/src/main/java/org/apache/sysml/parser/antlr4/DMLParserWrapper.java
+++ /dev/null
@@ -1,385 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.parser.antlr4;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.antlr.v4.runtime.BailErrorStrategy;
-import org.antlr.v4.runtime.DefaultErrorStrategy;
-import org.antlr.v4.runtime.atn.PredictionMode;
-import org.antlr.v4.runtime.misc.ParseCancellationException;
-import org.antlr.v4.runtime.tree.ParseTreeWalker;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.sysml.api.DMLScript;
-import org.apache.sysml.conf.ConfigurationManager;
-import org.apache.sysml.parser.AParserWrapper;
-import org.apache.sysml.parser.DMLProgram;
-import org.apache.sysml.parser.ForStatement;
-import org.apache.sysml.parser.ForStatementBlock;
-import org.apache.sysml.parser.FunctionStatementBlock;
-import org.apache.sysml.parser.IfStatement;
-import org.apache.sysml.parser.IfStatementBlock;
-import org.apache.sysml.parser.ImportStatement;
-import org.apache.sysml.parser.LanguageException;
-import org.apache.sysml.parser.ParForStatement;
-import org.apache.sysml.parser.ParForStatementBlock;
-import org.apache.sysml.parser.ParseException;
-import org.apache.sysml.parser.Statement;
-import org.apache.sysml.parser.StatementBlock;
-import org.apache.sysml.parser.WhileStatement;
-import org.apache.sysml.parser.WhileStatementBlock;
-import org.apache.sysml.parser.antlr4.DmlParser.DmlprogramContext;
-import org.apache.sysml.parser.antlr4.DmlParser.FunctionStatementContext;
-import org.apache.sysml.parser.antlr4.DmlParser.StatementContext;
-import org.apache.sysml.parser.antlr4.DmlSyntacticErrorListener.CustomDmlErrorListener;
-import org.apache.sysml.runtime.util.LocalFileUtils;
-
-/**
- * This is the main entry point for the Antlr4 parser.
- * Dml.g4 is the grammar file which enforces syntactic structure of DML program.
- * DmlSyntaticValidator on other hand captures little bit of semantic as well as does the job of translation of Antlr AST to DMLProgram.
- * At a high-level, DmlSyntaticValidator implements call-back methods that are called by walker.walk(validator, tree)
- * The callback methods are of two type: enterSomeASTNode() and exitSomeASTNode()
- * It is important to note that almost every node in AST has either ExpressionInfo or StatementInfo object associated with it.
- * The key design decision is that while "exiting" the node (i.e. callback to exitSomeASTNode), we use information in given AST node and construct an object of type Statement or Expression and put it in StatementInfo or ExpressionInfo respectively.
- * This way it avoids any bugs due to lookahead and one only has to "think as an AST node", thereby making any changes to parse code much simpler :)
- *
- * Note: to add additional builtin function, one only needs to modify DmlSyntaticValidator (which is java file and provides full Eclipse tooling support) not g4.
- *
- * To separate logic of semantic validation, DmlSyntaticValidatorHelper contains functions that do semantic validation. Currently, there is no semantic validation as most of it is delegated to subsequent validation phase.
- *
- * Whenever there is a parse error, it goes through DmlSyntacticErrorListener. This allows us to pipe the error messages to any future pipeline as well as control the format in an elegant manner.
- * There are three types of messages passed:
- * - Syntactic errors: When passed DML script doesnot conform to syntatic structure enforced by Dml.g4
- * - Validation errors: Errors due to translation of AST to DMLProgram
- * - Validation warnings: Messages to inform users that there might be potential bug in their program
- *
- * As of this moment, Antlr4ParserWrapper is stateful and cannot be multithreaded. This is not big deal because each users calls SystemML in different process.
- * If in future we intend to make it multi-threaded, look at cleanUpState method and resolve the dependency accordingly.
- *
- */
-public class DMLParserWrapper extends AParserWrapper
-{
- private static final Log LOG = LogFactory.getLog(DMLScript.class.getName());
-
- /**
- * Custom wrapper to convert statement into statement blocks. Called by doParse and in DmlSyntacticValidator for for, parfor, while, ...
- * @param current a statement
- * @return corresponding statement block
- */
- public static StatementBlock getStatementBlock(Statement current) {
- StatementBlock blk = null;
- if(current instanceof ParForStatement) {
- blk = new ParForStatementBlock();
- blk.addStatement(current);
- }
- else if(current instanceof ForStatement) {
- blk = new ForStatementBlock();
- blk.addStatement(current);
- }
- else if(current instanceof IfStatement) {
- blk = new IfStatementBlock();
- blk.addStatement(current);
- }
- else if(current instanceof WhileStatement) {
- blk = new WhileStatementBlock();
- blk.addStatement(current);
- }
- else {
- // This includes ImportStatement
- blk = new StatementBlock();
- blk.addStatement(current);
- }
- return blk;
- }
-
- /**
- * Parses the passed file with command line parameters. You can either pass both (local file) or just dmlScript (hdfs) or just file name (import command)
- * @param fileName either full path or null --> only used for better error handling
- * @param dmlScript required
- * @param argVals
- * @return
- * @throws ParseException
- */
- @Override
- public DMLProgram parse(String fileName, String dmlScript, HashMap<String,String> argVals)
- throws ParseException
- {
- DMLProgram prog = null;
-
- if(dmlScript == null || dmlScript.trim().isEmpty()) {
- throw new ParseException("Incorrect usage of parse. Please pass dmlScript not just filename");
- }
-
- // Set the pipeline required for ANTLR parsing
- DMLParserWrapper parser = new DMLParserWrapper();
- prog = parser.doParse(fileName, dmlScript, argVals);
-
- if(prog == null) {
- throw new ParseException("One or more errors found during parsing (could not construct AST for file: " + fileName + "). Cannot proceed ahead.");
- }
- return prog;
- }
-
- /**
- * This function is supposed to be called directly only from DmlSyntacticValidator when it encounters 'import'
- * @param fileName
- * @return null if atleast one error
- */
- public DMLProgram doParse(String fileName, String dmlScript, HashMap<String,String> argVals) throws ParseException {
- DMLProgram dmlPgm = null;
-
- org.antlr.v4.runtime.ANTLRInputStream in;
- try {
- if(dmlScript == null) {
- dmlScript = readDMLScript(fileName);
- }
-
- InputStream stream = new ByteArrayInputStream(dmlScript.getBytes());
- in = new org.antlr.v4.runtime.ANTLRInputStream(stream);
-// else {
-// if(!(new File(fileName)).exists()) {
-// throw new ParseException("ERROR: Cannot open file:" + fileName);
-// }
-// in = new org.antlr.v4.runtime.ANTLRInputStream(new java.io.FileInputStream(fileName));
-// }
- } catch (FileNotFoundException e) {
- throw new ParseException("ERROR: Cannot find file:" + fileName, e);
- } catch (IOException e) {
- throw new ParseException("ERROR: Cannot open file:" + fileName, e);
- } catch (LanguageException e) {
- throw new ParseException("ERROR: " + e.getMessage(), e);
- }
-
- DmlprogramContext ast = null;
- CustomDmlErrorListener errorListener = new CustomDmlErrorListener();
-
- try {
- DmlLexer lexer = new DmlLexer(in);
- org.antlr.v4.runtime.CommonTokenStream tokens = new org.antlr.v4.runtime.CommonTokenStream(lexer);
- DmlParser antlr4Parser = new DmlParser(tokens);
-
- boolean tryOptimizedParsing = false; // For now no optimization, since it is not able to parse integer value.
-
- if(tryOptimizedParsing) {
- // Try faster and simpler SLL
- antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
- antlr4Parser.removeErrorListeners();
- antlr4Parser.setErrorHandler(new BailErrorStrategy());
- try{
- ast = antlr4Parser.dmlprogram();
- // If successful, no need to try out full LL(*) ... SLL was enough
- }
- catch(ParseCancellationException ex) {
- // Error occurred, so now try full LL(*) for better error messages
- tokens.reset();
- antlr4Parser.reset();
- if(fileName != null) {
- errorListener.pushCurrentFileName(fileName);
- // DmlSyntacticErrorListener.currentFileName.push(fileName);
- }
- else {
- errorListener.pushCurrentFileName("MAIN_SCRIPT");
- // DmlSyntacticErrorListener.currentFileName.push("MAIN_SCRIPT");
- }
- // Set our custom error listener
- antlr4Parser.addErrorListener(errorListener);
- antlr4Parser.setErrorHandler(new DefaultErrorStrategy());
- antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL);
- ast = antlr4Parser.dmlprogram();
- }
- }
- else {
- // Set our custom error listener
- antlr4Parser.removeErrorListeners();
- antlr4Parser.addErrorListener(errorListener);
- errorListener.pushCurrentFileName(fileName);
-
- // Now do the parsing
- ast = antlr4Parser.dmlprogram();
- }
- }
- catch(Exception e) {
- throw new ParseException("ERROR: Cannot parse the program:" + fileName, e);
- }
-
-
- try {
- // Now convert the parse tree into DMLProgram
- // Do syntactic validation while converting
- org.antlr.v4.runtime.tree.ParseTree tree = ast;
- // And also do syntactic validation
- org.antlr.v4.runtime.tree.ParseTreeWalker walker = new ParseTreeWalker();
- DmlSyntacticValidatorHelper helper = new DmlSyntacticValidatorHelper(errorListener);
- DmlSyntacticValidator validator = new DmlSyntacticValidator(helper, errorListener.peekFileName(), argVals);
- walker.walk(validator, tree);
- errorListener.popFileName();
- if(errorListener.isAtleastOneError()) {
- return null;
- }
- dmlPgm = createDMLProgram(ast);
- }
- catch(Exception e) {
- throw new ParseException("ERROR: Cannot translate the parse tree into DMLProgram" + e.getMessage(), e);
- }
-
- return dmlPgm;
- }
-
- private DMLProgram createDMLProgram(DmlprogramContext ast) {
-
- DMLProgram dmlPgm = new DMLProgram();
-
- // First add all the functions
- for(FunctionStatementContext fn : ast.functionBlocks) {
- FunctionStatementBlock functionStmtBlk = new FunctionStatementBlock();
- functionStmtBlk.addStatement(fn.info.stmt);
- try {
- // TODO: currently the logic of nested namespace is not clear.
- String namespace = DMLProgram.DEFAULT_NAMESPACE;
- dmlPgm.addFunctionStatementBlock(namespace, fn.info.functionName, functionStmtBlk);
- } catch (LanguageException e) {
- LOG.error("line: " + fn.start.getLine() + ":" + fn.start.getCharPositionInLine() + " cannot process the function " + fn.info.functionName);
- return null;
- }
- }
-
- // Then add all the statements
- for(StatementContext stmtCtx : ast.blocks) {
- org.apache.sysml.parser.Statement current = stmtCtx.info.stmt;
- if(current == null) {
- LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the statement");
- return null;
- }
-
- if(current instanceof ImportStatement) {
- // Handle import statements separately
- if(stmtCtx.info.namespaces != null) {
- // Add the DMLProgram entries into current program
- for(Map.Entry<String, DMLProgram> entry : stmtCtx.info.namespaces.entrySet()) {
- dmlPgm.getNamespaces().put(entry.getKey(), entry.getValue());
-
-// // Don't add DMLProgram into the current program, just add function statements
- // dmlPgm.getNamespaces().put(entry.getKey(), entry.getValue());
- // Add function statements to current dml program
-// DMLProgram importedPgm = entry.getValue();
-// try {
-// for(FunctionStatementBlock importedFnBlk : importedPgm.getFunctionStatementBlocks()) {
-// if(importedFnBlk.getStatements() != null && importedFnBlk.getStatements().size() == 1) {
-// String functionName = ((FunctionStatement)importedFnBlk.getStatement(0)).getName();
-// System.out.println("Adding function => " + entry.getKey() + "::" + functionName);
-// TODO:33
-// dmlPgm.addFunctionStatementBlock(entry.getKey(), functionName, importedFnBlk);
-// }
-// else {
-// LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " incorrect number of functions in the imported function block .... strange");
-// return null;
-// }
-// }
-// if(importedPgm.getStatementBlocks() != null && importedPgm.getStatementBlocks().size() > 0) {
-// LOG.warn("Only the functions can be imported from the namespace " + entry.getKey());
-// }
-// } catch (LanguageException e) {
-// LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot import functions from the file in the import statement: " + e.getMessage());
-// return null;
-// }
- }
- }
- else {
- LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the import statement");
- return null;
- }
- }
-
- // Now wrap statement into individual statement block
- // merge statement will take care of merging these blocks
- dmlPgm.addStatementBlock(getStatementBlock(current));
- }
-
- dmlPgm.mergeStatementBlocks();
- return dmlPgm;
- }
-
- public static String readDMLScript( String script )
- throws IOException, LanguageException
- {
- String dmlScriptStr = null;
-
- //read DML script from file
- if(script == null)
- throw new LanguageException("DML script path was not specified!");
-
- StringBuilder sb = new StringBuilder();
- BufferedReader in = null;
- try
- {
- //read from hdfs or gpfs file system
- if( script.startsWith("hdfs:")
- || script.startsWith("gpfs:") )
- {
- if( !LocalFileUtils.validateExternalFilename(script, true) )
- throw new LanguageException("Invalid (non-trustworthy) hdfs filename.");
- FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf());
- Path scriptPath = new Path(script);
- in = new BufferedReader(new InputStreamReader(fs.open(scriptPath)));
- }
- // from local file system
- else
- {
- if( !LocalFileUtils.validateExternalFilename(script, false) )
- throw new LanguageException("Invalid (non-trustworthy) local filename.");
- in = new BufferedReader(new FileReader(script));
- }
-
- //core script reading
- String tmp = null;
- while ((tmp = in.readLine()) != null)
- {
- sb.append( tmp );
- sb.append( "\n" );
- }
- }
- catch (IOException ex)
- {
- LOG.error("Failed to read the script from the file system", ex);
- throw ex;
- }
- finally
- {
- if( in != null )
- in.close();
- }
-
- dmlScriptStr = sb.toString();
-
- return dmlScriptStr;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/antlr4/Dml.g4
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/Dml.g4 b/src/main/java/org/apache/sysml/parser/antlr4/Dml.g4
deleted file mode 100644
index 400a412..0000000
--- a/src/main/java/org/apache/sysml/parser/antlr4/Dml.g4
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-grammar Dml;
-
-@header
-{
- // Commenting the package name and explicitly passing it in build.xml to maintain compatibility with maven plugin
- // package org.apache.sysml.antlr4;
-}
-
-// DML Program is a list of expression
-// For now, we only allow global function definitions (not nested or inside a while block)
-dmlprogram: (blocks+=statement | functionBlocks+=functionStatement)* EOF;
-
-statement returns [ StatementInfo info ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new StatementInfo();
-} :
- // ------------------------------------------
- // ImportStatement
- 'source' '(' filePath = STRING ')' 'as' namespace=ID ';'* # ImportStatement
- | 'setwd' '(' pathValue = STRING ')' ';'* # PathStatement
- // ------------------------------------------
- // Treat function call as AssignmentStatement or MultiAssignmentStatement
- // For backward compatibility and also since the behavior of foo() * A + foo() ... where foo returns A
- // Convert FunctionCallIdentifier(paramExprs, ..) -> source
- | // TODO: Throw an informative error if user doesnot provide the optional assignment
- ( targetList+=dataIdentifier ('='|'<-') )? name=ID '(' (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? ')' ';'* # FunctionCallAssignmentStatement
- | '[' targetList+=dataIdentifier (',' targetList+=dataIdentifier)* ']' ('='|'<-') name=ID '(' (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? ')' ';'* # FunctionCallMultiAssignmentStatement
- // {notifyErrorListeners("Too many parentheses");}
- // ------------------------------------------
- // AssignmentStatement
- | targetList+=dataIdentifier op=('<-'|'=') 'ifdef' '(' commandLineParam=dataIdentifier ',' source=expression ')' ';'* # IfdefAssignmentStatement
- | targetList+=dataIdentifier op=('<-'|'=') source=expression ';'* # AssignmentStatement
- // ------------------------------------------
- // We don't support block statement
- // | '{' body+=expression ';'* ( body+=expression ';'* )* '}' # BlockStatement
- // ------------------------------------------
- // IfStatement
- | 'if' '(' predicate=expression ')' (ifBody+=statement ';'* | '{' (ifBody+=statement ';'*)* '}') ('else' (elseBody+=statement ';'* | '{' (elseBody+=statement ';'*)* '}'))? # IfStatement
- // ------------------------------------------
- // ForStatement & ParForStatement
- | 'for' '(' iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* ')' (body+=statement ';'* | '{' (body+=statement ';'* )* '}') # ForStatement
- // Convert strictParameterizedExpression to HashMap<String, String> for parForParams
- | 'parfor' '(' iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* ')' (body+=statement ';'* | '{' (body+=statement ';'*)* '}') # ParForStatement
- | 'while' '(' predicate=expression ')' (body+=statement ';'* | '{' (body+=statement ';'*)* '}') # WhileStatement
- // ------------------------------------------
-;
-
-iterablePredicate returns [ ExpressionInfo info ]
- @init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new ExpressionInfo();
- } :
- from=expression ':' to=expression #IterablePredicateColonExpression
- | ID '(' from=expression ',' to=expression ',' increment=expression ')' #IterablePredicateSeqExpression
- ;
-
-functionStatement returns [ StatementInfo info ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new StatementInfo();
-} :
- // ------------------------------------------
- // FunctionStatement & ExternalFunctionStatement
- // small change: only allow typed arguments here ... instead of data identifier
- name=ID ('<-'|'=') 'function' '(' ( inputParams+=typedArgNoAssign (',' inputParams+=typedArgNoAssign)* )? ')' ( 'return' '(' ( outputParams+=typedArgNoAssign (',' outputParams+=typedArgNoAssign)* )? ')' )? '{' (body+=statement ';'*)* '}' # InternalFunctionDefExpression
- | name=ID ('<-'|'=') 'externalFunction' '(' ( inputParams+=typedArgNoAssign (',' inputParams+=typedArgNoAssign)* )? ')' ( 'return' '(' ( outputParams+=typedArgNoAssign (',' outputParams+=typedArgNoAssign)* )? ')' )? 'implemented' 'in' '(' ( otherParams+=strictParameterizedKeyValueString (',' otherParams+=strictParameterizedKeyValueString)* )? ')' ';'* # ExternalFunctionDefExpression
- // ------------------------------------------
-;
-
-
-// Other data identifiers are typedArgNoAssign, parameterizedExpression and strictParameterizedExpression
-dataIdentifier returns [ ExpressionInfo dataInfo ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $dataInfo = new ExpressionInfo();
- // $dataInfo.expr = new org.apache.sysml.parser.DataIdentifier();
-} :
- // ------------------------------------------
- // IndexedIdentifier
- name=ID '[' (rowLower=expression (':' rowUpper=expression)?)? ',' (colLower=expression (':' colUpper=expression)?)? ']' # IndexedExpression
- // ------------------------------------------
- | ID # SimpleDataIdentifierExpression
- | COMMANDLINE_NAMED_ID # CommandlineParamExpression
- | COMMANDLINE_POSITION_ID # CommandlinePositionExpression
-;
-expression returns [ ExpressionInfo info ]
-@init {
- // This actions occurs regardless of how many alternatives in this rule
- $info = new ExpressionInfo();
- // $info.expr = new org.apache.sysml.parser.BinaryExpression(org.apache.sysml.parser.Expression.BinaryOp.INVALID);
-} :
- // ------------------------------------------
- // BinaryExpression
- // power
- <assoc=right> left=expression op='^' right=expression # PowerExpression
- // unary plus and minus
- | op=('-'|'+') left=expression # UnaryExpression
- // sequence - since we are only using this into for
- //| left=expression op=':' right=expression # SequenceExpression
- // matrix multiply
- | left=expression op='%*%' right=expression # MatrixMulExpression
- // modulus and integer division
- | left=expression op=('%/%' | '%%' ) right=expression # ModIntDivExpression
- // arithmetic multiply and divide
- | left=expression op=('*'|'/') right=expression # MultDivExpression
- // arithmetic addition and subtraction
- | left=expression op=('+'|'-') right=expression # AddSubExpression
- // ------------------------------------------
- // RelationalExpression
- | left=expression op=('>'|'>='|'<'|'<='|'=='|'!=') right=expression # RelationalExpression
- // ------------------------------------------
- // BooleanExpression
- // boolean not
- | op='!' left=expression # BooleanNotExpression
- // boolean and
- | left=expression op=('&'|'&&') right=expression # BooleanAndExpression
- // boolean or
- | left=expression op=('|'|'||') right=expression # BooleanOrExpression
-
- // ---------------------------------
- // only applicable for builtin function expressions
- | name=ID '(' (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? ')' ';'* # BuiltinFunctionExpression
-
- // 4. Atomic
- | '(' left=expression ')' # AtomicExpression
-
- // Should you allow indexed expression here ?
- // | '[' targetList+=expression (',' targetList+=expression)* ']' # MultiIdExpression
-
- // | BOOLEAN # ConstBooleanIdExpression
- | 'TRUE' # ConstTrueExpression
- | 'FALSE' # ConstFalseExpression
- | INT # ConstIntIdExpression
- | DOUBLE # ConstDoubleIdExpression
- | STRING # ConstStringIdExpression
- | dataIdentifier # DataIdExpression
- // Special
- // | 'NULL' | 'NA' | 'Inf' | 'NaN'
-;
-
-typedArgNoAssign : paramType=ml_type paramName=ID;
-parameterizedExpression : (paramName=ID '=')? paramVal=expression;
-strictParameterizedExpression : paramName=ID '=' paramVal=expression ;
-strictParameterizedKeyValueString : paramName=ID '=' paramVal=STRING ;
-ID : (ALPHABET (ALPHABET|DIGIT|'_')* '::')? ALPHABET (ALPHABET|DIGIT|'_')*
- // Special ID cases:
- // | 'matrix' // --> This is a special case which causes lot of headache
- | 'as.scalar' | 'as.matrix' | 'as.double' | 'as.integer' | 'as.logical' | 'index.return' | 'lower.tail'
-;
-// Unfortunately, we have datatype name clashing with builtin function name: matrix :(
-// Therefore, ugly work around for checking datatype
-ml_type : valueType | dataType '[' valueType ']';
-// Note to reduce number of keywords, these are case-sensitive,
-// To allow case-insenstive, 'int' becomes: ('i' | 'I') ('n' | 'N') ('t' | 'T')
-valueType: 'int' | 'integer' | 'string' | 'boolean' | 'double'
- | 'Int' | 'Integer' | 'String' | 'Boolean' | 'Double';
-dataType:
- // 'scalar' # ScalarDataTypeDummyCheck
- // |
- ID # MatrixDataTypeCheck //{ if($ID.text.compareTo("matrix") != 0) { notifyErrorListeners("incorrect datatype"); } }
- //| 'matrix' //---> See ID, this causes lot of headache
- ;
-INT : DIGIT+ [Ll]?;
-// BOOLEAN : 'TRUE' | 'FALSE';
-DOUBLE: DIGIT+ '.' DIGIT* EXP? [Ll]?
-| DIGIT+ EXP? [Ll]?
-| '.' DIGIT+ EXP? [Ll]?
-;
-DIGIT: '0'..'9';
-ALPHABET : [a-zA-Z] ;
-fragment EXP : ('E' | 'e') ('+' | '-')? INT ;
-COMMANDLINE_NAMED_ID: '$' ALPHABET (ALPHABET|DIGIT|'_')*;
-COMMANDLINE_POSITION_ID: '$' DIGIT+;
-
-// supports single and double quoted string with escape characters
-STRING: '"' ( ESC | ~[\\"] )*? '"' | '\'' ( ESC | ~[\\'] )*? '\'';
-fragment ESC : '\\' [abtnfrv"'\\] ;
-// Comments, whitespaces and new line
-LINE_COMMENT : '#' .*? '\r'? '\n' -> skip ;
-MULTILINE_BLOCK_COMMENT : '/*' .*? '*/' -> skip ;
-WHITESPACE : (' ' | '\t' | '\r' | '\n')+ -> skip ;
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticErrorListener.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticErrorListener.java b/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticErrorListener.java
deleted file mode 100644
index a7ed89d..0000000
--- a/src/main/java/org/apache/sysml/parser/antlr4/DmlSyntacticErrorListener.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysml.parser.antlr4;
-
-import org.antlr.v4.runtime.BaseErrorListener;
-import org.antlr.v4.runtime.RecognitionException;
-import org.antlr.v4.runtime.Recognizer;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.sysml.api.DMLScript;
-
-import java.util.Stack;
-
-public class DmlSyntacticErrorListener {
-
-
- private static final Log LOG = LogFactory.getLog(DMLScript.class.getName());
-
- public static class CustomDmlErrorListener extends BaseErrorListener {
-
- private boolean atleastOneError = false;
- private Stack<String> currentFileName = new Stack<String>();
-
- public void pushCurrentFileName(String currentFilePath) {
- currentFileName.push(currentFilePath);
- }
-
- public String peekFileName() {
- return currentFileName.peek();
- }
-
- public String popFileName() {
- return currentFileName.pop();
- }
-
- public void validationError(int line, int charPositionInLine, String msg) {
- try {
- setAtleastOneError(true);
- // Print error messages with file name
- if(currentFileName == null || currentFileName.empty()) {
- LOG.error("line "+line+":"+charPositionInLine+" "+msg);
- }
- else {
- String fileName = currentFileName.peek();
- LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg);
- }
- }
- catch(Exception e1) {
- LOG.error("ERROR: while customizing error message:" + e1);
- }
- }
-
- public void validationWarning(int line, int charPositionInLine, String msg) {
- try {
- //atleastOneError = true; ---> not an error, just warning
- // Print error messages with file name
- if(currentFileName == null || currentFileName.empty())
- LOG.warn("line "+line+":"+charPositionInLine+" "+msg);
- else {
- String fileName = currentFileName.peek();
- LOG.warn(fileName + " line "+line+":"+charPositionInLine+" "+msg);
- }
- }
- catch(Exception e1) {
- LOG.warn("ERROR: while customizing error message:" + e1);
- }
- }
-
- @Override
- public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol,
- int line, int charPositionInLine,
- String msg, RecognitionException e)
- {
- try {
- setAtleastOneError(true);
- // Print error messages with file name
- if(currentFileName == null || currentFileName.empty())
- LOG.error("line "+line+":"+charPositionInLine+" "+msg);
- else {
- String fileName = currentFileName.peek();
- LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg);
- }
- }
- catch(Exception e1) {
- LOG.error("ERROR: while customizing error message:" + e1);
- }
- }
-
- public boolean isAtleastOneError() {
- return atleastOneError;
- }
-
- public void setAtleastOneError(boolean atleastOneError) {
- this.atleastOneError = atleastOneError;
- }
- }
-}