You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by wa...@apache.org on 2017/01/12 21:49:48 UTC
asterixdb git commit: Add some functionalities to Grammar Extension
plugin
Repository: asterixdb
Updated Branches:
refs/heads/master 599ef8fb2 -> 456ff9ded
Add some functionalities to Grammar Extension plugin
- Can add a Java method at the end of the class definition
- Extend @merge replace "oldphrase" with "newphrase"
- Can add a construct at the end of the file
- Can deal with class modifier
Change-Id: Icafc01ebe591b81b3c3f69ea9da123dbfad19bb0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1431
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
BAD: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <ba...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/456ff9de
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/456ff9de
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/456ff9de
Branch: refs/heads/master
Commit: 456ff9ded50f2d6ed40ec9ba537fcd847810b584
Parents: 599ef8f
Author: Taewoo Kim <wa...@yahoo.com>
Authored: Wed Jan 11 14:16:16 2017 -0800
Committer: Taewoo Kim <wa...@yahoo.com>
Committed: Thu Jan 12 13:49:16 2017 -0800
----------------------------------------------------------------------
.../extension/grammar/GrammarExtensionMojo.java | 187 +++++++++++++++++--
.../src/test/resources/lang/extension.jj | 55 +++++-
.../basic-test/basic-test-plugin-config.xml | 1 +
3 files changed, 219 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/456ff9de/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
index 5b44e23..4c750e2 100644
--- a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
+++ b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
@@ -68,16 +68,23 @@ public class GrammarExtensionMojo extends AbstractMojo {
private static final String KWUNIMPORT = "unimport";
private static final String KWPACKAGE = "package";
private static final String NEWPRODUCTION = "@new";
+ // Adds a construct at the end of the file.
+ private static final String NEW_AT_THE_END_PRODUCTION = "@new_at_the_end";
+ // Adds a method at the end of the class definition.
+ private static final String NEW_AT_THE_END_CLASS_DEFINITION = "@new_at_the_class_def";
private static final String MERGEPRODUCTION = "@merge";
private static final String OVERRIDEPRODUCTION = "@override";
private static final String BEFORE = "before:";
private static final String AFTER = "after:";
private static final String REPLACE = "replace";
private static final String WITH = "with";
+ private static final String OPTION_TRUE = "true";
+ private static final String OPTION_FALSE = "false";
private static final List<String> KEYWORDS = Arrays
.asList(new String[] { KWCLASS, KWIMPORT, KWPACKAGE, PARSER_BEGIN, PARSER_END });
- private static final List<String> EXTENSIONKEYWORDS = Arrays
- .asList(new String[] { KWIMPORT, KWUNIMPORT, NEWPRODUCTION, OVERRIDEPRODUCTION, MERGEPRODUCTION });
+ private static final List<String> EXTENSIONKEYWORDS =
+ Arrays.asList(new String[] { KWIMPORT, KWUNIMPORT, NEWPRODUCTION, NEW_AT_THE_END_PRODUCTION,
+ NEW_AT_THE_END_CLASS_DEFINITION, OVERRIDEPRODUCTION, MERGEPRODUCTION });
private static final String REGEX_WS_DOT_SEMICOLON = "\\s|[.]|[;]";
private static final String REGEX_WS_PAREN = "\\s|[(]|[)]";
private static final String OPTIONS = "options";
@@ -87,14 +94,20 @@ public class GrammarExtensionMojo extends AbstractMojo {
private Map<String, String[]> mergeElements = new HashMap<>();
private List<Pair<String, String>> baseFinals = new ArrayList<>();
private List<Pair<String, String>> extensionFinals = new ArrayList<>();
+ private List<Pair<String, String>> extensionFinalsAtTheEnd = new ArrayList<>();
+ private List<Pair<String, String>> extensionMethodsAtTheClassDef = new ArrayList<>();
private List<List<String>> imports = new ArrayList<>();
private String baseClassName;
private String baseClassDef;
private String optionsBlock;
private boolean read = false;
private boolean shouldReplace = false;
- private String oldWord = null;
- private String newWord = null;
+
+ // Used in @merge replace. If set to true, applies each block's changes and stores them.
+ private boolean shouldApplyEachBlockChange = true;
+
+ private String oldPhrase = null;
+ private String newPhrase = null;
@Parameter(property = "grammarix.base")
private String base;
@@ -115,12 +128,16 @@ public class GrammarExtensionMojo extends AbstractMojo {
private String parserClassName;
private String lastIdentifier;
+ @Parameter(property = "grammarix.parserClassModifier")
+ private String parserClassModifier;
+
@Override
public void execute() throws MojoExecutionException {
base = new File(base).getAbsolutePath();
getLog().info("Base dir: " + base);
getLog().info("Grammar-base: " + gbase);
getLog().info("Grammar-extension: " + gextension);
+ getLog().info("Output: " + output);
processBase();
processExtension();
generateOutput();
@@ -166,7 +183,29 @@ public class GrammarExtensionMojo extends AbstractMojo {
writer.newLine();
// Class definition
- writer.write(baseClassDef.replaceAll(baseClassName, parserClassName));
+ String classDef = baseClassDef.replaceAll(baseClassName, parserClassName);
+ if (parserClassModifier != null && parserClassModifier.length() > 0) {
+ // Adds a class modifier if any.
+ classDef = classDef.replaceFirst(KWCLASS, parserClassModifier + " " + KWCLASS);
+ }
+
+ // Process extensions at the end of the class definition
+ if (!extensionMethodsAtTheClassDef.isEmpty()) {
+ int index = classDef.lastIndexOf(CLOSE_BRACE);
+ if (index != -1) {
+ String classDefExtension = "";
+ for (Pair<String, String> element : extensionMethodsAtTheClassDef) {
+ classDefExtension += toOutput(element.first);
+ classDefExtension += "\n";
+ classDefExtension += element.second;
+ classDefExtension += "\n";
+ }
+ classDef =
+ classDef.substring(0, index) + "\n" + classDefExtension + "\n" + classDef.substring(index);
+ }
+ }
+
+ writer.write(classDef);
writer.newLine();
// Parser End
@@ -213,6 +252,13 @@ public class GrammarExtensionMojo extends AbstractMojo {
writer.newLine();
}
+ for (Pair<String, String> element : extensionFinalsAtTheEnd) {
+ writer.write(toOutput(element.first));
+ writer.newLine();
+ writer.write(element.second);
+ writer.newLine();
+ }
+
} catch (Exception e) {
getLog().error(e);
throw new MojoExecutionException(e.getMessage(), e);
@@ -671,9 +717,15 @@ public class GrammarExtensionMojo extends AbstractMojo {
case NEWPRODUCTION:
nextOperation = NEWPRODUCTION;
break;
+ case NEW_AT_THE_END_PRODUCTION:
+ nextOperation = NEW_AT_THE_END_PRODUCTION;
+ break;
+ case NEW_AT_THE_END_CLASS_DEFINITION:
+ nextOperation = NEW_AT_THE_END_CLASS_DEFINITION;
+ break;
case MERGEPRODUCTION:
nextOperation = MERGEPRODUCTION;
- shouldReplace = shouldReplace(tokens);
+ shouldReplace = shouldReplace(tokens, position.line);
break;
case OVERRIDEPRODUCTION:
nextOperation = OVERRIDEPRODUCTION;
@@ -693,6 +745,10 @@ public class GrammarExtensionMojo extends AbstractMojo {
case NEWPRODUCTION:
handleNew(identifier, reader);
break;
+ case NEW_AT_THE_END_CLASS_DEFINITION:
+ readFinalProduction(identifier, reader);
+ addFinalProduction(identifier, extensionMethodsAtTheClassDef);
+ break;
case OVERRIDEPRODUCTION:
handleOverride(identifier, reader);
break;
@@ -704,12 +760,16 @@ public class GrammarExtensionMojo extends AbstractMojo {
}
nextOperation = NEWPRODUCTION;
} else if (openAngularIndex == 0) {
- if (nextOperation != NEWPRODUCTION) {
+ if (nextOperation != NEWPRODUCTION && nextOperation != NEW_AT_THE_END_PRODUCTION) {
throw new MojoExecutionException("Can only add new REGEX production kind");
}
position.index = position.line.indexOf(OPEN_ANGULAR);
readFinalProduction(identifier, reader);
- addFinalProduction(identifier, extensionFinals);
+ if (nextOperation == NEWPRODUCTION) {
+ addFinalProduction(identifier, extensionFinals);
+ } else if (nextOperation == NEW_AT_THE_END_PRODUCTION) {
+ addFinalProduction(identifier, extensionFinalsAtTheEnd);
+ }
} else if (identifier.length() > 0 || position.line.trim().length() > 0) {
identifier.append(position.line);
identifier.append('\n');
@@ -721,16 +781,64 @@ public class GrammarExtensionMojo extends AbstractMojo {
}
}
- private boolean shouldReplace(String[] tokens) throws MojoExecutionException {
+ private boolean shouldReplace(String[] tokens, String currentLine) throws MojoExecutionException {
boolean replace = false;
- if (tokens.length == 5) {
- if (tokens[1].equals(REPLACE) && tokens[3].equals(WITH)) {
- shouldReplace = true;
- oldWord = tokens[2];
- newWord = tokens[4];
+ String errMessage = "Allowed syntax after @merge: <REPLACE> \"oldPhrase\" <WITH> \"newPhrase\" <TRUE|FALSE>.";
+ String errMessage1 = "The old phrase should be place between two quotes. (E.g., \"old\")";
+ String errMessage2 = "The new phrase should be place between two quotes. (E.g., \"new\")";
+ // @merge replace "oldphrase" with "newphrase" proceedBlock:true/false
+ if (tokens.length >= 6) {
+ // Checks whether "replace" exists.
+ if (!tokens[1].equalsIgnoreCase(REPLACE)) {
+ throw new MojoExecutionException(errMessage);
+ }
+
+ // Checks whether "with" exists.
+ boolean withFound = false;
+ for (int i = 3; i < tokens.length; i++) {
+ if (tokens[i].equalsIgnoreCase(WITH)) {
+ withFound = true;
+ break;
+ }
+ }
+ if (!withFound) {
+ throw new MojoExecutionException(errMessage);
+ }
+
+ // Check whether the last parameter is true/false.
+ // If this is true, then we check all blocks and process before: and after:.
+ // If not, we don't process all blocks and replace "old" with "new" for all instances for
+ // the entire part of the given method.
+ if (tokens[tokens.length - 1].equalsIgnoreCase(OPTION_TRUE)) {
+ shouldApplyEachBlockChange = true;
+ } else if (tokens[tokens.length - 1].equalsIgnoreCase(OPTION_FALSE)) {
+ shouldApplyEachBlockChange = false;
} else {
- throw new MojoExecutionException("Allowed syntax after @merge: <REPLACE> oldWord <WITH> newWord");
+ throw new MojoExecutionException(errMessage);
+ }
+
+ // Gets the old phrase.
+ int oldStart = findQuotePos(currentLine, 0);
+ if (oldStart < 0) {
+ throw new MojoExecutionException(errMessage1);
+ }
+ int oldEnd = findQuotePos(currentLine, oldStart + 1);
+ if (oldEnd < 0) {
+ throw new MojoExecutionException(errMessage1);
+ }
+ oldPhrase = currentLine.substring(oldStart + 1, oldEnd);
+
+ // Gets the new phrase.
+ int newStart = findQuotePos(currentLine, oldEnd + 1);
+ if (newStart < 0) {
+ throw new MojoExecutionException(errMessage2);
+ }
+ int newEnd = findQuotePos(currentLine, newStart + 1);
+ if (newEnd < 0) {
+ throw new MojoExecutionException(errMessage2);
}
+ newPhrase = currentLine.substring(newStart + 1, newEnd);
+ replace = true;
}
return replace;
}
@@ -776,26 +884,37 @@ public class GrammarExtensionMojo extends AbstractMojo {
throw new MojoExecutionException(identifier.toString() + " doesn't exist in base grammar");
} else if (shouldReplace) {
Pair<String, String> baseMethods = extensibles.get(sig);
- baseMethods.first = baseMethods.first.replaceAll(oldWord, newWord);
- baseMethods.second = baseMethods.second.replaceAll(oldWord, newWord);
+ // Literally replaces the old phrase with the new phrase.
+ baseMethods.first = stringReplaceAll(baseMethods.first, oldPhrase, newPhrase);
+ baseMethods.second = stringReplaceAll(baseMethods.second, oldPhrase, newPhrase);
shouldReplace = false;
}
String[] amendments = new String[6];
- mergeElements.put(sig, amendments);
+ if (shouldApplyEachBlockChange) {
+ // Applies and stores each block's change only if shouldApplyEachBlockChange is set to true.
+ mergeElements.put(sig, amendments);
+ } else {
+ // If shouldApplyEachBlockChange is false, no result from each block's change are not stored.
+ shouldApplyEachBlockChange = true;
+ }
// we don't need the identifier anymore
identifier.setLength(0);
+ // The first block
readBlock(reader, OPEN_BRACE, CLOSE_BRACE);
String block = record.toString();
extractBeforeAndAfter(block, amendments, 0, 1);
record.reset();
position.index = 0;
position.line = reader.readLine();
+ // Skips empty lines.
while (position.line != null && position.line.trim().length() == 0) {
position.line = reader.readLine();
}
+ // The second block
int openBraceIndex = position.line.indexOf(OPEN_BRACE);
if (openBraceIndex > -1) {
position.index = openBraceIndex;
+ // Reads the entire part of the second block - two OPEN_BRACE and two CLOSE_BRACE.
readBlock(reader, OPEN_BRACE, CLOSE_BRACE);
} else {
throw new MojoExecutionException("merge element doesn't have a second block");
@@ -917,4 +1036,36 @@ public class GrammarExtensionMojo extends AbstractMojo {
}
return aString.toString();
}
+
+ /**
+ * Literally replaces the given string.
+ */
+ private String stringReplaceAll(String baseStr, String oldPhrase, String newPhrase) {
+ String resultStr = baseStr;
+ String tempStr;
+ int index = resultStr.indexOf(oldPhrase);
+ while (index < resultStr.length()) {
+ if (index < 0) {
+ return resultStr;
+ }
+ tempStr = resultStr.substring(index, index + oldPhrase.length());
+ if (tempStr.equals(oldPhrase)) {
+ resultStr = resultStr.substring(0, index) + newPhrase + resultStr.substring(index + oldPhrase.length());
+ }
+ index = resultStr.indexOf(oldPhrase, index + 1);
+ }
+ return resultStr;
+ }
+
+ /**
+ * Finds and returns the first occurrence index of a quote from startingIndex.
+ *
+ * @param baseStr
+ * @param startingIndex
+ * @return the substring if found. If not, this returns null.
+ */
+ private int findQuotePos(String baseStr, int startingIndex) {
+ return baseStr.indexOf(ExternalDataConstants.QUOTE, startingIndex);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/456ff9de/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
index fb143a3..63bd38a 100644
--- a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
+++ b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
@@ -1,14 +1,29 @@
+// If you want to put an additional import, just add import statements like the following.
+// import package name
import org.apache.asterix.lang.extension.EchoStatement;
-// to remove an import, we use the keyword, unimport
+
+// To remove an import, we use the keyword, unimport
// unimport package name
+// If you want to add a method in the class definition (before PARSER_END), use the following phrase and attach
+// new method right after the phrase.
+// @new_at_the_end
+@new_at_the_class_def
+ public void initScope() {
+ scopeStack.push(RootScopeFactory.createRootScope(this));
+ }
+
// Merging of non-terminals can only be done on non-terminals which conform to the following structure.
// Content will simply be prepended or appended to the base blocks.
// Note: refrain from using the strings "before:" and "after:" in the merge areas as that will break the merge.
// As a workaround, you can always override
-// one additional possible change is direct replacement and it can be done through
-// @merge replace "baseWord" with "extensionWord"
-
+// one additional possible change is direct replacement and it can be done through the followin syntax:
+// @merge replace "base phrase" with "extension phrase" true/false
+// Here, true/false tells whether the tool needs to process the three blocks.
+// If true, like normal @merge case, before and after clause in each block will be processed
+// after "base phrase" in the blocks have been replaced with "new phrase".
+// If false, then it just expects the blank form that consists of three blocks and not process them.
+// Only, "base phrase" in the blocks will be replaced with "new phrase".
@merge
Statement SingleStatement() throws ParseException:
{
@@ -26,6 +41,24 @@ Statement SingleStatement() throws ParseException:
}
}
+// In the following case, all instances of the first phrase inside of "" will be replaced with the second phrase in "".
+// Also, we don't check "before:" and "after:" section of each area. That check will be ignored since
+// the last parameter is set to false.
+@merge replace "nameComponents = QualifiedName() (<AS> var = Variable())?" with "nameComponents = QualifiedName() (<AS> var = Variable())? " false
+InsertStatement InsertStatement() throws ParseException:
+{
+ // merge area 1
+}
+{
+ (
+ // merge area 2
+ )
+ {
+ // merge area 3
+ }
+}
+
+
// The default
// Adding a new node. if a node exists, it will throw an exception.
@new
@@ -40,11 +73,21 @@ Statement EchoStatement() throws ParseException:
}
}
+// Overriding a non-terminal. if exists in base, it will be overriden, otherwise, it will be added
+// @override
+
+
+// Terminals can be added like the following.
<DEFAULT,IN_DBL_BRACE>
TOKEN :
{
<ECHO : "echo">
}
-// Overriding a non-terminal. if exists in base, it will be overriden, otherwise, it will be added
-// @override
+// If something needs to be added at the end of file, you can use @new_at_the_end like the following.
+@new_at_the_end
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <METAVARIABLE : "$$" <IDENTIFIER> >
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/456ff9de/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
index 5475fa4..3f929c1 100644
--- a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
+++ b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
@@ -41,6 +41,7 @@
<gextension>src/test/resources/lang/extension.jj</gextension>
<output>target/generated-sources/lang/grammar.jj</output>
<parserClassName>ExtendedParser</parserClassName>
+ <parserClassModifier></parserClassModifier>
<packageName>org.apache.asterix.lang.extension.parser</packageName>
</configuration>
<executions>