You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by qi...@apache.org on 2016/10/19 03:19:06 UTC
[07/28] incubator-eagle git commit: EAGLE-330 Fix Hive ql.Parser Hive
ql.Parser cant parser a hive query sql with keywords
https://issues.apache.org/jira/browse/EAGLE-330
EAGLE-330 Fix Hive ql.Parser
Hive ql.Parser cant parser a hive query sql with keywords
https://issues.apache.org/jira/browse/EAGLE-330
Author: @wgyumg <wg...@gmail.com>
Reviewer: @pkuwm <hl...@ebay.com>
Closes: #228
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/cbf3c76c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/cbf3c76c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/cbf3c76c
Branch: refs/heads/master
Commit: cbf3c76ce148b73379698d9626df6e1baa5162ab
Parents: f51474d
Author: yonzhang <yo...@gmail.com>
Authored: Tue Jun 7 16:01:41 2016 -0700
Committer: yonzhang <yo...@gmail.com>
Committed: Tue Jun 7 16:01:41 2016 -0700
----------------------------------------------------------------------
.../apache/eagle/security/hive/ql/Parser.java | 46 ++++++++++++++++++--
.../eagle/security/hive/ql/TestParser.java | 30 ++++++++++++-
2 files changed, 70 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/cbf3c76c/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/ql/Parser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/ql/Parser.java b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/ql/Parser.java
index 5ab0f4b..19ce003 100644
--- a/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/ql/Parser.java
+++ b/eagle-security/eagle-security-hive/src/main/java/org/apache/eagle/security/hive/ql/Parser.java
@@ -16,11 +16,19 @@
*/
package org.apache.eagle.security.hive.ql;
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.Token;
+import org.antlr.runtime.TokenRewriteStream;
+import org.antlr.runtime.TokenStream;
+import org.antlr.runtime.tree.CommonTree;
+import org.antlr.runtime.tree.CommonTreeAdaptor;
+import org.antlr.runtime.tree.TreeAdaptor;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.parse.ASTErrorNode;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.ParseDriver;
-import org.apache.hadoop.hive.ql.parse.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,6 +41,21 @@ import java.util.Set;
public class Parser {
private static final Logger LOG = LoggerFactory.getLogger(Parser.class);
+ static final TreeAdaptor adaptor = new CommonTreeAdaptor() {
+ @Override
+ public Object create(Token payload) {
+ return new ASTNode(payload);
+ }
+ @Override
+ public Object dupNode(Object t) {
+ return create(((CommonTree)t).token);
+ }
+ @Override
+ public Object errorNode(TokenStream input, Token start, Token stop, RecognitionException e) {
+ return new ASTErrorNode(input, start, stop, e);
+ }
+ };
+
private Set<String> tableSet;
private Set<String> columnSet;
private Map<String, String> columnAliasMap;
@@ -66,11 +89,26 @@ public class Parser {
* Parse an Hive QL into an Abstract Syntax Tree(AST).
* @param query
* @return
- * @throws ParseException
+ * @throws RecognitionException
*/
- public ASTNode generateAST(String query) throws ParseException {
+ public ASTNode generateAST(String query) throws RecognitionException {
+ // https://issues.apache.org/jira/browse/HIVE-10731
+ // https://issues.apache.org/jira/browse/HIVE-6617
+ HiveConf hiveConf = new HiveConf();
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_SQL11_RESERVED_KEYWORDS, false);
+
ParseDriver pd = new ParseDriver();
- return pd.parse(query);
+ ParseDriver.ANTLRNoCaseStringStream antlrStream = pd.new ANTLRNoCaseStringStream(query);
+ ParseDriver.HiveLexerX lexer = pd.new HiveLexerX(antlrStream);
+ lexer.setHiveConf(hiveConf);
+ TokenRewriteStream tokens = new TokenRewriteStream(lexer);
+
+ HiveParser parser = new HiveParser(tokens);
+ parser.setHiveConf(hiveConf);
+ parser.setTreeAdaptor(adaptor);
+ HiveParser.statement_return r = parser.statement();
+
+ return (ASTNode)r.getTree();
}
private void parseQL(ASTNode ast) {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/cbf3c76c/eagle-security/eagle-security-hive/src/test/java/org/apache/eagle/security/hive/ql/TestParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hive/src/test/java/org/apache/eagle/security/hive/ql/TestParser.java b/eagle-security/eagle-security-hive/src/test/java/org/apache/eagle/security/hive/ql/TestParser.java
index a5f5f82..0bd994f 100644
--- a/eagle-security/eagle-security-hive/src/test/java/org/apache/eagle/security/hive/ql/TestParser.java
+++ b/eagle-security/eagle-security-hive/src/test/java/org/apache/eagle/security/hive/ql/TestParser.java
@@ -22,7 +22,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.hadoop.hive.ql.parse.ASTNode;
-import org.apache.hadoop.hive.ql.parse.ParseException;
+import org.antlr.runtime.RecognitionException;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -67,7 +67,7 @@ public class TestParser {
}
}
- public void printQueryAST(String query) throws ParseException {
+ public void printQueryAST(String query) throws RecognitionException {
ASTNode root = parser.generateAST(query);
printTree(root, 0);
}
@@ -209,6 +209,32 @@ public class TestParser {
}
@Test
+ public void testQueryWithKeyWords() throws Exception {
+ String query = "SELECT id, user FROM db.table1";
+ String expectedOperation = "SELECT";
+ String expectedInsertTable = null;
+ Map<String, Set<String>> expectedTableColumn = new HashMap<String, Set<String>>();
+ Set<String> set = new HashSet<String>();
+ set.add("id");
+ set.add("user");
+ expectedTableColumn.put("db.table1", set);
+ _testParsingQuery(query, expectedOperation, expectedInsertTable, expectedTableColumn);
+ }
+
+ @Test
+ public void testInsertTableWithKeyWords() throws Exception {
+ String query = "INSERT OVERWRITE TABLE table2 SELECT id, user FROM db.table1";
+ String expectedOperation = "SELECT";
+ String expectedInsertTable = "table2";
+ Map<String, Set<String>> expectedTableColumn = new HashMap<String, Set<String>>();
+ Set<String> set = new HashSet<String>();
+ set.add("id");
+ set.add("user");
+ expectedTableColumn.put("db.table1", set);
+ _testParsingQuery(query, expectedOperation, expectedInsertTable, expectedTableColumn);
+ }
+
+ @Test
public void testCreateTable() throws Exception {
String query = "CREATE TABLE page_view(viewTime INT, userid BIGINT,\n" +
" page_url STRING, referrer_url STRING,\n" +