You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2016/10/13 06:44:47 UTC
lens git commit: LENS-1355 : Fix parsing fields named 'date'
Repository: lens
Updated Branches:
refs/heads/master 39252f9af -> 182f6dcac
LENS-1355 : Fix parsing fields named 'date'
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/182f6dca
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/182f6dca
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/182f6dca
Branch: refs/heads/master
Commit: 182f6dcacc0226dc0e070edaa3fc8ba4aa3e6146
Parents: 39252f9
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Thu Oct 13 12:14:30 2016 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Thu Oct 13 12:14:30 2016 +0530
----------------------------------------------------------------------
.../lens/cube/parse/AggregateResolver.java | 2 +-
.../lens/cube/parse/CubeQueryContext.java | 4 +--
.../apache/lens/cube/parse/GroupbyResolver.java | 4 +--
.../org/apache/lens/cube/parse/HQLParser.java | 26 +++++++++++++++++---
.../apache/lens/cube/parse/TestHQLParser.java | 9 ++++---
5 files changed, 33 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/182f6dca/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
index c522061..9658100 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
@@ -119,7 +119,7 @@ class AggregateResolver implements ContextRewriter {
cubeql.getExprCtx().getAllExprsQueried().get(expr).iterator(); itrContext.hasNext();) {
for (Iterator<ExprColumn.ExprSpec> itrCol =
itrContext.next().getExprCol().getExpressionSpecs().iterator(); itrCol.hasNext();) {
- ASTNode exprAST = HQLParser.parseExpr(itrCol.next().getExpr());
+ ASTNode exprAST = HQLParser.parseExpr(itrCol.next().getExpr(), cubeql.getConf());
if (HQLParser.isAggregateAST(exprAST)) {
return true;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/182f6dca/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 6d53d00..e83ae76 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -886,10 +886,10 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST {
String table = entry.getKey();
String rangeWhere = entry.getValue();
if (!StringUtils.isBlank(rangeWhere)) {
- ASTNode rangeAST = HQLParser.parseExpr(rangeWhere);
+ ASTNode rangeAST = HQLParser.parseExpr(rangeWhere, conf);
range.getParent().setChild(range.getChildIndex(), rangeAST);
}
- fact.getStorgeWhereClauseMap().put(table, HQLParser.parseExpr(getWhereString()));
+ fact.getStorgeWhereClauseMap().put(table, HQLParser.parseExpr(getWhereString(), conf));
}
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/182f6dca/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
index 26ae1e7..216ae52 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
@@ -71,7 +71,7 @@ class GroupbyResolver implements ContextRewriter {
String expr = sel.getExprWithoutAlias();
if (!groupByExprs.contains(expr)) {
if (!sel.isAggregate()) {
- ASTNode exprAST = HQLParser.parseExpr(expr);
+ ASTNode exprAST = HQLParser.parseExpr(expr, cubeql.getConf());
ASTNode groupbyAST = cubeql.getGroupByAST();
if (!isConstantsUsed(exprAST)) {
if (groupbyAST != null) {
@@ -130,7 +130,7 @@ class GroupbyResolver implements ContextRewriter {
int index = 0;
for (String expr : groupByExprs) {
if (!contains(selectExprs, expr)) {
- ASTNode exprAST = HQLParser.parseExpr(expr);
+ ASTNode exprAST = HQLParser.parseExpr(expr, cubeql.getConf());
addChildAtIndex(index, cubeql.getSelectAST(), exprAST);
index++;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/182f6dca/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
index 1976248..8a70535 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
@@ -31,6 +31,7 @@ import java.util.regex.Pattern;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -38,6 +39,8 @@ import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.parse.*;
import org.antlr.runtime.CommonToken;
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.TokenRewriteStream;
import org.antlr.runtime.tree.Tree;
import com.google.common.base.Optional;
@@ -196,15 +199,32 @@ public final class HQLParser {
}
public static ASTNode parseExpr(String expr) throws LensException {
- ParseDriver driver = new ParseDriver();
+ return parseExpr(expr, null);
+ }
+ public static ASTNode parseExpr(String expr, Configuration conf) throws LensException {
ASTNode tree;
try {
- tree = driver.parseExpression(expr);
- } catch (ParseException e) {
+ tree = parseExpression(expr, conf);
+ } catch (ParseException|RecognitionException e) {
throw new LensException(COULD_NOT_PARSE_EXPRESSION.getLensErrorInfo(), e, expr);
}
return ParseUtils.findRootNonNullToken(tree);
}
+ public static ASTNode parseExpression(String command, Configuration conf)
+ throws ParseException, RecognitionException {
+ ParseDriver driver = new ParseDriver();
+ ParseDriver.HiveLexerX lexer = driver.new HiveLexerX(driver.new ANTLRNoCaseStringStream(command));
+ if (conf != null) {
+ lexer.setHiveConf(conf);
+ }
+ TokenRewriteStream tokens = new TokenRewriteStream(lexer);
+ HiveParser parser = new HiveParser(tokens);
+ if (conf != null) {
+ parser.setHiveConf(conf);
+ }
+ parser.setTreeAdaptor(ParseDriver.adaptor);
+ return (ASTNode)parser.expression().getTree();
+ }
public static void printAST(ASTNode node) {
try {
http://git-wip-us.apache.org/repos/asf/lens/blob/182f6dca/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java
index 2619cb0..3165d3a 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java
@@ -494,15 +494,16 @@ public class TestHQLParser {
@DataProvider
public Object[][] exprDataProvider() {
return new Object[][] {
- {"a.b", true},
- {"a.date", false},
+ {"a.b", null, true},
+ {"a.date", null, false},
+ {"a.date", conf, true},
};
}
@Test(dataProvider = "exprDataProvider")
- public void testParseExpr(String expr, boolean success) {
+ public void testParseExpr(String expr, HiveConf conf, boolean success) {
try {
- HQLParser.parseExpr(expr);
+ HQLParser.parseExpr(expr, conf);
Assert.assertTrue(success);
} catch (LensException e) {
Assert.assertFalse(success);