You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by de...@apache.org on 2016/02/03 13:10:26 UTC
[06/51] [abbrv] lens git commit: LENS-851 : Replace columns with
aliases in where clause of the inner query
LENS-851 : Replace columns with aliases in where clause of the inner query
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/bf4c0bec
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/bf4c0bec
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/bf4c0bec
Branch: refs/heads/current-release-line
Commit: bf4c0bec023307417de75f4c13ed1c344fc1f06e
Parents: ff891e2
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Sat Dec 12 15:30:23 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Sat Dec 12 15:30:23 2015 +0530
----------------------------------------------------------------------
lens-api/src/main/resources/lens-errors.conf | 25 +++++--
.../lens/cube/error/LensCubeErrorCode.java | 6 +-
.../apache/lens/cube/metadata/ExprColumn.java | 60 ++++++++++-------
.../lens/cube/metadata/MetastoreUtil.java | 16 +++++
.../apache/lens/cube/parse/CandidateFact.java | 8 +--
.../apache/lens/cube/parse/GroupbyResolver.java | 15 +----
.../org/apache/lens/cube/parse/HQLParser.java | 11 +++-
.../lens/cube/parse/SingleFactHQLContext.java | 8 +--
.../parse/SingleFactMultiStorageHQLContext.java | 68 +++++++++++++++-----
.../apache/lens/cube/parse/UnionHQLContext.java | 2 +-
.../cube/metadata/TestCubeMetastoreClient.java | 10 +--
.../lens/cube/metadata/TestExprColumn.java | 20 +++---
.../apache/lens/cube/parse/CubeTestSetup.java | 2 +-
.../lens/cube/parse/TestCubeRewriter.java | 4 +-
.../apache/lens/server/metastore/JAXBUtils.java | 8 +--
15 files changed, 161 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-api/src/main/resources/lens-errors.conf
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/lens-errors.conf b/lens-api/src/main/resources/lens-errors.conf
index ca8562f..c880543 100644
--- a/lens-api/src/main/resources/lens-errors.conf
+++ b/lens-api/src/main/resources/lens-errors.conf
@@ -284,9 +284,15 @@ lensCubeErrorsForQuery = [
}
{
- errorCode = 3031
- httpStatusCode = ${BAD_REQUEST}
- errorMsg = "The query is answerable from two storages but union is disabled."
+ errorCode = 3031
+ httpStatusCode = ${BAD_REQUEST}
+ errorMsg = "The query is answerable from two storages but union is disabled."
+ }
+
+ {
+ errorCode = 3032
+ httpStatusCode = ${INTERNAL_SERVER_ERROR}
+ errorMsg = "Could not parse expression %s"
}
]
@@ -298,10 +304,17 @@ lensCubeErrorsForMetastore = [
}
{
- errorCode = 3102
- httpStatusCode = ${BAD_REQUEST}
- errorMsg = "No timeline found for fact=%s, storage=%s, update period=%s, partition column=%s."
+ errorCode = 3102
+ httpStatusCode = ${BAD_REQUEST}
+ errorMsg = "No timeline found for fact=%s, storage=%s, update period=%s, partition column=%s."
}
+
+ {
+ errorCode = 3103
+ httpStatusCode = ${BAD_REQUEST}
+ errorMsg = "The Expression %s is Not Parsable."
+ }
+
]
lensCubeErrors = ${lensCubeErrorsForQuery}${lensCubeErrorsForMetastore}
http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
index 6c5dc2f..68cd80b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java
@@ -54,10 +54,12 @@ public enum LensCubeErrorCode {
NO_CANDIDATE_FACT_AVAILABLE(3028, 1200),
NO_CANDIDATE_DIM_STORAGE_TABLES(3029, 1300),
NO_STORAGE_TABLE_AVAIABLE(3030, 1400),
- STORAGE_UNION_DISABLED(3031, 100),
+ STORAGE_UNION_DISABLED(3031, 1500),
+ COULD_NOT_PARSE_EXPRESSION(3032, 1500),
// Error codes greater than 3100 are errors while doing a metastore operation.
ERROR_IN_ENTITY_DEFINITION(3101, 100),
- TIMELINE_ABSENT(3102, 100);
+ TIMELINE_ABSENT(3102, 100),
+ EXPRESSION_NOT_PARSABLE(3103, 1500);
public LensErrorInfo getLensErrorInfo() {
return this.errorInfo;
http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java
index b418517..da87e31 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java
@@ -23,17 +23,14 @@ import java.io.UnsupportedEncodingException;
import java.util.*;
import org.apache.lens.cube.parse.HQLParser;
+import org.apache.lens.server.api.error.LensException;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.parse.ASTNode;
-import org.apache.hadoop.hive.ql.parse.ParseException;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-import lombok.ToString;
+import lombok.*;
public class ExprColumn extends CubeColumn {
public static final char EXPRESSION_DELIMITER = '|';
@@ -46,11 +43,11 @@ public class ExprColumn extends CubeColumn {
private int hashCode;
// for backward compatibility
- public ExprColumn(FieldSchema column, String displayString, String expression) {
+ public ExprColumn(FieldSchema column, String displayString, String expression) throws LensException {
this(column, displayString, new ExprSpec(expression, null, null));
}
- public ExprColumn(FieldSchema column, String displayString, ExprSpec... expressions) {
+ public ExprColumn(FieldSchema column, String displayString, ExprSpec... expressions) throws LensException {
super(column.getName(), column.getComment(), displayString, null, null, 0.0);
if (expressions == null || expressions.length == 0) {
@@ -124,6 +121,7 @@ public class ExprColumn extends CubeColumn {
public static class ExprSpec {
@Getter
@Setter
+ @NonNull
private String expr;
@Getter
@Setter
@@ -136,20 +134,18 @@ public class ExprColumn extends CubeColumn {
private boolean hasHashCode = false;
private transient int hashCode;
- public ExprSpec(String expr, Date startTime, Date endTime) {
+ public ExprSpec(@NonNull String expr, Date startTime, Date endTime) throws LensException {
this.expr = expr;
this.startTime = startTime;
this.endTime = endTime;
+ // validation
+ getASTNode();
}
- public synchronized ASTNode getASTNode() {
+ public synchronized ASTNode getASTNode() throws LensException {
if (astNode == null) {
- try {
- if (StringUtils.isNotBlank(expr)) {
- astNode = HQLParser.parseExpr(getExpr());
- }
- } catch (ParseException e) {
- throw new IllegalArgumentException("Expression can't be parsed: " + getExpr(), e);
+ if (StringUtils.isNotBlank(expr)) {
+ astNode = MetastoreUtil.parseExpr(getExpr());
}
}
return astNode;
@@ -160,8 +156,14 @@ public class ExprColumn extends CubeColumn {
if (!hasHashCode) {
final int prime = 31;
int result = 1;
- if (getASTNode() != null) {
- String exprNormalized = HQLParser.getString(getASTNode());
+ ASTNode astNode;
+ try {
+ astNode = getASTNode();
+ } catch (LensException e) {
+ throw new IllegalArgumentException(e);
+ }
+ if (astNode != null) {
+ String exprNormalized = HQLParser.getString(astNode);
result = prime * result + exprNormalized.hashCode();
}
result = prime * result + ((getStartTime() == null) ? 0 : COLUMN_TIME_FORMAT.get().format(
@@ -262,9 +264,17 @@ public class ExprColumn extends CubeColumn {
return false;
}
// Compare expressions for both - compare ASTs
- List<ASTNode> myExpressions = getExpressionASTList();
- List<ASTNode> otherExpressions = other.getExpressionASTList();
-
+ List<ASTNode> myExpressions, otherExpressions;
+ try {
+ myExpressions = getExpressionASTList();
+ } catch (LensException e) {
+ throw new IllegalArgumentException(e);
+ }
+ try {
+ otherExpressions = other.getExpressionASTList();
+ } catch (LensException e) {
+ throw new IllegalArgumentException(e);
+ }
for (int i = 0; i < myExpressions.size(); i++) {
if (!HQLParser.equalsAST(myExpressions.get(i), otherExpressions.get(i))) {
return false;
@@ -316,11 +326,11 @@ public class ExprColumn extends CubeColumn {
*
* @return the ast
*/
- public ASTNode getAst() {
+ public ASTNode getAst() throws LensException {
return getExpressionASTList().get(0);
}
- public List<ASTNode> getExpressionASTList() {
+ public List<ASTNode> getExpressionASTList() throws LensException {
synchronized (expressionSet) {
if (astNodeList.isEmpty()) {
for (ExprSpec expr : expressionSet) {
@@ -366,15 +376,15 @@ public class ExprColumn extends CubeColumn {
* Add an expression to existing set of expressions for this column
*
* @param expression
- * @throws ParseException
+ * @throws LensException
*/
- public void addExpression(ExprSpec expression) throws ParseException {
+ public void addExpression(ExprSpec expression) throws LensException {
if (expression == null || expression.getExpr().isEmpty()) {
throw new IllegalArgumentException("Empty expression not allowed");
}
// Validate if expression can be correctly parsed
- HQLParser.parseExpr(expression.getExpr());
+ MetastoreUtil.parseExpr(expression.getExpr());
synchronized (expressionSet) {
expressionSet.add(expression);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
index 4ec049c..deb5368 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
@@ -19,14 +19,20 @@
package org.apache.lens.cube.metadata;
+import static org.apache.lens.cube.error.LensCubeErrorCode.EXPRESSION_NOT_PARSABLE;
import static org.apache.lens.cube.metadata.MetastoreConstants.*;
import java.text.ParseException;
import java.util.*;
+import org.apache.lens.server.api.error.LensException;
+
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.ParseDriver;
+import org.apache.hadoop.hive.ql.parse.ParseUtils;
import com.google.common.collect.Sets;
@@ -536,4 +542,14 @@ public class MetastoreUtil {
}
return null;
}
+ public static ASTNode parseExpr(String expr) throws LensException {
+ ParseDriver driver = new ParseDriver();
+ ASTNode tree;
+ try {
+ tree = driver.parseExpression(expr);
+ } catch (org.apache.hadoop.hive.ql.parse.ParseException e) {
+ throw new LensException(EXPRESSION_NOT_PARSABLE.getLensErrorInfo(), e, e.getMessage(), expr);
+ }
+ return ParseUtils.findRootNonNullToken(tree);
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
index 1884bde..2338ba7 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
-import org.apache.hadoop.hive.ql.parse.ParseException;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.antlr.runtime.CommonToken;
@@ -159,12 +158,7 @@ public class CandidateFact implements CandidateTable {
TimeRange range = cubeql.getTimeRanges().get(i);
String rangeWhere = rangeToWhereClause.get(range);
if (!StringUtils.isBlank(rangeWhere)) {
- ASTNode rangeAST;
- try {
- rangeAST = HQLParser.parseExpr(rangeWhere);
- } catch (ParseException e) {
- throw new LensException(e);
- }
+ ASTNode rangeAST = HQLParser.parseExpr(rangeWhere);
rangeAST.setParent(timenodes.get(i).parent);
timenodes.get(i).parent.setChild(timenodes.get(i).childIndex, rangeAST);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
index 97088a1..da74713 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
@@ -31,7 +31,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
-import org.apache.hadoop.hive.ql.parse.ParseException;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.tree.Tree;
@@ -73,12 +72,7 @@ class GroupbyResolver implements ContextRewriter {
if (!groupByExprs.contains(expr)) {
if (!cubeql.isAggregateExpr(expr)) {
- ASTNode exprAST;
- try {
- exprAST = HQLParser.parseExpr(expr);
- } catch (ParseException e) {
- throw new LensException(e);
- }
+ ASTNode exprAST = HQLParser.parseExpr(expr);
ASTNode groupbyAST = cubeql.getGroupByAST();
if (!isConstantsUsed(exprAST)) {
if (groupbyAST != null) {
@@ -140,12 +134,7 @@ class GroupbyResolver implements ContextRewriter {
int index = 0;
for (String expr : groupByExprs) {
if (!contains(cubeql, selectExprs, expr)) {
- ASTNode exprAST;
- try {
- exprAST = HQLParser.parseExpr(expr);
- } catch (ParseException e) {
- throw new LensException(e);
- }
+ ASTNode exprAST = HQLParser.parseExpr(expr);
addChildAtIndex(index, cubeql.getSelectAST(), exprAST);
index++;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
index 9a9d134..7cea7d5 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
@@ -18,10 +18,10 @@
*/
package org.apache.lens.cube.parse;
+import static org.apache.lens.cube.error.LensCubeErrorCode.COULD_NOT_PARSE_EXPRESSION;
import static org.apache.lens.cube.error.LensCubeErrorCode.SYNTAX_ERROR;
import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
-import static org.apache.hadoop.hive.ql.parse.HiveParser.Number;
import java.io.IOException;
import java.lang.reflect.Field;
@@ -170,9 +170,14 @@ public final class HQLParser {
return tree;
}
- public static ASTNode parseExpr(String expr) throws ParseException {
+ public static ASTNode parseExpr(String expr) throws LensException {
ParseDriver driver = new ParseDriver();
- ASTNode tree = driver.parseExpression(expr);
+ ASTNode tree;
+ try {
+ tree = driver.parseExpression(expr);
+ } catch (ParseException e) {
+ throw new LensException(COULD_NOT_PARSE_EXPRESSION.getLensErrorInfo(), e, e.getMessage());
+ }
return ParseUtils.findRootNonNullToken(tree);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java
index f7271e5..de52b0a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java
@@ -26,7 +26,6 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.parse.ASTNode;
-import org.apache.hadoop.hive.ql.parse.ParseException;
/**
* HQL context class which passes down all query strings to come from DimOnlyHQLContext and works with fact being
@@ -67,12 +66,7 @@ class SingleFactHQLContext extends DimOnlyHQLContext {
String rangeWhere = entry.getKey();
if (!StringUtils.isBlank(rangeWhere)) {
- ASTNode rangeAST;
- try {
- rangeAST = HQLParser.parseExpr(rangeWhere);
- } catch (ParseException e) {
- throw new LensException(e);
- }
+ ASTNode rangeAST = HQLParser.parseExpr(rangeWhere);
rangeAST.setParent(range.getParent());
range.getParent().setChild(range.getChildIndex(), rangeAST);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
index 418ef5a..96b1d05 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
@@ -19,7 +19,7 @@
package org.apache.lens.cube.parse;
-import static org.apache.lens.cube.parse.HQLParser.getString;
+import static org.apache.lens.cube.parse.HQLParser.*;
import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
@@ -66,7 +66,7 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
@Override
public int hashCode() {
if (!hashCodeComputed) {
- hashCode = HQLParser.getString(ast).hashCode();
+ hashCode = getString(ast).hashCode();
hashCodeComputed = true;
}
return hashCode;
@@ -74,8 +74,8 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
@Override
public boolean equals(Object o) {
- return o instanceof HashableASTNode && this.hashCode() == o.hashCode() && HQLParser.getString(this.getAST())
- .trim().equalsIgnoreCase(HQLParser.getString(((HashableASTNode) o).getAST()).trim());
+ return o instanceof HashableASTNode && this.hashCode() == o.hashCode() && getString(this.getAST())
+ .trim().equalsIgnoreCase(getString(((HashableASTNode) o).getAST()).trim());
}
}
@@ -86,6 +86,7 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
super(query, fact);
processSelectAST();
processGroupByAST();
+ processWhereAST();
processHavingAST();
processOrderByAST();
processLimit();
@@ -94,10 +95,10 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
private void processSelectAST() {
query.getSelectFinalAliases().clear();
- ASTNode originalSelectAST = HQLParser.copyAST(query.getSelectAST());
+ ASTNode originalSelectAST = copyAST(query.getSelectAST());
query.setSelectAST(new ASTNode(originalSelectAST.getToken()));
ASTNode outerSelectAST = processExpression(originalSelectAST);
- setSelect(HQLParser.getString(outerSelectAST));
+ setSelect(getString(outerSelectAST));
}
private void processGroupByAST() {
@@ -106,16 +107,25 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
}
}
+ private void processWhereAST() throws LensException {
+ for (String storageTable : fact.getStorgeWhereClauseMap().keySet()) {
+ ASTNode tree = parseExpr(fact.getStorgeWhereClauseMap().get(storageTable));
+ ASTNode replaced = replaceAST(tree);
+ fact.getStorgeWhereClauseMap().put(storageTable, getString(replaced));
+ }
+ }
+
private void processHavingAST() throws LensException {
if (query.getHavingAST() != null) {
- setHaving(HQLParser.getString(processExpression(query.getHavingAST())));
+ setHaving(getString(processExpression(query.getHavingAST())));
query.setHavingAST(null);
}
}
+
private void processOrderByAST() {
if (query.getOrderByAST() != null) {
- setOrderby(HQLParser.getString(processExpression(query.getOrderByAST())));
+ setOrderby(getString(processExpression(query.getOrderByAST())));
query.setOrderByAST(null);
}
}
@@ -124,6 +134,7 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
setLimit(query.getLimitValue());
query.setLimitValue(null);
}
+
/*
Perform a DFS on the provided AST, and Create an AST of similar structure with changes specific to the
inner query - outer query dynamics. The resultant AST is supposed to be used in outer query.
@@ -147,11 +158,11 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
if (astNode == null) {
return null;
}
- if (innerToOuterASTs.containsKey(new HashableASTNode(astNode))) {
- return innerToOuterASTs.get(new HashableASTNode(astNode));
- }
- if (HQLParser.isAggregateAST(astNode)) {
- ASTNode innerSelectASTWithoutAlias = HQLParser.copyAST(astNode);
+ if (isAggregateAST(astNode)) {
+ if (innerToOuterASTs.containsKey(new HashableASTNode(astNode))) {
+ return innerToOuterASTs.get(new HashableASTNode(astNode));
+ }
+ ASTNode innerSelectASTWithoutAlias = copyAST(astNode);
ASTNode innerSelectExprAST = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR));
innerSelectExprAST.addChild(innerSelectASTWithoutAlias);
String alias = decideAlias(astNode);
@@ -164,8 +175,11 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
outerAST.addChild(dotAST);
innerToOuterASTs.put(new HashableASTNode(innerSelectASTWithoutAlias), outerAST);
return outerAST;
- } else if (HQLParser.isTableColumnAST(astNode)) {
- ASTNode innerSelectASTWithoutAlias = HQLParser.copyAST(astNode);
+ } else if (isTableColumnAST(astNode)) {
+ if (innerToOuterASTs.containsKey(new HashableASTNode(astNode))) {
+ return innerToOuterASTs.get(new HashableASTNode(astNode));
+ }
+ ASTNode innerSelectASTWithoutAlias = copyAST(astNode);
ASTNode innerSelectExprAST = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR));
innerSelectExprAST.addChild(innerSelectASTWithoutAlias);
String alias = decideAlias(astNode);
@@ -186,6 +200,30 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
}
}
+ /**
+ * Transforms the inner query's AST so that aliases are used now instead of column names.
+ * Does so in-place, without creating new ASTNode instances.
+ * @param astNode inner query's AST Node to transform
+ * @return Transformed AST Node.
+ */
+ private ASTNode replaceAST(ASTNode astNode) {
+ if (astNode == null) {
+ return null;
+ }
+ if (isAggregateAST(astNode) || isTableColumnAST(astNode)) {
+ if (innerToOuterASTs.containsKey(new HashableASTNode(astNode))) {
+ ASTNode ret = innerToOuterASTs.get(new HashableASTNode(astNode));
+ // Set parent null for quicker GC
+ astNode.setParent(null);
+ return ret;
+ }
+ }
+ for (int i = 0; i < astNode.getChildCount(); i++) {
+ astNode.setChild(i, replaceAST((ASTNode) astNode.getChild(i)));
+ }
+ return astNode;
+ }
+
private void addToInnerSelectAST(ASTNode selectExprAST) {
if (query.getSelectAST() == null) {
query.setSelectAST(new ASTNode(new CommonToken(TOK_SELECT)));
http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionHQLContext.java
index c9ba561..e6ee989 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionHQLContext.java
@@ -33,7 +33,7 @@ import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public abstract class UnionHQLContext extends SimpleHQLContext {
protected final CubeQueryContext query;
- private final CandidateFact fact;
+ protected final CandidateFact fact;
List<HQLContextInterface> hqlContexts = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index c6ce6ad..0fef13f 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -43,11 +43,7 @@ import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Partition;
-import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.parse.ParseException;
+import org.apache.hadoop.hive.ql.metadata.*;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
@@ -305,7 +301,7 @@ public class TestCubeMetastoreClient {
new DerivedCube(derivedCubeNameWithProps, measures, dimensions, CUBE_PROPERTIES, 0L, cubeWithProps);
}
- private static void defineUberDims() {
+ private static void defineUberDims() throws LensException {
// Define zip dimension
zipAttrs.add(new BaseDimAttribute(new FieldSchema("zipcode", "int", "code")));
zipAttrs.add(new BaseDimAttribute(new FieldSchema("f1", "string", "field1")));
@@ -411,7 +407,7 @@ public class TestCubeMetastoreClient {
expr1.setExpr("contact(countrydim.name");
stateCountryExpr.addExpression(expr1);
fail("Expected add expression to fail because of syntax error");
- } catch (ParseException exc) {
+ } catch (LensException exc) {
// Pass
}
city.alterExpression(stateCountryExpr);
http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestExprColumn.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestExprColumn.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestExprColumn.java
index 8770f1a..0153b2d 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestExprColumn.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestExprColumn.java
@@ -28,7 +28,9 @@ import java.util.Iterator;
import java.util.Map;
import java.util.TimeZone;
+import org.apache.lens.cube.error.LensCubeErrorCode;
import org.apache.lens.cube.metadata.ExprColumn.ExprSpec;
+import org.apache.lens.server.api.error.LensException;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -142,7 +144,7 @@ public class TestExprColumn {
}
@Test
- public void testExprColumnCreationErrors() {
+ public void testExprColumnCreationErrors() throws LensException {
FieldSchema colSchema = new FieldSchema("errorColumn", "double", "multi exprcol");
// no expression spec passed
@@ -157,16 +159,16 @@ public class TestExprColumn {
try {
ExprColumn col1 = new ExprColumn(colSchema, "NoExprInExprSpec", new ExprSpec(null, null, null));
fail(col1 + " should not be created");
- } catch (IllegalArgumentException e) {
- assertTrue(e.getMessage().contains("No expression string specified for column errorColumn at index:0"));
+ } catch (NullPointerException e) {
+ // pass
}
// Parse error in expr passed in exprspec
try {
ExprColumn col1 = new ExprColumn(colSchema, "NoExprInExprSpec", new ExprSpec("(a+b", null, null));
fail(col1 + " should not be created");
- } catch (IllegalArgumentException e) {
- assertTrue(e.getMessage().contains("Expression can't be parsed: (a+b"), e.getMessage());
+ } catch (LensException e) {
+ assertEquals(e.getErrorCode(), LensCubeErrorCode.EXPRESSION_NOT_PARSABLE.getLensErrorInfo().getErrorCode());
}
// Parse error in expr passed in exprspec
@@ -174,8 +176,8 @@ public class TestExprColumn {
ExprColumn col1 = new ExprColumn(colSchema, "NoExprInExprSpec", new ExprSpec("a + b", null, null),
new ExprSpec("(a+b", null, null));
fail(col1 + " should not be created");
- } catch (IllegalArgumentException e) {
- assertTrue(e.getMessage().contains("Expression can't be parsed: (a+b"));
+ } catch (LensException e) {
+ assertEquals(e.getErrorCode(), LensCubeErrorCode.EXPRESSION_NOT_PARSABLE.getLensErrorInfo().getErrorCode());
}
// no expression passed in exprspec
@@ -183,8 +185,8 @@ public class TestExprColumn {
ExprColumn col1 = new ExprColumn(colSchema, "NoExprInExprSpecAt1", new ExprSpec("a + b", null, null),
new ExprSpec(null, null, null));
fail(col1 + " should not be created");
- } catch (IllegalArgumentException e) {
- assertTrue(e.getMessage().contains("No expression string specified for column errorColumn at index:1"));
+ } catch (NullPointerException e) {
+ // pass
}
// startTime after endTime
http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 2a50d74..3f01dbe 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -1561,7 +1561,7 @@ public class CubeTestSetup {
}
// DimWithTwoStorages
- private void createCityTable(CubeMetastoreClient client) throws HiveException, ParseException {
+ private void createCityTable(CubeMetastoreClient client) throws HiveException, ParseException, LensException {
Set<CubeDimAttribute> cityAttrs = new HashSet<CubeDimAttribute>();
cityAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "code")));
cityAttrs.add(new BaseDimAttribute(new FieldSchema("name", "string", "city name")));
http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index 3be9406..0f05556 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -416,7 +416,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
conf.setBoolean(CubeQueryConfUtil.ENABLE_STORAGES_UNION, true);
hqlQuery = rewrite("select cityid as `City ID`, msr8, msr7 as `Third measure` "
- + "from testCube where " + TWO_MONTHS_RANGE_UPTO_HOURS, conf);
+ + "from testCube where cityid = 'a' and zipcode = 'b' and " + TWO_MONTHS_RANGE_UPTO_HOURS, conf);
expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider,
"SELECT testcube.alias0 as `City ID`, sum(testcube.alias1) + max(testcube.alias2), "
@@ -426,7 +426,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
"select testcube.cityid as `alias0`, sum(testcube.msr2) as `alias1`, "
+ "max(testcube.msr3) as `alias2`, "
+ "sum(case when testcube.cityid = 'x' then testcube.msr21 else testcube.msr22 end) as `alias3`",
- null, "group by testcube.cityid");
+ "testcube.alias0 = 'a' and testcube.zipcode = 'b'", "group by testcube.cityid");
compareQueries(hqlQuery, expected);
http://git-wip-us.apache.org/repos/asf/lens/blob/bf4c0bec/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
index a5883f7..817c84c 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
@@ -316,7 +316,7 @@ public final class JAXBUtils {
return xes;
}
- private static ExprSpec[] exprSpecFromXExprColumn(Collection<XExprSpec> xesList) {
+ private static ExprSpec[] exprSpecFromXExprColumn(Collection<XExprSpec> xesList) throws LensException {
List<ExprSpec> esArray = new ArrayList<ExprSpec>(xesList.size());
for (XExprSpec xes : xesList) {
esArray.add(new ExprSpec(xes.getExpr(), getDateFromXML(xes.getStartTime()), getDateFromXML(xes.getEndTime())));
@@ -478,7 +478,7 @@ public final class JAXBUtils {
return jc;
}
- public static ExprColumn hiveExprColumnFromXExprColumn(XExprColumn xe) {
+ public static ExprColumn hiveExprColumnFromXExprColumn(XExprColumn xe) throws LensException {
ExprColumn ec = new ExprColumn(new FieldSchema(xe.getName(), xe.getType().toLowerCase(),
xe.getDescription()),
xe.getDisplayString(),
@@ -598,7 +598,7 @@ public final class JAXBUtils {
return null;
}
- Storage storage = null;
+ Storage storage;
try {
Class<?> clazz = Class.forName(xs.getClassname());
Constructor<?> constructor = clazz.getConstructor(String.class);
@@ -924,7 +924,7 @@ public final class JAXBUtils {
return ret;
}
- public static Dimension dimensionFromXDimension(XDimension dimension) {
+ public static Dimension dimensionFromXDimension(XDimension dimension) throws LensException {
Set<CubeDimAttribute> dims = new LinkedHashSet<CubeDimAttribute>();
for (XDimAttribute xd : dimension.getAttributes().getDimAttribute()) {
dims.add(hiveDimAttrFromXDimAttr(xd));