You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by su...@apache.org on 2016/05/25 11:17:35 UTC
lens git commit: LENS-1149 : Union queries with dim-attribute
expressions are not written properly
Repository: lens
Updated Branches:
refs/heads/master 62cafa448 -> a4b4b85d8
LENS-1149 : Union queries with dim-attribute expressions are not written properly
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/a4b4b85d
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/a4b4b85d
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/a4b4b85d
Branch: refs/heads/master
Commit: a4b4b85d8179f1d7c7f0773f47b7aeebe2a0234d
Parents: 62cafa4
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Wed May 25 16:47:03 2016 +0530
Committer: sushilmohanty <su...@apache.org>
Committed: Wed May 25 16:47:03 2016 +0530
----------------------------------------------------------------------
.../parse/SingleFactMultiStorageHQLContext.java | 169 ++++++++++++-------
.../apache/lens/cube/parse/CubeTestSetup.java | 2 +
.../lens/cube/parse/TestUnionQueries.java | 68 ++++++++
3 files changed, 180 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/a4b4b85d/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
index 1af031a..9b48213 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
@@ -66,7 +66,7 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
private void processSelectAST() {
ASTNode originalSelectAST = MetastoreUtil.copyAST(ast.getSelectAST());
ast.setSelectAST(new ASTNode(originalSelectAST.getToken()));
- ASTNode outerSelectAST = processExpression(originalSelectAST);
+ ASTNode outerSelectAST = processSelectExpression(originalSelectAST);
setSelect(getString(outerSelectAST));
}
@@ -86,7 +86,7 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
private void processOrderByAST() {
if (ast.getOrderByAST() != null) {
- setOrderby(getString(processExpression(ast.getOrderByAST())));
+ setOrderby(getString(processOrderbyExpression(ast.getOrderByAST())));
ast.setOrderByAST(null);
}
}
@@ -96,51 +96,117 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
ast.setLimitValue(null);
}
+ private ASTNode processExpression(ASTNode astNode) {
+ if (astNode == null) {
+ return null;
+ }
+ ASTNode outerExpression = new ASTNode(astNode);
+ // iterate over all children of the ast and get outer ast corresponding to it.
+ for (Node child : astNode.getChildren()) {
+ outerExpression.addChild(getOuterAST((ASTNode)child));
+ }
+ return outerExpression;
+ }
+
+ private ASTNode processSelectExpression(ASTNode astNode) {
+ if (astNode == null) {
+ return null;
+ }
+ ASTNode outerExpression = new ASTNode(astNode);
+ // iterate over all children of the ast and get outer ast corresponding to it.
+ for (Node node : astNode.getChildren()) {
+ ASTNode child = (ASTNode)node;
+ ASTNode outerSelect = new ASTNode(child);
+ ASTNode selectExprAST = (ASTNode)child.getChild(0);
+ ASTNode outerAST = getOuterAST(selectExprAST);
+ outerSelect.addChild(outerAST);
+
+ // has an alias? add it
+ if (child.getChildCount() > 1) {
+ outerSelect.addChild(child.getChild(1));
+ }
+ outerExpression.addChild(outerSelect);
+ }
+ return outerExpression;
+ }
+
+ private ASTNode processOrderbyExpression(ASTNode astNode) {
+ if (astNode == null) {
+ return null;
+ }
+ ASTNode outerExpression = new ASTNode(astNode);
+ // sample orderby AST looks the following :
+ /*
+ TOK_ORDERBY
+ TOK_TABSORTCOLNAMEDESC
+ TOK_NULLS_LAST
+ .
+ TOK_TABLE_OR_COL
+ testcube
+ cityid
+ TOK_TABSORTCOLNAMEASC
+ TOK_NULLS_FIRST
+ .
+ TOK_TABLE_OR_COL
+ testcube
+ stateid
+ TOK_TABSORTCOLNAMEASC
+ TOK_NULLS_FIRST
+ .
+ TOK_TABLE_OR_COL
+ testcube
+ zipcode
+ */
+ for (Node node : astNode.getChildren()) {
+ ASTNode child = (ASTNode)node;
+ ASTNode outerOrderby = new ASTNode(child);
+ ASTNode tokNullsChild = (ASTNode) child.getChild(0);
+ ASTNode outerTokNullsChild = new ASTNode(tokNullsChild);
+ outerTokNullsChild.addChild(getOuterAST((ASTNode)tokNullsChild.getChild(0)));
+ outerOrderby.addChild(outerTokNullsChild);
+ outerExpression.addChild(outerOrderby);
+ }
+ return outerExpression;
+ }
/*
+
Perform a DFS on the provided AST, and Create an AST of similar structure with changes specific to the
inner query - outer query dynamics. The resultant AST is supposed to be used in outer query.
Base cases:
1. ast is null => null
- 2. ast is table.column => add this to inner select expressions, generate alias, return cube.alias. Memoize the
- mapping table.column => cube.alias
- 3. ast is aggregate_function(table.column) => add aggregate_function(table.column) to inner select expressions,
+ 2. ast is aggregate_function(table.column) => add aggregate_function(table.column) to inner select expressions,
generate alias, return aggregate_function(cube.alias). Memoize the mapping
aggregate_function(table.column) => aggregate_function(cube.alias)
Assumption is aggregate_function is transitive i.e. f(a,b,c,d) = f(f(a,b), f(c,d)). SUM, MAX, MIN etc
are transitive, while AVG, COUNT etc are not. For non-transitive aggregate functions, the re-written
query will be incorrect.
- 4. If given ast is memoized as mentioned in the above cases, return the mapping.
-
- Recursive case:
- Copy the root node, process children recursively and add as children to the copied node. Return the copied node.
+ 3. ast has aggregates - iterate over children and add the non aggregate nodes as is and recursively get outer ast
+ for aggregate.
+ 4. If no aggregates, simply select its alias in outer ast.
+ 5. If given ast is memorized as mentioned in the above cases, return the mapping.
*/
- private ASTNode processExpression(ASTNode astNode) {
+ private ASTNode getOuterAST(ASTNode astNode) {
if (astNode == null) {
return null;
}
+ if (innerToOuterASTs.containsKey(new HashableASTNode(astNode))) {
+ return innerToOuterASTs.get(new HashableASTNode(astNode));
+ }
if (isAggregateAST(astNode)) {
- if (innerToOuterASTs.containsKey(new HashableASTNode(astNode))) {
- return innerToOuterASTs.get(new HashableASTNode(astNode));
+ return processAggregate(astNode);
+ } else if (hasAggregate(astNode)) {
+ ASTNode outerAST = new ASTNode(astNode);
+ for (Node child : astNode.getChildren()) {
+ ASTNode childAST = (ASTNode) child;
+ if (hasAggregate(childAST)) {
+ outerAST.addChild(getOuterAST(childAST));
+ } else {
+ outerAST.addChild(childAST);
+ }
}
- ASTNode innerSelectASTWithoutAlias = MetastoreUtil.copyAST(astNode);
- ASTNode innerSelectExprAST = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR));
- innerSelectExprAST.addChild(innerSelectASTWithoutAlias);
- String alias = aliasDecider.decideAlias(astNode);
- ASTNode aliasNode = new ASTNode(new CommonToken(Identifier, alias));
- innerSelectExprAST.addChild(aliasNode);
- addToInnerSelectAST(innerSelectExprAST);
- ASTNode dotAST = getDotAST(query.getCube().getName(), alias);
- ASTNode outerAST = new ASTNode(new CommonToken(TOK_FUNCTION));
- //TODO: take care or non-transitive aggregate functions
- outerAST.addChild(new ASTNode(new CommonToken(Identifier, astNode.getChild(0).getText())));
- outerAST.addChild(dotAST);
- innerToOuterASTs.put(new HashableASTNode(innerSelectASTWithoutAlias), outerAST);
return outerAST;
- } else if (isTableColumnAST(astNode) || isNonAggregateFunctionAST(astNode)) {
- if (innerToOuterASTs.containsKey(new HashableASTNode(astNode))) {
- return innerToOuterASTs.get(new HashableASTNode(astNode));
- }
+ } else {
ASTNode innerSelectASTWithoutAlias = MetastoreUtil.copyAST(astNode);
ASTNode innerSelectExprAST = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR));
innerSelectExprAST.addChild(innerSelectASTWithoutAlias);
@@ -151,39 +217,24 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
ASTNode outerAST = getDotAST(query.getCube().getName(), alias);
innerToOuterASTs.put(new HashableASTNode(innerSelectASTWithoutAlias), outerAST);
return outerAST;
- } else {
- ASTNode outerHavingExpression = new ASTNode(astNode);
- if (astNode.getChildren() != null) {
- for (Node child : astNode.getChildren()) {
- outerHavingExpression.addChild(processExpression((ASTNode) child));
- }
- }
- return outerHavingExpression;
}
}
- /**
- * Transforms the inner query's AST so that aliases are used now instead of column names.
- * Does so in-place, without creating new ASTNode instances.
- * @param astNode inner query's AST Node to transform
- * @return Transformed AST Node.
- */
- private ASTNode replaceAST(ASTNode astNode) {
- if (astNode == null) {
- return null;
- }
- if (isAggregateAST(astNode) || isTableColumnAST(astNode) || isNonAggregateFunctionAST(astNode)) {
- if (innerToOuterASTs.containsKey(new HashableASTNode(astNode))) {
- ASTNode ret = innerToOuterASTs.get(new HashableASTNode(astNode));
- // Set parent null for quicker GC
- astNode.setParent(null);
- return ret;
- }
- }
- for (int i = 0; i < astNode.getChildCount(); i++) {
- astNode.setChild(i, replaceAST((ASTNode) astNode.getChild(i)));
- }
- return astNode;
+ private ASTNode processAggregate(ASTNode astNode) {
+ ASTNode innerSelectASTWithoutAlias = MetastoreUtil.copyAST(astNode);
+ ASTNode innerSelectExprAST = new ASTNode(new CommonToken(HiveParser.TOK_SELEXPR));
+ innerSelectExprAST.addChild(innerSelectASTWithoutAlias);
+ String alias = aliasDecider.decideAlias(astNode);
+ ASTNode aliasNode = new ASTNode(new CommonToken(Identifier, alias));
+ innerSelectExprAST.addChild(aliasNode);
+ addToInnerSelectAST(innerSelectExprAST);
+ ASTNode dotAST = getDotAST(query.getCube().getName(), alias);
+ ASTNode outerAST = new ASTNode(new CommonToken(TOK_FUNCTION));
+ //TODO: take care or non-transitive aggregate functions
+ outerAST.addChild(new ASTNode(new CommonToken(Identifier, astNode.getChild(0).getText())));
+ outerAST.addChild(dotAST);
+ innerToOuterASTs.put(new HashableASTNode(innerSelectASTWithoutAlias), outerAST);
+ return outerAST;
}
private void addToInnerSelectAST(ASTNode selectExprAST) {
http://git-wip-us.apache.org/repos/asf/lens/blob/a4b4b85d/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 2ab3fd0..86db011 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -651,6 +651,8 @@ public class CubeTestSetup {
"concat(cubecity.name, \":\", cubestate.name)"));
exprs.add(new ExprColumn(new FieldSchema("cityStateName", "String", "city state"), "City State",
"concat('CityState:', cubecity.statename)"));
+ exprs.add(new ExprColumn(new FieldSchema("isIndia", "String", "is indian city/state"), "Is Indian City/state",
+ "cubecity.name == 'DELHI' OR cubestate.name == 'KARNATAKA' OR cubestate.name == 'MAHARASHTRA'"));
exprs.add(new ExprColumn(new FieldSchema("cubeStateName", "String", "statename from cubestate"), "CubeState Name",
"substr(cubestate.name, 5)"));
exprs.add(new ExprColumn(new FieldSchema("substrdim2big1", "String", "substr of dim2big1"), "dim2big1 substr",
http://git-wip-us.apache.org/repos/asf/lens/blob/a4b4b85d/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java
index 5eea123..f5657e5 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionQueries.java
@@ -209,7 +209,75 @@ public class TestUnionQueries extends TestQueryRewrite {
} finally {
getStorageToUpdatePeriodMap().clear();
}
+ }
+ @Test
+ public void testDimAttrExpressionQuery() throws Exception {
+ Configuration conf = getConf();
+ conf.set(getValidStorageTablesKey("testfact"), "C1_testFact,C2_testFact");
+ conf.set(getValidUpdatePeriodsKey("testfact", "C1"), "DAILY,HOURLY");
+ conf.set(getValidUpdatePeriodsKey("testfact2", "C1"), "YEARLY");
+ conf.set(getValidUpdatePeriodsKey("testfact", "C2"), "MONTHLY,DAILY");
+ ArrayList<String> storages = Lists.newArrayList("c1_testfact", "c2_testfact");
+ getStorageToUpdatePeriodMap().put("c1_testfact", Lists.newArrayList(HOURLY, DAILY));
+ getStorageToUpdatePeriodMap().put("c2_testfact", Lists.newArrayList(MONTHLY));
+ StoragePartitionProvider provider = new StoragePartitionProvider() {
+ @Override
+ public Map<String, String> providePartitionsForStorage(String storage) {
+ return getWhereForMonthlyDailyAndHourly2monthsUnionQuery(storage);
+ }
+ };
+ String hqlQuery = rewrite("select asciicity as `City Name`, cityAndState as citystate, isIndia as isIndia,"
+ + " msr8, msr7 as `Third measure` "
+ + "from testCube where asciicity = 'c' and cityname = 'a' and zipcode = 'b' and "
+ + TWO_MONTHS_RANGE_UPTO_HOURS, conf);
+ String expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider,
+ "SELECT testcube.alias0 as `City Name`, testcube.alias1 as citystate, testcube.alias2 as isIndia, "
+ + "sum(testcube.alias3) + max(testcube.alias4), "
+ + "case when sum(testcube.alias3) = 0 then 0 else sum(testcube.alias5)/sum(testcube.alias3) end "
+ + "as `Third Measure`",
+ null, " group by testcube.alias0, testcube.alias1, testcube.alias2",
+ "select ascii(cubecity.name) as `alias0`, concat(cubecity.name, \":\", cubestate.name) as alias1,"
+ + "cubecity.name == 'DELHI' OR cubestate.name == 'KARNATAKA' OR cubestate.name == 'MAHARASHTRA' as alias2,"
+ + "sum(testcube.msr2) as `alias3`, max(testcube.msr3) as `alias4`, "
+ + "sum(case when testcube.cityid = 'x' then testcube.msr21 else testcube.msr22 end) as `alias5`", " join "
+ + getDbName() + "c1_statetable cubestate on testcube.stateid = cubestate.id and (cubestate.dt = 'latest') join"
+ + getDbName() + "c1_citytable cubecity on testcube.cityid = cubecity.id and (cubecity.dt = 'latest')",
+ "ascii(cubecity.name) = 'c' and cubecity.name = 'a' and testcube.zipcode = 'b'",
+ " group by ascii(cubecity.name)), concat(cubecity.name, \":\", cubestate.name),"
+ + "cubecity.name == 'DELHI' OR cubestate.name == 'KARNATAKA' OR cubestate.name == 'MAHARASHTRA'");
+ compareQueries(hqlQuery, expected);
+ }
+
+ @Test
+ public void testNonAggregateOverAggregateFunction() throws Exception {
+ Configuration conf = getConf();
+ conf.set(getValidStorageTablesKey("testfact"), "C1_testFact,C2_testFact");
+ conf.set(getValidUpdatePeriodsKey("testfact", "C1"), "DAILY,HOURLY");
+ conf.set(getValidUpdatePeriodsKey("testfact2", "C1"), "YEARLY");
+ conf.set(getValidUpdatePeriodsKey("testfact", "C2"), "MONTHLY,DAILY");
+ ArrayList<String> storages = Lists.newArrayList("c1_testfact", "c2_testfact");
+ getStorageToUpdatePeriodMap().put("c1_testfact", Lists.newArrayList(HOURLY, DAILY));
+ getStorageToUpdatePeriodMap().put("c2_testfact", Lists.newArrayList(MONTHLY));
+ StoragePartitionProvider provider = new StoragePartitionProvider() {
+ @Override
+ public Map<String, String> providePartitionsForStorage(String storage) {
+ return getWhereForMonthlyDailyAndHourly2monthsUnionQuery(storage);
+ }
+ };
+ String hqlQuery = rewrite("select cityid as `City ID`, msr3 as `Measure 3`, "
+ + "round(SUM(msr2)) as `Measure 2` from testCube" + " where "
+ + TWO_MONTHS_RANGE_UPTO_HOURS + " group by zipcode having msr4 > 10 order by cityid desc, stateid asc, zipcode "
+ + "asc limit 5",
+ conf);
+ String expected = getExpectedUnionQuery(TEST_CUBE_NAME, storages, provider,
+ "SELECT testcube.alias0 as `City ID`,max(testcube.alias1) as `Measure 3`,round(sum(testcube.alias2)) as "
+ + "`Measure 2` ", null, "group by testcube.alias3 having count(testcube.alias4) > 10 "
+ + "order by testcube.alias0 desc, testcube.alias5 asc, testcube.alias3 asc limit 5",
+ "SELECT testcube.cityid as `alias0`, max(testcube.msr3) as `alias1`, sum(testcube.msr2) as `alias2`, "
+ + "testcube.zipcode as `alias3`, count(testcube .msr4) as `alias4`, (testcube.stateid) as `alias5` FROM ",
+ null, "GROUP BY testcube.zipcode");
+ compareQueries(hqlQuery, expected);
}
@Test