You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2017/03/29 09:54:52 UTC
lens git commit: LENS-1403 : Measures getting repeated in inner
select of union query
Repository: lens
Updated Branches:
refs/heads/lens-1381 363f132d1 -> d45c5384c
LENS-1403 : Measures getting repeated in inner select of union query
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d45c5384
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d45c5384
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d45c5384
Branch: refs/heads/lens-1381
Commit: d45c5384ccab119aa263e3bc4b2c3a6c78f8c993
Parents: 363f132
Author: Sushil Mohanty <su...@gmail.com>
Authored: Wed Mar 29 15:24:04 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Wed Mar 29 15:24:04 2017 +0530
----------------------------------------------------------------------
.../lens/cube/parse/StorageCandidate.java | 2 +
.../lens/cube/parse/UnionQueryWriter.java | 20 +++++---
.../lens/cube/parse/TestBaseCubeQueries.java | 30 +++++------
.../cube/parse/TestUnionAndJoinCandidates.java | 53 ++++++++++++--------
4 files changed, 63 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/d45c5384/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index 17f3af8..628e9aa 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -340,6 +340,8 @@ public class StorageCandidate implements Candidate, CandidateTable {
if (this == cubeql.getPickedCandidate()) {
CandidateUtil.updateFinalAlias(queryAst.getSelectAST(), cubeql);
updateOrderByWithFinalAlias(queryAst.getOrderByAST(), queryAst.getSelectAST());
+ } else {
+ queryAst.setHavingAST(null);
}
return CandidateUtil
.buildHQLString(queryAst.getSelectString(), fromString, whereString, queryAst.getGroupByString(),
http://git-wip-us.apache.org/repos/asf/lens/blob/d45c5384/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
index f9717fa..f2325f1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/UnionQueryWriter.java
@@ -142,7 +142,11 @@ public class UnionQueryWriter {
ASTNode expr = innerToOuterSelectASTs.containsKey(new HQLParser.HashableASTNode(node))
? innerToOuterSelectASTs.get(new HQLParser.HashableASTNode(node))
: innerToOuterHavingASTs.get(new HQLParser.HashableASTNode(node));
- node.getParent().setChild(0, expr);
+ if (node.getChildCount() > 1) {
+ node.replaceChildren(1, 1, expr.getChild(1));
+ } else {
+ node.replaceChildren(0, 0, expr);
+ }
}
}
for (int i = 0; i < node.getChildCount(); i++) {
@@ -191,7 +195,7 @@ public class UnionQueryWriter {
ASTNode outerOrderby = new ASTNode(child);
ASTNode tokNullsChild = (ASTNode) child.getChild(0);
ASTNode outerTokNullsChild = new ASTNode(tokNullsChild);
- outerTokNullsChild.addChild(getOuterAST((ASTNode) tokNullsChild.getChild(0), null, aliasDecider, null, true, cubeql.getBaseCube().getDimAttributeNames()));
+ outerTokNullsChild.addChild(innerToOuterSelectASTs.get(new HQLParser.HashableASTNode((ASTNode) tokNullsChild)));
outerOrderby.addChild(outerTokNullsChild);
outerExpression.addChild(outerOrderby);
}
@@ -299,8 +303,7 @@ public class UnionQueryWriter {
private List<ASTNode> getProjectedNonDefaultPhrases() {
List<ASTNode> phrases = new ArrayList<>();
- int selectPhraseCount = cubeql.getSelectPhrases().size();
- for (int i = 0; i < selectPhraseCount; i++) {
+ for (int i = 0; i < storageCandidates.iterator().next().getQueryAst().getSelectAST().getChildCount(); i++) {
for (StorageCandidate sc : storageCandidates) {
ASTNode selectAST = sc.getQueryAst().getSelectAST();
if (isNodeDefault((ASTNode) selectAST.getChild(i))) {
@@ -359,10 +362,13 @@ public class UnionQueryWriter {
}
}
}
- updateOuterSelectDuplicateAliases(queryAst.getSelectAST(), aliasMap);
+ updateOuterASTDuplicateAliases(queryAst.getSelectAST(), aliasMap);
+ if (queryAst.getHavingAST() != null) {
+ updateOuterASTDuplicateAliases(queryAst.getHavingAST(), aliasMap);
+ }
}
- public void updateOuterSelectDuplicateAliases(ASTNode node,
+ public void updateOuterASTDuplicateAliases(ASTNode node,
Map<String, List<String>> aliasMap) {
if (node.getToken().getType() == HiveParser.DOT) {
String table = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier).toString();
@@ -380,7 +386,7 @@ public class UnionQueryWriter {
}
for (int i = 0; i < node.getChildCount(); i++) {
ASTNode child = (ASTNode) node.getChild(i);
- updateOuterSelectDuplicateAliases(child, aliasMap);
+ updateOuterASTDuplicateAliases(child, aliasMap);
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/d45c5384/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
index 591b24b..ba8a5e4 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
@@ -1024,11 +1024,11 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
+ "having msr12 > 2 and roundedmsr2 > 0 and msr2 > 100", conf);
expected1 = getExpectedQuery(cubeName,
"SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, "
- + "sum(0.0) as `alias3`, sum(0.0) as `alias4` FROM ", null, " group by basecube.dim1, basecube.dim11",
+ + "sum(0.0) as `alias3` FROM ", null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
expected2 = getExpectedQuery(cubeName,
"SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum(0.0) as `alias2`, "
- + "sum((basecube.msr2)) as `alias3`, sum((basecube.msr2)) as `alias4` FROM ", null,
+ + "sum((basecube.msr2)) as `alias3` FROM ", null,
" group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
@@ -1036,8 +1036,8 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
compareContains(expected2, hqlQuery);
assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `dim1`, "
+ "(basecube.alias1) as `dim11` from"), hqlQuery);
- assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("HAVING ((sum((basecube.alias2)) > 2) and "
- + "(round((sum((basecube.alias4)) / 1000)) > 0) and (sum((basecube.alias4)) > 100))"), hqlQuery);
+ assertTrue(hqlQuery.contains("UNION ALL") && hqlQuery.endsWith("HAVING ((sum((basecube.alias2)) > 2) "
+ + "and (round((sum((basecube.alias3)) / 1000)) > 0) and (sum((basecube.alias3)) > 100))"), hqlQuery);
hqlQuery = rewrite("select dim1, dim11 from basecube where " + TWO_DAYS_RANGE
+ "having msr12+roundedmsr2 <= 1000", conf);
@@ -1063,13 +1063,13 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
hqlQuery = rewrite("select dim1, dim11 from basecube where " + TWO_DAYS_RANGE
+ "having msr12 > 2 and roundedmsr2 > 0 and msr12+roundedmsr2 <= 1000", conf);
expected1 = getExpectedQuery(cubeName,
- "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12))"
- + " as `alias2`, sum(0.0) as `alias3`, sum((basecube.msr12)) as `alias4`, sum(0.0) as `alias5` FROM ",
+ "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, "
+ + "sum(0.0) as `alias3` FROM ",
null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
expected2 = getExpectedQuery(cubeName,
- "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum(0.0) as `alias2`, sum((basecube.msr2)) "
- + "as `alias3`, sum(0.0) as `alias4`, sum((basecube.msr2)) as `alias5` FROM ",
+ "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum(0.0) as `alias2`, "
+ + "sum((basecube.msr2)) as `alias3` FROM ",
null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
@@ -1078,20 +1078,20 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `dim1`, (basecube.alias1) "
+ "as `dim11` from "), hqlQuery);
assertTrue(hqlQuery.contains("UNION ALL")
- && hqlQuery.endsWith("HAVING ((sum((basecube.alias4)) > 2) and (round((sum((basecube.alias5)) / 1000)) > 0) "
- + "and ((sum((basecube.alias4)) + round((sum((basecube.alias5)) / 1000))) <= 1000))"), hqlQuery);
+ && hqlQuery.endsWith("HAVING ((sum((basecube.alias2)) > 2) and (round((sum((basecube.alias3)) / 1000)) > 0) "
+ + "and ((sum((basecube.alias2)) + round((sum((basecube.alias3)) / 1000))) <= 1000))"), hqlQuery);
hqlQuery = rewrite("select dim1, dim11 from basecube where " + TWO_DAYS_RANGE
+ "having msr12 > 2 or roundedmsr2 > 0 or msr12+roundedmsr2 <= 1000", conf);
expected1 = getExpectedQuery(cubeName,
"SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum((basecube.msr12)) as `alias2`, "
- + "sum(0.0) as `alias3`, sum((basecube.msr12)) as `alias4`, sum(0.0) as `alias5` FROM ",
+ + "sum(0.0) as `alias3` FROM ",
null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
expected2 = getExpectedQuery(cubeName,
- "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum(0.0) as `alias2`, sum((basecube.msr2)) "
- + "as `alias3`, sum(0.0) as `alias4`, sum((basecube.msr2)) as `alias5` FROM ",
+ "SELECT (basecube.dim1) as `alias0`, (basecube.dim11) as `alias1`, sum(0.0) as `alias2`, "
+ + "sum((basecube.msr2)) as `alias3` FROM ",
null, " group by basecube.dim1, basecube.dim11",
getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
@@ -1100,7 +1100,7 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
assertTrue(hqlQuery.toLowerCase().startsWith("select (basecube.alias0) as `dim1`, (basecube.alias1) "
+ "as `dim11` from"), hqlQuery);
assertTrue(hqlQuery.contains("UNION ALL")
- && hqlQuery.endsWith("HAVING ((sum((basecube.alias4)) > 2) or (round((sum((basecube.alias5)) / 1000)) > 0) or "
- + "((sum((basecube.alias4)) + round((sum((basecube.alias5)) / 1000))) <= 1000))"), hqlQuery);
+ && hqlQuery.endsWith("HAVING ((sum((basecube.alias2)) > 2) or (round((sum((basecube.alias3)) / 1000)) > 0) "
+ + "or ((sum((basecube.alias2)) + round((sum((basecube.alias3)) / 1000))) <= 1000))"), hqlQuery);
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/d45c5384/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
index 931f789..f5f7f3e 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestUnionAndJoinCandidates.java
@@ -25,6 +25,8 @@ import static org.apache.lens.cube.parse.TestCubeRewriter.compareContains;
import static org.testng.Assert.*;
+import jodd.util.StringUtil;
+import org.apache.hadoop.util.StringUtils;
import org.apache.lens.server.api.LensServerAPITestUtil;
import org.apache.lens.server.api.error.LensException;
@@ -36,38 +38,47 @@ import org.testng.annotations.Test;
public class TestUnionAndJoinCandidates extends TestQueryRewrite {
- private Configuration testConf;
+ private Configuration conf;
@BeforeTest
public void setupDriver() throws Exception {
- testConf = LensServerAPITestUtil.getConfiguration(
- DISABLE_AUTO_JOINS, false,
- ENABLE_SELECT_TO_GROUPBY, true,
- ENABLE_GROUP_BY_TO_SELECT, true,
- DISABLE_AGGREGATE_RESOLVER, false,
- ENABLE_STORAGES_UNION, true);
+ conf = LensServerAPITestUtil.getConfigurationWithParams(getConf(),
+ //Supported storage
+ CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C1",
+ // Storage tables
+ getValidStorageTablesKey("union_join_ctx_fact1"), "C1_union_join_ctx_fact1",
+ getValidStorageTablesKey("union_join_ctx_fact2"), "C1_union_join_ctx_fact2",
+ getValidStorageTablesKey("union_join_ctx_fact3"), "C1_union_join_ctx_fact3",
+ // Update periods
+ getValidUpdatePeriodsKey("union_join_ctx_fact1", "C1"), "DAILY",
+ getValidUpdatePeriodsKey("union_join_ctx_fact2", "C1"), "DAILY",
+ getValidUpdatePeriodsKey("union_join_ctx_fact3", "C1"), "DAILY");
+ conf.setBoolean(DISABLE_AUTO_JOINS, false);
+ conf.setBoolean(ENABLE_SELECT_TO_GROUPBY, true);
+ conf.setBoolean(ENABLE_GROUP_BY_TO_SELECT, true);
+ conf.setBoolean(DISABLE_AGGREGATE_RESOLVER, false);
+ conf.setBoolean(ENABLE_STORAGES_UNION, true);
}
@Override
public Configuration getConf() {
- return new Configuration(testConf);
+ return new Configuration();
}
@Test
+ public void testDuplicateProjectedFieldExclusion() throws ParseException, LensException {
+ String colsSelected = " union_join_ctx_cityid , union_join_ctx_msr1_greater_than_100, "
+ + " sum(union_join_ctx_msr1) ";
+ String whereCond = " union_join_ctx_zipcode = 'a' and union_join_ctx_cityid = 'b' and "
+ + "(" + TWO_MONTHS_RANGE_UPTO_DAYS + ")";
+ String rewrittenQuery = rewrite("select " + colsSelected + " from basecube where " + whereCond, conf);
+ assertTrue(rewrittenQuery.contains("UNION ALL"));
+ assertEquals(StringUtil.count(rewrittenQuery, "sum((basecube.union_join_ctx_msr1))"), 2);
+ }
+
+ @Test
public void testFinalCandidateRewrittenQuery() throws ParseException, LensException {
try {
- Configuration conf = LensServerAPITestUtil.getConfigurationWithParams(getConf(),
- //Supported storage
- CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C1",
- // Storage tables
- getValidStorageTablesKey("union_join_ctx_fact1"), "C1_union_join_ctx_fact1",
- getValidStorageTablesKey("union_join_ctx_fact2"), "C1_union_join_ctx_fact2",
- getValidStorageTablesKey("union_join_ctx_fact3"), "C1_union_join_ctx_fact3",
- // Update periods
- getValidUpdatePeriodsKey("union_join_ctx_fact1", "C1"), "DAILY",
- getValidUpdatePeriodsKey("union_join_ctx_fact2", "C1"), "DAILY",
- getValidUpdatePeriodsKey("union_join_ctx_fact3", "C1"), "DAILY");
-
// Query with non projected measure in having clause.
String colsSelected = "union_join_ctx_cityid, sum(union_join_ctx_msr2) ";
String having = " having sum(union_join_ctx_msr1) > 100";
@@ -153,4 +164,6 @@ public class TestUnionAndJoinCandidates extends TestQueryRewrite {
getStorageToUpdatePeriodMap().clear();
}
}
+
+
}