You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2016/05/16 07:11:44 UTC
lens git commit: LENS-1028 : Convert dimension filter to fact filters
for perfomace improvement on outer join queries
Repository: lens
Updated Branches:
refs/heads/master 797899c31 -> bbf2154ee
LENS-1028 : Convert dimension filter to fact filters for perfomace improvement on outer join queries
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/bbf2154e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/bbf2154e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/bbf2154e
Branch: refs/heads/master
Commit: bbf2154ee541907db727249ff96b14dbceec4537
Parents: 797899c
Author: Sushil Mohanty <su...@gmail.com>
Authored: Mon May 16 12:38:59 2016 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Mon May 16 12:38:59 2016 +0530
----------------------------------------------------------------------
.../apache/lens/cube/parse/CandidateFact.java | 15 +-
.../lens/cube/parse/CubeQueryConfUtil.java | 2 +
.../lens/cube/parse/CubeQueryContext.java | 146 ++++++++++++++++++-
.../apache/lens/cube/parse/DefaultQueryAST.java | 22 ++-
.../apache/lens/cube/parse/DimHQLContext.java | 5 +-
.../apache/lens/cube/parse/GroupbyResolver.java | 2 +-
.../lens/cube/parse/MultiFactHQLContext.java | 4 +-
.../org/apache/lens/cube/parse/QueryAST.java | 10 +-
.../lens/cube/parse/join/AutoJoinContext.java | 2 +-
.../lens/cube/parse/TestCubeRewriter.java | 105 +++++++++++++
src/site/apt/user/olap-query-conf.apt | 2 +
11 files changed, 278 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/bbf2154e/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
index cd7a02c..e19e03a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
@@ -82,6 +82,8 @@ public class CandidateFact implements CandidateTable, QueryAST {
@Getter
private final Map<String, ASTNode> storgeWhereClauseMap = new HashMap<>();
@Getter
+ private final Map<String, String> storgeWhereStringMap = new HashMap<>();
+ @Getter
private final Map<TimeRange, Map<String, LinkedHashSet<FactPartition>>> rangeToStoragePartMap = new HashMap<>();
@Getter
private final Map<TimeRange, Map<String, String>> rangeToStorageWhereMap = new HashMap<>();
@@ -164,6 +166,9 @@ public class CandidateFact implements CandidateTable, QueryAST {
public ASTNode getStorageWhereClause(String storageTable) {
return storgeWhereClauseMap.get(storageTable);
}
+ public String getStorageWhereString(String storageTable) {
+ return storgeWhereStringMap.get(storageTable);
+ }
public boolean isExpressionAnswerable(ASTNode node, CubeQueryContext context) throws LensException {
return getColumns().containsAll(HQLParser.getColsInExpr(context.getAliasForTableName(context.getCube()), node));
@@ -274,18 +279,18 @@ public class CandidateFact implements CandidateTable, QueryAST {
return result;
}
- public String getSelectTree() {
+ public String getSelectString() {
return HQLParser.getString(selectAST);
}
- public String getWhereTree() {
+ public String getWhereString() {
if (whereAST != null) {
return HQLParser.getString(whereAST);
}
return null;
}
- public String getHavingTree() {
+ public String getHavingString() {
if (havingAST != null) {
return HQLParser.getString(havingAST);
}
@@ -293,7 +298,7 @@ public class CandidateFact implements CandidateTable, QueryAST {
}
@Override
- public String getOrderByTree() {
+ public String getOrderByString() {
if (orderByAST != null) {
return HQLParser.getString(orderByAST);
}
@@ -314,7 +319,7 @@ public class CandidateFact implements CandidateTable, QueryAST {
return dimFieldIndices;
}
- public String getGroupByTree() {
+ public String getGroupByString() {
if (groupByAST != null) {
return HQLParser.getString(groupByAST);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/bbf2154e/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
index a57292c..49ed5ef 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
@@ -115,4 +115,6 @@ public final class CubeQueryConfUtil {
public static final boolean DEFAULT_DO_FLATTENING_OF_BRIDGE_TABLE_EARLY = false;
public static final String BRIDGE_TABLE_FIELD_ARRAY_FILTER = "lens.cube.query.bridge.table.field.array.filter";
public static final String DEFAULT_BRIDGE_TABLE_FIELD_ARRAY_FILTER = "array_contains";
+ public static final String REWRITE_DIM_FILTER_TO_FACT_FILTER = "lens.cube.query.rewrite.dim.filter.to.fact.filter";
+ public static final boolean DEFAULT_REWRITE_DIM_FILTER_TO_FACT_FILTER = false;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/bbf2154e/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 6f016f2..0fc8549 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -35,8 +35,11 @@ import org.apache.lens.cube.error.LensCubeErrorCode;
import org.apache.lens.cube.error.NoCandidateDimAvailableException;
import org.apache.lens.cube.error.NoCandidateFactAvailableException;
import org.apache.lens.cube.metadata.*;
+import org.apache.lens.cube.metadata.join.TableRelationship;
import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
import org.apache.lens.cube.parse.join.AutoJoinContext;
+import org.apache.lens.cube.parse.join.JoinClause;
+import org.apache.lens.cube.parse.join.JoinTree;
import org.apache.lens.cube.parse.join.JoinUtils;
import org.apache.lens.server.api.error.LensException;
@@ -637,25 +640,25 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST {
}
}
- public String getSelectTree() {
+ public String getSelectString() {
return HQLParser.getString(selectAST);
}
- public String getWhereTree() {
+ public String getWhereString() {
if (whereAST != null) {
return HQLParser.getString(whereAST);
}
return null;
}
- public String getGroupByTree() {
+ public String getGroupByString() {
if (groupByAST != null) {
return HQLParser.getString(groupByAST);
}
return null;
}
- public String getHavingTree() {
+ public String getHavingString() {
if (havingAST != null) {
return HQLParser.getString(havingAST);
}
@@ -670,7 +673,7 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST {
return qb.getQbJoinTree();
}
- public String getOrderByTree() {
+ public String getOrderByString() {
if (orderByAST != null) {
return HQLParser.getString(orderByAST);
}
@@ -884,7 +887,7 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST {
ASTNode rangeAST = HQLParser.parseExpr(rangeWhere);
range.getParent().setChild(range.getChildIndex(), rangeAST);
}
- fact.getStorgeWhereClauseMap().put(table, HQLParser.parseExpr(getWhereTree()));
+ fact.getStorgeWhereClauseMap().put(table, HQLParser.parseExpr(getWhereString()));
}
}
}
@@ -982,9 +985,22 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST {
}
}
}
-
if (cfacts == null || cfacts.size() == 1) {
- updateFromString(cfacts == null? null: cfacts.iterator().next(), dimsToQuery);
+ updateFromString(cfacts == null ? null : cfacts.iterator().next(), dimsToQuery);
+ }
+ //update dim filter with fact filter
+ if (cfacts != null && cfacts.size() > 0) {
+ for (CandidateFact cfact : cfacts) {
+ if (!cfact.getStorageTables().isEmpty()) {
+ for (String qualifiedStorageTable : cfact.getStorageTables()) {
+ String storageTable = qualifiedStorageTable.substring(qualifiedStorageTable.indexOf(".") + 1);
+ String where = getWhere(cfact, autoJoinCtx,
+ cfact.getStorageWhereClause(storageTable), getAliasForTableName(cfact.getBaseTable().getName()),
+ shouldReplaceDimFilterWithFactFilter(), storageTable, dimsToQuery);
+ cfact.getStorgeWhereStringMap().put(storageTable, where);
+ }
+ }
+ }
}
hqlContext = createHQLContext(cfacts, dimsToQuery, factDimMap);
return hqlContext.toHQL();
@@ -1179,6 +1195,10 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST {
return getConf().getBoolean(REPLACE_TIMEDIM_WITH_PART_COL, DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL);
}
+ public boolean shouldReplaceDimFilterWithFactFilter() {
+ return getConf().getBoolean(REWRITE_DIM_FILTER_TO_FACT_FILTER, DEFAULT_REWRITE_DIM_FILTER_TO_FACT_FILTER);
+ }
+
public String getPartitionColumnOfTimeDim(String timeDimName) {
return getPartitionColumnOfTimeDim(cube, timeDimName);
}
@@ -1290,4 +1310,114 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST {
public ImmutableSet<String> getQueriedTimeDimCols() {
return ImmutableSet.copyOf(this.queriedTimeDimCols);
}
+
+ private String getWhere(CandidateFact cfact, AutoJoinContext autoJoinCtx,
+ ASTNode node, String cubeAlias,
+ boolean shouldReplaceDimFilter, String storageTable,
+ Map<Dimension, CandidateDim> dimToQuery) throws LensException {
+ String whereString;
+ if (autoJoinCtx != null && shouldReplaceDimFilter) {
+ List<String> allfilters = new ArrayList<>();
+ getAllFilters(node, cubeAlias, allfilters, autoJoinCtx.getJoinClause(cfact), dimToQuery);
+ whereString = StringUtils.join(allfilters, " and ");
+ } else {
+ whereString = HQLParser.getString(cfact.getStorageWhereClause(storageTable));
+ }
+ return whereString;
+ }
+
+ private List<String> getAllFilters(ASTNode node, String cubeAlias, List<String> allFilters,
+ JoinClause joinClause, Map<Dimension, CandidateDim> dimToQuery)
+ throws LensException {
+
+ if (node.getToken().getType() == HiveParser.KW_AND) {
+ // left child is and
+ if (node.getChild(0).getType() == HiveParser.KW_AND) {
+ // take right corresponding to right
+ String table = getTableFromFilterAST((ASTNode) node.getChild(1));
+ allFilters.add(getFilter(table, cubeAlias, node, joinClause, 1, dimToQuery));
+ } else if (node.getChildCount() > 1) {
+ for (int i = 0; i < node.getChildCount(); i++) {
+ String table = getTableFromFilterAST((ASTNode) node.getChild(i));
+ allFilters.add(getFilter(table, cubeAlias, node, joinClause, i, dimToQuery));
+ }
+ }
+ }
+ for (int i = 0; i < node.getChildCount(); i++) {
+ ASTNode child = (ASTNode) node.getChild(i);
+ getAllFilters(child, cubeAlias, allFilters, joinClause, dimToQuery);
+ }
+ return allFilters;
+ }
+
+ private String getFilter(String table, String cubeAlias, ASTNode node, JoinClause joinClause,
+ int index, Map<Dimension, CandidateDim> dimToQuery)
+ throws LensException{
+ String filter;
+ if (table != null && !table.equals(cubeAlias) && getStarJoin(joinClause, table) != null) {
+ //rewrite dim filter to fact filter if its a star join with fact
+ filter = buildFactSubqueryFromDimFilter(getStarJoin(joinClause, table),
+ (ASTNode) node.getChild(index), table, dimToQuery, cubeAlias);
+ } else {
+ filter = HQLParser.getString((ASTNode) node.getChild(index));
+ }
+ return filter;
+ }
+
+ private TableRelationship getStarJoin(JoinClause joinClause, String table) {
+ TableRelationship rel;
+ for (Map.Entry<TableRelationship, JoinTree> entry : joinClause.getJoinTree().getSubtrees().entrySet()) {
+ if (entry.getValue().getDepthFromRoot() == 1 && table.equals(entry.getValue().getAlias())) {
+ return entry.getKey();
+ }
+ }
+ return null;
+ }
+
+ private String getTableFromFilterAST(ASTNode node) {
+
+ if (node.getToken().getType() == HiveParser.DOT) {
+ return HQLParser.findNodeByPath((ASTNode) node,
+ TOK_TABLE_OR_COL, Identifier).getText();
+ } else {
+ // recurse down
+ for (int i = 0; i < node.getChildCount(); i++) {
+ ASTNode child = (ASTNode) node.getChild(i);
+ String ret = getTableFromFilterAST(child);
+ if (ret != null) {
+ return ret;
+ }
+ }
+ }
+ return null;
+ }
+
+ private String buildFactSubqueryFromDimFilter(TableRelationship tabRelation, ASTNode dimFilter,
+ String dimAlias, Map<Dimension, CandidateDim> dimToQuery,
+ String cubeAlias)
+ throws LensException {
+ StringBuilder builder = new StringBuilder();
+ String storageClause = dimToQuery.get(tabRelation.getToTable()).getWhereClause();
+
+ builder.append(cubeAlias)
+ .append(".")
+ .append(tabRelation.getFromColumn())
+ .append(" in ( ")
+ .append("select ")
+ .append(tabRelation.getToColumn())
+ .append(" from ")
+ .append(dimToQuery.get(tabRelation.getToTable()).getStorageString(dimAlias))
+ .append(" where ")
+ .append(HQLParser.getString((ASTNode) dimFilter));
+ if (storageClause != null) {
+ builder.append(" and ")
+ .append(String.format(storageClause, dimAlias))
+ .append(" ) ");
+ } else {
+ builder.append(" ) ");
+ }
+
+ return builder.toString();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lens/blob/bbf2154e/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java
index 0697e78..c9993f3 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java
@@ -31,26 +31,21 @@ public class DefaultQueryAST implements QueryAST {
private ASTNode selectAST, whereAST, groupByAST, havingAST, joinAST, orderByAST;
private Integer limitValue;
private String fromString;
+ private String whereString;
- public String getSelectTree() {
- return HQLParser.getString(selectAST);
- }
- public String getWhereTree() {
- if (whereAST != null) {
- return HQLParser.getString(whereAST);
- }
- return null;
+ public String getSelectString() {
+ return HQLParser.getString(selectAST);
}
- public String getGroupByTree() {
+ public String getGroupByString() {
if (groupByAST != null) {
return HQLParser.getString(groupByAST);
}
return null;
}
- public String getHavingTree() {
+ public String getHavingString() {
if (havingAST != null) {
return HQLParser.getString(havingAST);
}
@@ -58,7 +53,7 @@ public class DefaultQueryAST implements QueryAST {
}
@Override
- public String getOrderByTree() {
+ public String getOrderByString() {
if (orderByAST != null) {
return HQLParser.getString(orderByAST);
}
@@ -68,8 +63,9 @@ public class DefaultQueryAST implements QueryAST {
public static DefaultQueryAST fromCandidateFact(CandidateFact fact, String storageTable, QueryAST ast) throws
LensException {
return new DefaultQueryAST(ast.getSelectAST(),
- fact.getStorageWhereClause(storageTable.substring(storageTable.indexOf(".") + 1)),
+ null,
ast.getGroupByAST(), ast.getHavingAST(), ast.getJoinAST(), ast.getOrderByAST(), ast.getLimitValue(),
- ast.getFromString());
+ ast.getFromString(),
+ fact.getStorageWhereString(storageTable.substring(storageTable.indexOf(".") + 1)));
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/bbf2154e/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
index 7c14be7..95d6572 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
@@ -47,10 +47,11 @@ abstract class DimHQLContext extends SimpleHQLContext {
}
DimHQLContext(CubeQueryContext query, Map<Dimension, CandidateDim> dimsToQuery,
Set<Dimension> queriedDims, QueryAST ast) throws LensException {
- super(ast.getSelectTree(), ast.getGroupByTree(), ast.getOrderByTree(), ast.getHavingTree(), ast.getLimitValue());
+ super(ast.getSelectString(), ast.getGroupByString(), ast.getOrderByString(),
+ ast.getHavingString(), ast.getLimitValue());
this.query = query;
this.dimsToQuery = dimsToQuery;
- this.where = ast.getWhereTree();
+ this.where = ast.getWhereString();
this.queriedDims = queriedDims;
this.astFromString = ast.getFromString();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/bbf2154e/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
index 9674f73..8beeb9d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
@@ -164,7 +164,7 @@ class GroupbyResolver implements ContextRewriter {
selectExprs.add(s.trim());
}
List<String> groupByExprs = new ArrayList<String>();
- if (cubeql.getGroupByTree() != null) {
+ if (cubeql.getGroupByString() != null) {
String[] gby = getExpressions(cubeql.getGroupByAST(), cubeql).toArray(new String[]{});
for (String g : gby) {
groupByExprs.add(g.trim());
http://git-wip-us.apache.org/repos/asf/lens/blob/bbf2154e/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
index 4d6ce9e..e90da4c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
@@ -72,7 +72,7 @@ class MultiFactHQLContext extends SimpleHQLContext {
}
private String getOrderbyString() {
- return query.getOrderByTree();
+ return query.getOrderByString();
}
private String getHavingString() {
@@ -84,7 +84,7 @@ class MultiFactHQLContext extends SimpleHQLContext {
}
private String getWhereString() {
- return query.getWhereTree();
+ return query.getWhereString();
}
public String toHQL() throws LensException {
http://git-wip-us.apache.org/repos/asf/lens/blob/bbf2154e/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java
index f064dcb..7298604 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java
@@ -22,17 +22,17 @@ import org.apache.hadoop.hive.ql.parse.ASTNode;
public interface QueryAST {
- String getSelectTree();
+ String getSelectString();
String getFromString();
- String getWhereTree();
+ String getWhereString();
- String getHavingTree();
+ String getHavingString();
- String getOrderByTree();
+ String getOrderByString();
- String getGroupByTree();
+ String getGroupByString();
Integer getLimitValue();
http://git-wip-us.apache.org/repos/asf/lens/blob/bbf2154e/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java
index 8b24f70..3d5c5ac 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java
@@ -99,7 +99,7 @@ public class AutoJoinContext {
return autoJoinTarget;
}
- private JoinClause getJoinClause(CandidateFact fact) {
+ public JoinClause getJoinClause(CandidateFact fact) {
if (fact == null || !factClauses.containsKey(fact)) {
return minCostClause;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/bbf2154e/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index f4ef5ca..9053e21 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -752,6 +752,111 @@ public class TestCubeRewriter extends TestQueryRewrite {
}
@Test
+ public void testConvertDimFilterToFactFilterForSingleFact() throws Exception {
+ Configuration conf = getConf();
+ conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false);
+ conf.set(DRIVER_SUPPORTED_STORAGES, "C3");
+ conf.setBoolean(DISABLE_AUTO_JOINS, false);
+ conf.setBoolean(REWRITE_DIM_FILTER_TO_FACT_FILTER, true);
+
+ // filter with =
+ String hql = rewrite(
+ "select cubecountry.name, msr2 from" + " testCube" + " where cubecountry.region = 'asia' and "
+ + TWO_DAYS_RANGE, conf);
+ String filterSubquery = "testcube.countryid in ( select id from TestQueryRewrite.c3_countrytable_partitioned "
+ + "cubecountry where ((cubecountry.region) = 'asia') and (cubecountry.dt = 'latest') )";
+ //assertTrue(hql.contains(filterSubquery));
+
+ // filter with or
+ hql = rewrite(
+ "select cubecountry.name, msr2 from" + " testCube" + " where (cubecountry.region = 'asia' "
+ + "or cubecountry.region = 'europe') and " + TWO_DAYS_RANGE , conf);
+ filterSubquery = "testcube.countryid in ( select id from TestQueryRewrite.c3_countrytable_partitioned "
+ + "cubecountry where (((cubecountry.region) = 'asia') or ((cubecountry.region) = 'europe')) "
+ + "and (cubecountry.dt = 'latest') )";
+ //assertTrue(hql.contains(filterSubquery));
+
+ //filter with in
+ hql = rewrite(
+ "select cubecountry.name, msr2 from" + " testCube" + " where cubecountry.region in ('asia','europe') "
+ + "and " + TWO_DAYS_RANGE , conf);
+ filterSubquery = "testcube.countryid in ( select id from TestQueryRewrite.c3_countrytable_partitioned "
+ + "cubecountry where (cubecountry.region) in ('asia' , 'europe') and (cubecountry.dt = 'latest') )";
+ //assertTrue(hql.contains(filterSubquery));
+
+ //filter with not in
+ hql = rewrite(
+ "select cubecountry.name, msr2 from" + " testCube" + " where cubecountry.region not in ('asia','europe') "
+ + "and " + TWO_DAYS_RANGE , conf);
+ filterSubquery = "testcube.countryid in ( select id from TestQueryRewrite.c3_countrytable_partitioned "
+ + "cubecountry where (cubecountry.region) not in ('asia' , 'europe') and (cubecountry.dt = 'latest') )";
+ //assertTrue(hql.contains(filterSubquery));
+
+ //filter with !=
+ hql = rewrite(
+ "select cubecountry.name, msr2 from" + " testCube" + " where cubecountry.region != 'asia' "
+ + "and " + TWO_DAYS_RANGE , conf);
+ filterSubquery = "testcube.countryid in ( select id from TestQueryRewrite.c3_countrytable_partitioned "
+ + "cubecountry where ((cubecountry.region) != 'asia') and (cubecountry.dt = 'latest') )";
+ //assertTrue(hql.contains(filterSubquery));
+
+ //filter with cube alias
+ hql = rewrite(
+ "select cubecountry.name, msr2 from" + " testCube as t" + " where cubecountry.region = 'asia' "
+ + "and zipcode = 'x' and " + TWO_DAYS_RANGE , conf);
+ filterSubquery = "t.countryid in ( select id from TestQueryRewrite.c3_countrytable_partitioned "
+ + "cubecountry where ((cubecountry.region) = 'asia') and (cubecountry.dt = 'latest') )";
+ //assertTrue(hql.contains(filterSubquery));
+ }
+
+ @Test
+ public void testConvertDimFilterToFactFilterForMultiFact() throws Exception {
+ Configuration conf = getConf();
+ conf.set(getValidStorageTablesKey("testfact"), "C1_testFact,C2_testFact");
+ conf.set(getValidUpdatePeriodsKey("testfact", "C1"), "DAILY,HOURLY");
+ conf.set(getValidUpdatePeriodsKey("testfact2", "C1"), "YEARLY");
+ conf.set(getValidUpdatePeriodsKey("testfact", "C2"), "MONTHLY,DAILY");
+ conf.setBoolean(DISABLE_AUTO_JOINS, false);
+ conf.setBoolean(REWRITE_DIM_FILTER_TO_FACT_FILTER, true);
+ ArrayList<String> storages = Lists.newArrayList("c1_testfact", "c2_testfact");
+ try {
+ getStorageToUpdatePeriodMap().put("c1_testfact", Lists.newArrayList(HOURLY, DAILY));
+ getStorageToUpdatePeriodMap().put("c2_testfact", Lists.newArrayList(MONTHLY));
+
+ // Union query
+ String hqlQuery;
+ StoragePartitionProvider provider = new StoragePartitionProvider() {
+ @Override
+ public Map<String, String> providePartitionsForStorage(String storage) {
+ return getWhereForMonthlyDailyAndHourly2monthsUnionQuery(storage);
+ }
+ };
+ try {
+ rewrite("select cityid as `City ID`, msr8, msr7 as `Third measure` "
+ + "from testCube where " + TWO_MONTHS_RANGE_UPTO_HOURS, conf);
+ fail("Union feature is disabled, should have failed");
+ } catch (LensException e) {
+ assertEquals(e.getErrorCode(), LensCubeErrorCode.STORAGE_UNION_DISABLED.getLensErrorInfo().getErrorCode());
+ }
+ conf.setBoolean(CubeQueryConfUtil.ENABLE_STORAGES_UNION, true);
+
+ hqlQuery = rewrite("select asciicity as `City Name`, msr8, msr7 as `Third measure` "
+ + "from testCube where asciicity = 'c' and cityname = 'a' and zipcode = 'b' and "
+ + TWO_MONTHS_RANGE_UPTO_HOURS, conf);
+
+ String filter1 = "testcube.cityid in ( select id from TestQueryRewrite.c1_citytable cubecity "
+ + "where (ascii((cubecity.name)) = 'c') and (cubecity.dt = 'latest') )";
+ String filter2 = "testcube.cityid in ( select id from TestQueryRewrite.c1_citytable cubecity "
+ + "where ((cubecity.name) = 'a') and (cubecity.dt = 'latest') )";
+
+ assertTrue(hqlQuery.contains(filter1));
+ assertTrue(hqlQuery.contains(filter2));
+
+ } finally {
+ getStorageToUpdatePeriodMap().clear();
+ }
+ }
+ @Test
public void testCubeGroupbyWithConstantProjected() throws Exception {
// check constants
Configuration conf = getConf();
http://git-wip-us.apache.org/repos/asf/lens/blob/bbf2154e/src/site/apt/user/olap-query-conf.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/user/olap-query-conf.apt b/src/site/apt/user/olap-query-conf.apt
index 6372fb1..8533ca7 100644
--- a/src/site/apt/user/olap-query-conf.apt
+++ b/src/site/apt/user/olap-query-conf.apt
@@ -78,4 +78,6 @@ OLAP query configuration
*--+--+---+--+
|25|lens.cube.query.valid.fact.${facttable}.storagetables| |List of comma separated storage tables that are valid for a fact. If no value is specified, all storage tables are valid|
*--+--+---+--+
+|26|lens.cube.query.rewrite.dim.filter.to.fact.filter|false|Falg specifies if dimension filter has to be rewritten as fact filter. for eg. where dim.name in ('x', 'y') will become where fact.dimid in (select dim.id from dim where dim.name in ('x','y'))|
+*--+--+---+--+
The configuration parameters and their default values