You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by sh...@apache.org on 2015/12/30 08:11:06 UTC
[47/50] [abbrv] lens git commit: LENS-552: Union support across
storage tables in multi fact query
LENS-552: Union support across storage tables in multi fact query
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/04f5a822
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/04f5a822
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/04f5a822
Branch: refs/heads/LENS-581
Commit: 04f5a8223f652baa5cfbebec7f8f9a2886df5076
Parents: bf1053b
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Thu Dec 24 12:39:33 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Dec 24 12:39:33 2015 +0530
----------------------------------------------------------------------
.../lens/cube/parse/AggregateResolver.java | 12 +-
.../apache/lens/cube/parse/AliasReplacer.java | 5 +-
.../apache/lens/cube/parse/AutoJoinContext.java | 2 +-
.../apache/lens/cube/parse/CandidateFact.java | 130 ++++---------------
.../apache/lens/cube/parse/ColumnResolver.java | 2 +-
.../lens/cube/parse/CubeQueryContext.java | 69 +++++-----
.../lens/cube/parse/CubeQueryRewriter.java | 2 +-
.../apache/lens/cube/parse/DefaultQueryAST.java | 74 +++++++++++
.../cube/parse/DenormalizationResolver.java | 18 +--
.../apache/lens/cube/parse/DimHQLContext.java | 6 +-
.../lens/cube/parse/DimOnlyHQLContext.java | 15 +--
.../lens/cube/parse/ExpressionResolver.java | 24 ++--
.../apache/lens/cube/parse/FactHQLContext.java | 65 ----------
.../apache/lens/cube/parse/GroupbyResolver.java | 2 -
.../org/apache/lens/cube/parse/HQLParser.java | 1 -
.../apache/lens/cube/parse/JoinResolver.java | 17 +--
.../lens/cube/parse/MultiFactHQLContext.java | 46 +++----
.../org/apache/lens/cube/parse/QueryAST.java | 86 ++++++++++++
.../lens/cube/parse/SingleFactHQLContext.java | 96 --------------
.../parse/SingleFactMultiStorageHQLContext.java | 52 +++++---
.../SingleFactSingleStorageHQLContext.java | 85 ++++++++++++
.../lens/cube/parse/StorageTableResolver.java | 43 +++---
.../apache/lens/cube/metadata/DateFactory.java | 11 ++
.../apache/lens/cube/parse/CubeTestSetup.java | 2 +-
.../lens/cube/parse/TestBaseCubeQueries.java | 24 ++--
.../lens/cube/parse/TestCubeRewriter.java | 53 ++++++--
26 files changed, 481 insertions(+), 461 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
index 39bd1cc..fd7036a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
@@ -47,8 +47,6 @@ import lombok.extern.slf4j.Slf4j;
*/
@Slf4j
class AggregateResolver implements ContextRewriter {
- public AggregateResolver(Configuration conf) {
- }
@Override
public void rewriteContext(CubeQueryContext cubeql) throws LensException {
@@ -166,7 +164,7 @@ class AggregateResolver implements ContextRewriter {
String colname;
if (node.getToken().getType() == HiveParser.TOK_TABLE_OR_COL) {
- colname = ((ASTNode) node.getChild(0)).getText();
+ colname = node.getChild(0).getText();
} else {
// node in 'alias.column' format
ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier);
@@ -193,15 +191,9 @@ class AggregateResolver implements ContextRewriter {
throw new LensException(LensCubeErrorCode.NO_DEFAULT_AGGREGATE.getLensErrorInfo(), colname);
}
ASTNode fnroot = new ASTNode(new CommonToken(HiveParser.TOK_FUNCTION));
- fnroot.setParent(node.getParent());
-
ASTNode fnIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, aggregateFn));
- fnIdentNode.setParent(fnroot);
fnroot.addChild(fnIdentNode);
-
- node.setParent(fnroot);
fnroot.addChild(node);
-
return fnroot;
}
} else {
@@ -224,7 +216,7 @@ class AggregateResolver implements ContextRewriter {
String colname;
if (node.getToken().getType() == HiveParser.TOK_TABLE_OR_COL) {
- colname = ((ASTNode) node.getChild(0)).getText();
+ colname = node.getChild(0).getText();
} else {
// node in 'alias.column' format
ASTNode colIdent = (ASTNode) node.getChild(1);
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
index 0656049..e629731 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
@@ -81,7 +81,7 @@ class AliasReplacer implements ContextRewriter {
replaceAliases(cubeql.getWhereAST(), 0, colToTableAlias);
- replaceAliases(cubeql.getJoinTree(), 0, colToTableAlias);
+ replaceAliases(cubeql.getJoinAST(), 0, colToTableAlias);
// Update the aggregate expression set
AggregateResolver.updateAggregates(cubeql.getSelectAST(), cubeql);
@@ -183,7 +183,6 @@ class AliasReplacer implements ContextRewriter {
ASTNode aliasNode = (ASTNode) node.getChild(0);
ASTNode newAliasIdent = new ASTNode(new CommonToken(HiveParser.Identifier, newAlias));
aliasNode.setChild(0, newAliasIdent);
- newAliasIdent.setParent(aliasNode);
} else {
// Just a column ref, we need to make it alias.col
// '.' will become the parent node
@@ -192,9 +191,7 @@ class AliasReplacer implements ContextRewriter {
ASTNode tabRefNode = new ASTNode(new CommonToken(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL"));
tabRefNode.addChild(aliasIdentNode);
- aliasIdentNode.setParent(tabRefNode);
dot.addChild(tabRefNode);
- tabRefNode.setParent(dot);
ASTNode colIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, colName));
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/AutoJoinContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AutoJoinContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AutoJoinContext.java
index 9472506..7f13c6c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AutoJoinContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AutoJoinContext.java
@@ -101,7 +101,7 @@ public class AutoJoinContext {
}
private JoinClause getJoinClause(CandidateFact fact) {
- if (fact == null) {
+ if (fact == null || !factClauses.containsKey(fact)) {
return minCostClause;
}
return factClauses.get(fact);
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
index 2338ba7..c305244 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
@@ -37,14 +37,14 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.antlr.runtime.CommonToken;
import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import lombok.Getter;
+import lombok.Setter;
/**
* Holds context of a candidate fact table.
*/
-public class CandidateFact implements CandidateTable {
+public class CandidateFact implements CandidateTable, QueryAST {
final CubeFactTable fact;
@Getter
private Set<String> storageTables;
@@ -52,27 +52,32 @@ public class CandidateFact implements CandidateTable {
private int numQueriedParts = 0;
@Getter
private final Set<FactPartition> partsQueried = Sets.newHashSet();
- @Getter
- private final Map<TimeRange, String> rangeToWhereClause = Maps.newHashMap();
private CubeInterface baseTable;
+ @Getter @Setter
private ASTNode selectAST;
+ @Getter @Setter
private ASTNode whereAST;
- private ASTNode groupbyAST;
+ @Getter @Setter
+ private ASTNode groupByAST;
+ @Getter @Setter
private ASTNode havingAST;
- private ASTNode joinTree;
+ @Getter @Setter
+ private ASTNode joinAST;
+ @Getter @Setter
+ private ASTNode orderByAST;
+ @Getter @Setter
+ private Integer limitValue;
private List<TimeRangeNode> timenodes = Lists.newArrayList();
private final List<Integer> selectIndices = Lists.newArrayList();
private final List<Integer> dimFieldIndices = Lists.newArrayList();
private Collection<String> columns;
@Getter
- private final Map<String, String> storgeWhereClauseMap = new HashMap<String, String>();
+ private final Map<String, String> storgeWhereClauseMap = new HashMap<>();
@Getter
- private final Map<TimeRange, Map<String, LinkedHashSet<FactPartition>>> rangeToStoragePartMap =
- new HashMap<TimeRange, Map<String, LinkedHashSet<FactPartition>>>();
+ private final Map<TimeRange, Map<String, LinkedHashSet<FactPartition>>> rangeToStoragePartMap = new HashMap<>();
@Getter
- private final Map<TimeRange, Map<String, String>> rangeToStorageWhereMap =
- new HashMap<TimeRange, Map<String, String>>();
+ private final Map<TimeRange, Map<String, String>> rangeToStorageWhereMap = new HashMap<>();
CandidateFact(CubeFactTable fact, CubeInterface cube) {
this.fact = fact;
@@ -114,57 +119,25 @@ public class CandidateFact implements CandidateTable {
numQueriedParts += incr;
}
- private void updateTimeRanges(ASTNode root, ASTNode parent, int childIndex) throws LensException {
- if (root == null) {
- return;
- } else if (root.getToken().getType() == TOK_FUNCTION) {
- ASTNode fname = HQLParser.findNodeByPath(root, Identifier);
- if (fname != null && CubeQueryContext.TIME_RANGE_FUNC.equalsIgnoreCase(fname.getText())) {
- timenodes.add(new TimeRangeNode(root, parent, childIndex));
- }
- } else {
- for (int i = 0; i < root.getChildCount(); i++) {
- ASTNode child = (ASTNode) root.getChild(i);
- updateTimeRanges(child, root, i);
- }
- }
- }
-
// copy ASTs from CubeQueryContext
public void copyASTs(CubeQueryContext cubeql) throws LensException {
- this.selectAST = HQLParser.copyAST(cubeql.getSelectAST());
- this.whereAST = HQLParser.copyAST(cubeql.getWhereAST());
- if (cubeql.getJoinTree() != null) {
- this.joinTree = HQLParser.copyAST(cubeql.getJoinTree());
+ setSelectAST(HQLParser.copyAST(cubeql.getSelectAST()));
+ setWhereAST(HQLParser.copyAST(cubeql.getWhereAST()));
+ if (cubeql.getJoinAST() != null) {
+ setJoinAST(HQLParser.copyAST(cubeql.getJoinAST()));
}
if (cubeql.getGroupByAST() != null) {
- this.groupbyAST = HQLParser.copyAST(cubeql.getGroupByAST());
+ setGroupByAST(HQLParser.copyAST(cubeql.getGroupByAST()));
}
if (cubeql.getHavingAST() != null) {
- this.havingAST = HQLParser.copyAST(cubeql.getHavingAST());
+ setHavingAST(HQLParser.copyAST(cubeql.getHavingAST()));
}
- // copy timeranges
- updateTimeRanges(this.whereAST, null, 0);
}
public String getWhereClause(String storageTable) {
return getStorgeWhereClauseMap().get(storageTable);
}
- public void updateTimeranges(CubeQueryContext cubeql) throws LensException {
- // Update WhereAST with range clause
- // resolve timerange positions and replace it by corresponding where clause
- for (int i = 0; i < cubeql.getTimeRanges().size(); i++) {
- TimeRange range = cubeql.getTimeRanges().get(i);
- String rangeWhere = rangeToWhereClause.get(range);
- if (!StringUtils.isBlank(rangeWhere)) {
- ASTNode rangeAST = HQLParser.parseExpr(rangeWhere);
- rangeAST.setParent(timenodes.get(i).parent);
- timenodes.get(i).parent.setChild(timenodes.get(i).childIndex, rangeAST);
- }
- }
- }
-
/**
* Update the ASTs to include only the fields queried from this fact, in all the expressions
*
@@ -322,54 +295,15 @@ public class CandidateFact implements CandidateTable {
return null;
}
- public String getGroupbyTree() {
- if (groupbyAST != null) {
- return HQLParser.getString(groupbyAST);
+ @Override
+ public String getOrderByTree() {
+ if (orderByAST != null) {
+ return HQLParser.getString(orderByAST);
}
return null;
}
- /**
- * @return the selectAST
- */
- public ASTNode getSelectAST() {
- return selectAST;
- }
-
- /**
- * @param selectAST the selectAST to set
- */
- public void setSelectAST(ASTNode selectAST) {
- this.selectAST = selectAST;
- }
-
- /**
- * @return the whereAST
- */
- public ASTNode getWhereAST() {
- return whereAST;
- }
- /**
- * @param whereAST the whereAST to set
- */
- public void setWhereAST(ASTNode whereAST) {
- this.whereAST = whereAST;
- }
-
- /**
- * @return the havingAST
- */
- public ASTNode getHavingAST() {
- return havingAST;
- }
-
- /**
- * @param havingAST the havingAST to set
- */
- public void setHavingAST(ASTNode havingAST) {
- this.havingAST = havingAST;
- }
/**
* @return the selectIndices
@@ -385,13 +319,9 @@ public class CandidateFact implements CandidateTable {
return dimFieldIndices;
}
- public ASTNode getGroupByAST() {
- return groupbyAST;
- }
-
public String getGroupByTree() {
- if (groupbyAST != null) {
- return HQLParser.getString(groupbyAST);
+ if (groupByAST != null) {
+ return HQLParser.getString(groupByAST);
}
return null;
}
@@ -413,8 +343,4 @@ public class CandidateFact implements CandidateTable {
}
return timePartDimensions;
}
-
- public ASTNode getJoinTree() {
- return joinTree;
- }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java
index b95595a..75aa3f4 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java
@@ -61,7 +61,7 @@ class ColumnResolver implements ContextRewriter {
}
getColsForSelectTree(cubeql);
getColsForWhereTree(cubeql);
- getColsForTree(cubeql, cubeql.getJoinTree(), cubeql);
+ getColsForTree(cubeql, cubeql.getJoinAST(), cubeql);
getColsForTree(cubeql, cubeql.getGroupByAST(), cubeql);
getColsForTree(cubeql, cubeql.getHavingAST(), cubeql);
getColsForTree(cubeql, cubeql.getOrderByAST(), cubeql);
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 4034a54..1fd1d17 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -21,9 +21,7 @@ package org.apache.lens.cube.parse;
import static org.apache.lens.cube.parse.CubeQueryConfUtil.*;
-import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier;
-import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TABLE_OR_COL;
-import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TMP_FILE;
+import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
import static com.google.common.base.Preconditions.checkArgument;
@@ -50,16 +48,11 @@ import org.codehaus.jackson.map.ObjectMapper;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.Getter;
-import lombok.Setter;
-import lombok.ToString;
+import lombok.*;
import lombok.extern.slf4j.Slf4j;
@Slf4j
-public class CubeQueryContext implements TrackQueriedColumns {
+public class CubeQueryContext implements TrackQueriedColumns, QueryAST {
public static final String TIME_RANGE_FUNC = "time_range_in";
public static final String NOW = "now";
public static final String DEFAULT_TABLE = "_default_";
@@ -653,7 +646,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
return null;
}
- public ASTNode getJoinTree() {
+ public ASTNode getJoinAST() {
return qb.getParseInfo().getJoinExpr();
}
@@ -688,8 +681,8 @@ public class CubeQueryContext implements TrackQueriedColumns {
}
String getQBFromString(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery) throws LensException {
- String fromString = null;
- if (getJoinTree() == null) {
+ String fromString;
+ if (getJoinAST() == null) {
if (cube != null) {
fromString = fact.getStorageString(getAliasForTableName(cube.getName()));
} else {
@@ -858,6 +851,23 @@ public class CubeQueryContext implements TrackQueriedColumns {
@Getter private Collection<CandidateFact> pickedFacts;
@Getter private Collection<CandidateDim> pickedDimTables;
+ private void addRangeClauses(CandidateFact fact) throws LensException {
+ if (fact != null) {
+ // resolve timerange positions and replace it by corresponding where clause
+ for (TimeRange range : getTimeRanges()) {
+ for (Map.Entry<String, String> entry : fact.getRangeToStorageWhereMap().get(range).entrySet()) {
+ String table = entry.getKey();
+ String rangeWhere = entry.getValue();
+ if (!StringUtils.isBlank(rangeWhere)) {
+ ASTNode rangeAST = HQLParser.parseExpr(rangeWhere);
+ range.getParent().setChild(range.getChildIndex(), rangeAST);
+ }
+ fact.getStorgeWhereClauseMap().put(table, getWhereTree());
+ }
+ }
+ }
+ }
+
public String toHQL() throws LensException {
Set<CandidateFact> cfacts = pickCandidateFactToQuery();
Map<Dimension, CandidateDim> dimsToQuery = pickCandidateDimsToQuery(dimensions);
@@ -872,11 +882,13 @@ public class CubeQueryContext implements TrackQueriedColumns {
// copy ASTs for each fact
for (CandidateFact cfact : cfacts) {
cfact.copyASTs(this);
- cfact.updateTimeranges(this);
- factDimMap.put(cfact, new HashSet<Dimension>(dimsToQuery.keySet()));
+ factDimMap.put(cfact, new HashSet<>(dimsToQuery.keySet()));
}
- } else {
- SingleFactHQLContext.addRangeClauses(this, cfacts.iterator().next());
+ }
+ }
+ if (cfacts != null) {
+ for (CandidateFact fact : cfacts) {
+ addRangeClauses(fact);
}
}
@@ -884,7 +896,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
Set<Dimension> exprDimensions = new HashSet<Dimension>();
if (cfacts != null) {
for (CandidateFact cfact : cfacts) {
- Set<Dimension> factExprDimTables = exprCtx.rewriteExprCtx(cfact, dimsToQuery, cfacts.size() > 1);
+ Set<Dimension> factExprDimTables = exprCtx.rewriteExprCtx(cfact, dimsToQuery, cfacts.size() > 1 ? cfact : this);
exprDimensions.addAll(factExprDimTables);
if (cfacts.size() > 1) {
factDimMap.get(cfact).addAll(factExprDimTables);
@@ -892,7 +904,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
}
} else {
// dim only query
- exprDimensions.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, false));
+ exprDimensions.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, this));
}
dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimensions));
@@ -940,25 +952,22 @@ public class CubeQueryContext implements TrackQueriedColumns {
}
}
}
- hqlContext = createHQLContext(cfacts, dimsToQuery, factDimMap, this);
+ hqlContext = createHQLContext(cfacts, dimsToQuery, factDimMap);
return hqlContext.toHQL();
}
private HQLContextInterface createHQLContext(Set<CandidateFact> facts, Map<Dimension, CandidateDim> dimsToQuery,
- Map<CandidateFact, Set<Dimension>> factDimMap, CubeQueryContext query) throws LensException {
+ Map<CandidateFact, Set<Dimension>> factDimMap) throws LensException {
if (facts == null || facts.size() == 0) {
- return new DimOnlyHQLContext(dimsToQuery, query);
+ return new DimOnlyHQLContext(dimsToQuery, this, this);
} else if (facts.size() == 1 && facts.iterator().next().getStorageTables().size() > 1) {
//create single fact with multiple storage context
- if (!conf.getBoolean(ENABLE_STORAGES_UNION, DEFAULT_ENABLE_STORAGES_UNION)) {
- throw new LensException(LensCubeErrorCode.STORAGE_UNION_DISABLED.getLensErrorInfo());
- }
- return new SingleFactMultiStorageHQLContext(facts.iterator().next(), dimsToQuery, query);
+ return new SingleFactMultiStorageHQLContext(facts.iterator().next(), dimsToQuery, this, this);
} else if (facts.size() == 1 && facts.iterator().next().getStorageTables().size() == 1) {
// create single fact context
- return new SingleFactHQLContext(facts.iterator().next(), dimsToQuery, query);
+ return new SingleFactSingleStorageHQLContext(facts.iterator().next(), dimsToQuery, this, this);
} else {
- return new MultiFactHQLContext(facts, dimsToQuery, factDimMap, query);
+ return new MultiFactHQLContext(facts, dimsToQuery, factDimMap, this);
}
}
@@ -979,10 +988,6 @@ public class CubeQueryContext implements TrackQueriedColumns {
return tblAliasToColumns.get(getAliasForTableName(tblName));
}
- public void addColumnsQueried(AbstractCubeTable table, String column) {
- addColumnsQueried(getAliasForTableName(table.getName()), column);
- }
-
public void addColumnsQueriedWithTimeDimCheck(String alias, String timeDimColumn) {
if (!shouldReplaceTimeDimWithPart()) {
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
index e0759b0..c1fd0a5 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
@@ -148,7 +148,7 @@ public class CubeQueryRewriter {
// Resolve candidate fact tables and dimension tables for columns queried
rewriters.add(candidateTblResolver);
// Resolve aggregations and generate base select tree
- rewriters.add(new AggregateResolver(conf));
+ rewriters.add(new AggregateResolver());
rewriters.add(new GroupbyResolver(conf));
rewriters.add(new FieldValidator());
// Resolve joins and generate base join tree
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java
new file mode 100644
index 0000000..0997f37
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.parse;
+
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class DefaultQueryAST implements QueryAST {
+ private ASTNode selectAST, whereAST, groupByAST, havingAST, joinAST, orderByAST;
+ private Integer limitValue;
+
+ public String getSelectTree() {
+ return HQLParser.getString(selectAST);
+ }
+
+ public String getWhereTree() {
+ if (whereAST != null) {
+ return HQLParser.getString(whereAST);
+ }
+ return null;
+ }
+
+ public String getGroupByTree() {
+ if (groupByAST != null) {
+ return HQLParser.getString(groupByAST);
+ }
+ return null;
+ }
+
+
+ public String getHavingTree() {
+ if (havingAST != null) {
+ return HQLParser.getString(havingAST);
+ }
+ return null;
+ }
+
+ @Override
+ public String getOrderByTree() {
+ if (orderByAST != null) {
+ return HQLParser.getString(orderByAST);
+ }
+ return null;
+ }
+
+ public static DefaultQueryAST fromCandidateFact(CandidateFact fact, String storageTable, QueryAST ast) throws
+ LensException {
+ return new DefaultQueryAST(ast.getSelectAST(),
+ HQLParser.parseExpr(fact.getWhereClause(storageTable.substring(storageTable.indexOf(".") + 1))),
+ ast.getGroupByAST(), ast.getHavingAST(), ast.getJoinAST(), ast.getOrderByAST(), ast.getLimitValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
index a576f3a..5c8bd84 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
@@ -281,19 +281,15 @@ public class DenormalizationResolver implements ContextRewriter {
}
private void replaceReferencedColumns(CandidateFact cfact, boolean replaceFact) throws LensException {
+ QueryAST ast = cubeql;
if (replaceFact
&& (tableToRefCols.get(cfact.getName()) != null && !tableToRefCols.get(cfact.getName()).isEmpty())) {
- resolveClause(cubeql, cfact.getSelectAST());
- resolveClause(cubeql, cfact.getWhereAST());
- resolveClause(cubeql, cfact.getGroupByAST());
- resolveClause(cubeql, cfact.getHavingAST());
- } else {
- resolveClause(cubeql, cubeql.getSelectAST());
- resolveClause(cubeql, cubeql.getWhereAST());
- resolveClause(cubeql, cubeql.getGroupByAST());
- resolveClause(cubeql, cubeql.getHavingAST());
-
+ ast = cfact;
}
+ resolveClause(cubeql, ast.getSelectAST());
+ resolveClause(cubeql, ast.getWhereAST());
+ resolveClause(cubeql, ast.getGroupByAST());
+ resolveClause(cubeql, ast.getHavingAST());
resolveClause(cubeql, cubeql.getOrderByAST());
}
@@ -320,11 +316,9 @@ public class DenormalizationResolver implements ContextRewriter {
ASTNode newTableNode =
new ASTNode(new CommonToken(HiveParser.Identifier, query.getAliasForTableName(refered.getDestTable())));
tableNode.setChild(0, newTableNode);
- newTableNode.setParent(tableNode);
ASTNode newColumnNode = new ASTNode(new CommonToken(HiveParser.Identifier, refered.getRefColumn()));
node.setChild(1, newColumnNode);
- newColumnNode.setParent(node);
} else {
// recurse down
for (int i = 0; i < node.getChildCount(); i++) {
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
index bcfc1f6..b253b94 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
@@ -44,7 +44,11 @@ abstract class DimHQLContext extends SimpleHQLContext {
public CubeQueryContext getQuery() {
return query;
}
-
+ DimHQLContext(CubeQueryContext query, Map<Dimension, CandidateDim> dimsToQuery,
+ Set<Dimension> queriedDims, QueryAST ast) throws LensException {
+ this(query, dimsToQuery, queriedDims, ast.getSelectTree(), ast.getWhereTree(), ast.getGroupByTree(),
+ ast.getOrderByTree(), ast.getHavingTree(), ast.getLimitValue());
+ }
DimHQLContext(CubeQueryContext query, Map<Dimension, CandidateDim> dimsToQuery,
Set<Dimension> queriedDims, String select, String where,
String groupby, String orderby, String having, Integer limit) throws LensException {
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java
index 0c43d98..d22287b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java
@@ -32,16 +32,15 @@ import org.apache.lens.server.api.error.LensException;
*/
class DimOnlyHQLContext extends DimHQLContext {
- DimOnlyHQLContext(Map<Dimension, CandidateDim> dimsToQuery, CubeQueryContext query) throws LensException {
- super(query, dimsToQuery, dimsToQuery.keySet(), query.getSelectTree(),
- query.getWhereTree(), query.getGroupByTree(), query.getOrderByTree(),
- query.getHavingTree(), query.getLimitValue());
+ DimOnlyHQLContext(Map<Dimension, CandidateDim> dimsToQuery, CubeQueryContext query, QueryAST ast)
+ throws LensException {
+ this(dimsToQuery, dimsToQuery.keySet(), query, ast);
}
- DimOnlyHQLContext(Map<Dimension, CandidateDim> dimsToQuery, CubeQueryContext query, String whereClause)
+ DimOnlyHQLContext(Map<Dimension, CandidateDim> dimsToQuery, Set<Dimension> dimsQueried,
+ CubeQueryContext query, QueryAST ast)
throws LensException {
- super(query, dimsToQuery, dimsToQuery.keySet(), query.getSelectTree(), whereClause, query.getGroupByTree(), query
- .getOrderByTree(), query.getHavingTree(), query.getLimitValue());
+ super(query, dimsToQuery, dimsQueried, ast);
}
public String toHQL() throws LensException {
@@ -49,7 +48,7 @@ class DimOnlyHQLContext extends DimHQLContext {
}
protected String getFromTable() throws LensException {
- if (query.getAutoJoinCtx() != null && query.getAutoJoinCtx().isJoinsResolved()) {
+ if (query.isAutoJoinResolved()) {
return getDimsToQuery().get(query.getAutoJoinCtx().getAutoJoinTarget()).getStorageString(
query.getAliasForTableName(query.getAutoJoinCtx().getAutoJoinTarget().getName()));
} else {
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 776021d..26514d8 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -432,7 +432,7 @@ class ExpressionResolver implements ContextRewriter {
}
public Set<Dimension> rewriteExprCtx(CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery,
- boolean replaceFact) throws LensException {
+ QueryAST queryAST) throws LensException {
Set<Dimension> exprDims = new HashSet<Dimension>();
if (!allExprsQueried.isEmpty()) {
// pick expressions for fact
@@ -446,7 +446,7 @@ class ExpressionResolver implements ContextRewriter {
}
}
// Replace picked expressions in all the base trees
- replacePickedExpressions(cfact, replaceFact);
+ replacePickedExpressions(queryAST);
for (Set<PickedExpression> peSet : pickedExpressions.values()) {
for (PickedExpression pe : peSet) {
exprDims.addAll(pe.pickedCtx.exprDims);
@@ -457,21 +457,13 @@ class ExpressionResolver implements ContextRewriter {
return exprDims;
}
- private void replacePickedExpressions(CandidateFact cfact, boolean replaceFact)
+ private void replacePickedExpressions(QueryAST queryAST)
throws LensException {
- if (replaceFact) {
- replaceAST(cubeql, cfact.getSelectAST());
- replaceAST(cubeql, cfact.getWhereAST());
- replaceAST(cubeql, cfact.getJoinTree());
- replaceAST(cubeql, cfact.getGroupByAST());
- replaceAST(cubeql, cfact.getHavingAST());
- } else {
- replaceAST(cubeql, cubeql.getSelectAST());
- replaceAST(cubeql, cubeql.getWhereAST());
- replaceAST(cubeql, cubeql.getJoinTree());
- replaceAST(cubeql, cubeql.getGroupByAST());
- replaceAST(cubeql, cubeql.getHavingAST());
- }
+ replaceAST(cubeql, queryAST.getSelectAST());
+ replaceAST(cubeql, queryAST.getWhereAST());
+ replaceAST(cubeql, queryAST.getJoinAST());
+ replaceAST(cubeql, queryAST.getGroupByAST());
+ replaceAST(cubeql, queryAST.getHavingAST());
replaceAST(cubeql, cubeql.getOrderByAST());
}
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/FactHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/FactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/FactHQLContext.java
deleted file mode 100644
index 6c44233..0000000
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/FactHQLContext.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.cube.parse;
-
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.lens.cube.metadata.Dimension;
-import org.apache.lens.server.api.error.LensException;
-
-import lombok.extern.slf4j.Slf4j;
-
-/**
- * HQL context class which passes all query strings from the fact and works with required dimensions for the fact.
- */
-@Slf4j
-public class FactHQLContext extends DimHQLContext {
-
- private final CandidateFact fact;
- private final Set<Dimension> factDims;
-
- FactHQLContext(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, Set<Dimension> factDims,
- CubeQueryContext query) throws LensException {
- super(query, dimsToQuery, factDims, fact.getSelectTree(), fact.getWhereTree(), fact.getGroupByTree(), null, fact
- .getHavingTree(), null);
- this.fact = fact;
- this.factDims = factDims;
- log.info("factDims:{} for fact:{}", factDims, fact);
- }
-
- @Override
- protected Set<Dimension> getQueriedDimSet() {
- return factDims;
- }
-
- @Override
- protected CandidateFact getQueriedFact() {
- return fact;
- }
-
- protected String getFromTable() throws LensException {
- return query.getQBFromString(fact, getDimsToQuery());
- }
-
- public CandidateFact getFactToQuery() {
- return fact;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
index da74713..9674f73 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
@@ -78,7 +78,6 @@ class GroupbyResolver implements ContextRewriter {
if (groupbyAST != null) {
// groupby ast exists, add the expression to AST
groupbyAST.addChild(exprAST);
- exprAST.setParent(groupbyAST);
} else {
// no group by ast exist, create one
ASTNode newAST = new ASTNode(new CommonToken(TOK_GROUPBY));
@@ -153,7 +152,6 @@ class GroupbyResolver implements ContextRewriter {
parent.setChild(i + 1, ch);
}
parent.setChild(index, child);
- child.setParent(parent);
}
@Override
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
index 7cea7d5..6c3d4c3 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
@@ -293,7 +293,6 @@ public final class HQLParser {
if (original.getChildren() != null) {
for (Object o : original.getChildren()) {
ASTNode childCopy = copyAST((ASTNode) o);
- childCopy.setParent(copy);
copy.addChild(childCopy);
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
index 1385584..de3a16e 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
@@ -87,8 +87,8 @@ class JoinResolver implements ContextRewriter {
boolean joinResolverDisabled = cubeql.getConf().getBoolean(CubeQueryConfUtil.DISABLE_AUTO_JOINS,
CubeQueryConfUtil.DEFAULT_DISABLE_AUTO_JOINS);
if (joinResolverDisabled) {
- if (cubeql.getJoinTree() != null) {
- cubeQB.setQbJoinTree(genJoinTree(cubeQB, cubeql.getJoinTree(), cubeql));
+ if (cubeql.getJoinAST() != null) {
+ cubeQB.setQbJoinTree(genJoinTree(cubeql.getJoinAST(), cubeql));
}
} else {
autoResolveJoins(cubeql);
@@ -336,7 +336,7 @@ class JoinResolver implements ContextRewriter {
}
// Recursively find out join conditions
- private QBJoinTree genJoinTree(QB qb, ASTNode joinParseTree, CubeQueryContext cubeql) throws LensException {
+ private QBJoinTree genJoinTree(ASTNode joinParseTree, CubeQueryContext cubeql) throws LensException {
QBJoinTree joinTree = new QBJoinTree();
JoinCond[] condn = new JoinCond[1];
@@ -388,7 +388,7 @@ class JoinResolver implements ContextRewriter {
} else if (isJoinToken(left)) {
// Left subtree is join token itself, so recurse down
- QBJoinTree leftTree = genJoinTree(qb, left, cubeql);
+ QBJoinTree leftTree = genJoinTree(left, cubeql);
joinTree.setJoinSrc(leftTree);
@@ -436,12 +436,9 @@ class JoinResolver implements ContextRewriter {
return joinTree;
}
- private boolean isJoinToken(ASTNode node) {
- if ((node.getToken().getType() == TOK_JOIN) || (node.getToken().getType() == TOK_LEFTOUTERJOIN)
+ private static boolean isJoinToken(ASTNode node) {
+ return (node.getToken().getType() == TOK_JOIN) || (node.getToken().getType() == TOK_LEFTOUTERJOIN)
|| (node.getToken().getType() == TOK_RIGHTOUTERJOIN) || (node.getToken().getType() == TOK_FULLOUTERJOIN)
- || (node.getToken().getType() == TOK_LEFTSEMIJOIN) || (node.getToken().getType() == TOK_UNIQUEJOIN)) {
- return true;
- }
- return false;
+ || (node.getToken().getType() == TOK_LEFTSEMIJOIN) || (node.getToken().getType() == TOK_UNIQUEJOIN);
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
index 113d8de..1a729f8 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
@@ -34,18 +34,24 @@ import com.google.common.collect.Lists;
*/
class MultiFactHQLContext extends SimpleHQLContext {
- private Map<Dimension, CandidateDim> dimsToQuery;
private Set<CandidateFact> facts;
private CubeQueryContext query;
- private Map<CandidateFact, Set<Dimension>> factDimMap;
+ private Map<CandidateFact, SimpleHQLContext> factHQLContextMap = new HashMap<>();
MultiFactHQLContext(Set<CandidateFact> facts, Map<Dimension, CandidateDim> dimsToQuery,
Map<CandidateFact, Set<Dimension>> factDimMap, CubeQueryContext query) throws LensException {
super();
this.query = query;
this.facts = facts;
- this.dimsToQuery = dimsToQuery;
- this.factDimMap = factDimMap;
+ for (CandidateFact fact : facts) {
+ if (fact.getStorageTables().size() > 1) {
+ factHQLContextMap.put(fact, new SingleFactMultiStorageHQLContext(fact, dimsToQuery, query, fact));
+ } else {
+ factHQLContextMap.put(fact,
+ new SingleFactSingleStorageHQLContext(fact, dimsToQuery, factDimMap.get(fact), query,
+ DefaultQueryAST.fromCandidateFact(fact, fact.getStorageTables().iterator().next(), fact)));
+ }
+ }
}
protected void setMissingExpressions() throws LensException {
@@ -78,8 +84,7 @@ class MultiFactHQLContext extends SimpleHQLContext {
}
private String getSelectString() throws LensException {
- Map<Integer, List<Integer>> selectToFactIndex =
- new HashMap<Integer, List<Integer>>(query.getSelectAST().getChildCount());
+ Map<Integer, List<Integer>> selectToFactIndex = new HashMap<>(query.getSelectAST().getChildCount());
int fi = 1;
for (CandidateFact fact : facts) {
for (int ind : fact.getSelectIndices()) {
@@ -116,33 +121,14 @@ class MultiFactHQLContext extends SimpleHQLContext {
return select.toString();
}
- public Map<Dimension, CandidateDim> getDimsToQuery() {
- return dimsToQuery;
- }
-
- public Set<CandidateFact> getFactsToQuery() {
- return facts;
- }
-
private String getFromString() throws LensException {
StringBuilder fromBuilder = new StringBuilder();
int aliasCount = 1;
- Iterator<CandidateFact> iter = facts.iterator();
- while (iter.hasNext()) {
- CandidateFact fact = iter.next();
- if (fact.getStorageTables().size() > 1) {
- // Not supported right now.
- throw new LensException(LensCubeErrorCode.STORAGE_UNION_DISABLED.getLensErrorInfo());
- }
- FactHQLContext facthql = new FactHQLContext(fact, dimsToQuery, factDimMap.get(fact), query);
- fromBuilder.append("(");
- fromBuilder.append(facthql.toHQL());
- fromBuilder.append(")");
- fromBuilder.append(" mq" + aliasCount);
- aliasCount++;
- if (iter.hasNext()) {
- fromBuilder.append(" full outer join ");
- }
+ String sep = "";
+ for (CandidateFact fact : facts) {
+ SimpleHQLContext facthql = factHQLContextMap.get(fact);
+ fromBuilder.append(sep).append("(").append(facthql.toHQL()).append(")").append(" mq").append(aliasCount++);
+ sep = " full outer join ";
}
CandidateFact firstFact = facts.iterator().next();
if (!firstFact.getDimFieldIndices().isEmpty()) {
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java
new file mode 100644
index 0000000..31680ca
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.parse;
+
+
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+
+
+interface QueryAST {
+
+ String getSelectTree();
+
+ String getWhereTree();
+
+ String getHavingTree();
+
+ String getOrderByTree();
+
+ String getGroupByTree();
+
+ Integer getLimitValue();
+
+ void setLimitValue(Integer integer);
+
+ /**
+ * @return the selectAST
+ */
+
+ ASTNode getSelectAST();
+
+ /**
+ * @param selectAST the selectAST to set
+ */
+
+ void setSelectAST(ASTNode selectAST);
+
+ /**
+ * @return the whereAST
+ */
+
+ ASTNode getWhereAST();
+
+ /**
+ * @param whereAST the whereAST to set
+ */
+
+ void setWhereAST(ASTNode whereAST);
+
+ /**
+ * @return the havingAST
+ */
+
+ ASTNode getHavingAST();
+
+ /**
+ * @param havingAST the havingAST to set
+ */
+
+ void setHavingAST(ASTNode havingAST);
+
+ ASTNode getGroupByAST();
+
+ void setGroupByAST(ASTNode havingAST);
+
+ ASTNode getJoinAST();
+
+ ASTNode getOrderByAST();
+
+ void setOrderByAST(ASTNode node);
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java
deleted file mode 100644
index de52b0a..0000000
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.cube.parse;
-
-import java.util.Map;
-
-import org.apache.lens.cube.metadata.Dimension;
-import org.apache.lens.cube.metadata.TimeRange;
-import org.apache.lens.server.api.error.LensException;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hive.ql.parse.ASTNode;
-
-/**
- * HQL context class which passes down all query strings to come from DimOnlyHQLContext and works with fact being
- * queried.
- * <p/>
- * Updates from string with join clause expanded
- */
-class SingleFactHQLContext extends DimOnlyHQLContext {
-
- private final CandidateFact fact;
- private String storageAlias;
-
- SingleFactHQLContext(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, CubeQueryContext query)
- throws LensException {
- super(dimsToQuery, query);
- this.fact = fact;
- }
-
- SingleFactHQLContext(CandidateFact fact, String storageAlias, Map<Dimension, CandidateDim> dimsToQuery,
- CubeQueryContext query, String whereClause) throws LensException {
- super(dimsToQuery, query, whereClause);
- this.fact = fact;
- this.storageAlias = storageAlias;
- }
-
-
- public CandidateFact getFactToQuery() {
- return fact;
- }
-
- static void addRangeClauses(CubeQueryContext query, CandidateFact fact) throws LensException {
- if (fact != null) {
- // resolve timerange positions and replace it by corresponding where
- // clause
- for (TimeRange range : query.getTimeRanges()) {
- for (Map.Entry<String, String> entry : fact.getRangeToStorageWhereMap().get(range).entrySet()) {
- String table = entry.getValue();
- String rangeWhere = entry.getKey();
-
- if (!StringUtils.isBlank(rangeWhere)) {
- ASTNode rangeAST = HQLParser.parseExpr(rangeWhere);
- rangeAST.setParent(range.getParent());
- range.getParent().setChild(range.getChildIndex(), rangeAST);
- }
- fact.getStorgeWhereClauseMap().put(table, query.getWhereTree());
- }
- }
- }
- }
-
-
- @Override
- protected String getFromTable() throws LensException {
- if (getQuery().getAutoJoinCtx() != null && getQuery().getAutoJoinCtx().isJoinsResolved()) {
- if (storageAlias != null) {
- return storageAlias;
- } else {
- return fact.getStorageString(getQuery().getAliasForTableName(getQuery().getCube().getName()));
- }
- } else {
- if (fact.getStorageTables().size() == 1) {
- return getQuery().getQBFromString(fact, getDimsToQuery());
- } else {
- return storageAlias;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
index 96b1d05..e531e6b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
@@ -19,6 +19,8 @@
package org.apache.lens.cube.parse;
+import static org.apache.lens.cube.parse.CubeQueryConfUtil.DEFAULT_ENABLE_STORAGES_UNION;
+import static org.apache.lens.cube.parse.CubeQueryConfUtil.ENABLE_STORAGES_UNION;
import static org.apache.lens.cube.parse.HQLParser.*;
import static org.apache.hadoop.hive.ql.parse.HiveParser.*;
@@ -27,6 +29,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lens.cube.error.LensCubeErrorCode;
import org.apache.lens.cube.metadata.Dimension;
import org.apache.lens.server.api.error.LensException;
@@ -42,6 +45,7 @@ import lombok.Data;
public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
+ private final QueryAST ast;
int aliasCounter = 0;
@Data
@@ -81,29 +85,33 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
private Map<HashableASTNode, ASTNode> innerToOuterASTs = new HashMap<>();
- SingleFactMultiStorageHQLContext(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, CubeQueryContext query)
+ SingleFactMultiStorageHQLContext(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery,
+ CubeQueryContext query, QueryAST ast)
throws LensException {
super(query, fact);
+ if (!query.getConf().getBoolean(ENABLE_STORAGES_UNION, DEFAULT_ENABLE_STORAGES_UNION)) {
+ throw new LensException(LensCubeErrorCode.STORAGE_UNION_DISABLED.getLensErrorInfo());
+ }
+ this.ast = ast;
processSelectAST();
processGroupByAST();
processWhereAST();
processHavingAST();
processOrderByAST();
processLimit();
- setHqlContexts(getUnionContexts(fact, dimsToQuery, query));
+ setHqlContexts(getUnionContexts(fact, dimsToQuery, query, ast));
}
private void processSelectAST() {
- query.getSelectFinalAliases().clear();
- ASTNode originalSelectAST = copyAST(query.getSelectAST());
- query.setSelectAST(new ASTNode(originalSelectAST.getToken()));
+ ASTNode originalSelectAST = copyAST(ast.getSelectAST());
+ ast.setSelectAST(new ASTNode(originalSelectAST.getToken()));
ASTNode outerSelectAST = processExpression(originalSelectAST);
setSelect(getString(outerSelectAST));
}
private void processGroupByAST() {
- if (query.getGroupByAST() != null) {
- setGroupby(getString(processExpression(query.getGroupByAST())));
+ if (ast.getGroupByAST() != null) {
+ setGroupby(getString(processExpression(ast.getGroupByAST())));
}
}
@@ -111,28 +119,29 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
for (String storageTable : fact.getStorgeWhereClauseMap().keySet()) {
ASTNode tree = parseExpr(fact.getStorgeWhereClauseMap().get(storageTable));
ASTNode replaced = replaceAST(tree);
+ //TODO: optimize parse/unparse cycle
fact.getStorgeWhereClauseMap().put(storageTable, getString(replaced));
}
}
private void processHavingAST() throws LensException {
- if (query.getHavingAST() != null) {
- setHaving(getString(processExpression(query.getHavingAST())));
- query.setHavingAST(null);
+ if (ast.getHavingAST() != null) {
+ setHaving(getString(processExpression(ast.getHavingAST())));
+ ast.setHavingAST(null);
}
}
private void processOrderByAST() {
- if (query.getOrderByAST() != null) {
- setOrderby(getString(processExpression(query.getOrderByAST())));
- query.setOrderByAST(null);
+ if (ast.getOrderByAST() != null) {
+ setOrderby(getString(processExpression(ast.getOrderByAST())));
+ ast.setOrderByAST(null);
}
}
private void processLimit() {
- setLimit(query.getLimitValue());
- query.setLimitValue(null);
+ setLimit(ast.getLimitValue());
+ ast.setLimitValue(null);
}
/*
@@ -171,6 +180,7 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
addToInnerSelectAST(innerSelectExprAST);
ASTNode dotAST = getDotAST(query.getCube().getName(), alias);
ASTNode outerAST = new ASTNode(new CommonToken(TOK_FUNCTION));
+ //TODO: take care or non-transitive aggregate functions
outerAST.addChild(new ASTNode(new CommonToken(Identifier, astNode.getChild(0).getText())));
outerAST.addChild(dotAST);
innerToOuterASTs.put(new HashableASTNode(innerSelectASTWithoutAlias), outerAST);
@@ -225,10 +235,10 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
}
private void addToInnerSelectAST(ASTNode selectExprAST) {
- if (query.getSelectAST() == null) {
- query.setSelectAST(new ASTNode(new CommonToken(TOK_SELECT)));
+ if (ast.getSelectAST() == null) {
+ ast.setSelectAST(new ASTNode(new CommonToken(TOK_SELECT)));
}
- query.getSelectAST().addChild(selectExprAST);
+ ast.getSelectAST().addChild(selectExprAST);
}
private ASTNode getDotAST(String tableAlias, String fieldAlias) {
@@ -245,13 +255,13 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
}
private static ArrayList<HQLContextInterface> getUnionContexts(CandidateFact fact, Map<Dimension, CandidateDim>
- dimsToQuery, CubeQueryContext query)
+ dimsToQuery, CubeQueryContext query, QueryAST ast)
throws LensException {
ArrayList<HQLContextInterface> contexts = new ArrayList<>();
String alias = query.getAliasForTableName(query.getCube().getName());
for (String storageTable : fact.getStorageTables()) {
- SingleFactHQLContext ctx = new SingleFactHQLContext(fact, storageTable + " " + alias, dimsToQuery, query,
- fact.getWhereClause(storageTable.substring(storageTable.indexOf(".") + 1)));
+ SingleFactSingleStorageHQLContext ctx = new SingleFactSingleStorageHQLContext(fact, storageTable + " " + alias,
+ dimsToQuery, query, DefaultQueryAST.fromCandidateFact(fact, storageTable, ast));
contexts.add(ctx);
}
return contexts;
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java
new file mode 100644
index 0000000..b1a3b3f
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.parse;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.lens.cube.metadata.Dimension;
+import org.apache.lens.server.api.error.LensException;
+
+/**
+ * HQL context class which passes down all query strings to come from DimOnlyHQLContext and works with fact being
+ * queried.
+ * <p/>
+ * Updates from string with join clause expanded
+ */
+class SingleFactSingleStorageHQLContext extends DimOnlyHQLContext {
+
+ private final CandidateFact fact;
+ private final Set<Dimension> queriedDimSet;
+ private String storageAlias;
+
+ SingleFactSingleStorageHQLContext(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery,
+ CubeQueryContext query, QueryAST ast)
+ throws LensException {
+ this(fact, dimsToQuery, dimsToQuery.keySet(), query, ast);
+ }
+
+ SingleFactSingleStorageHQLContext(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery,
+ Set<Dimension> dimsQueried, CubeQueryContext query, QueryAST ast)
+ throws LensException {
+ super(dimsToQuery, dimsQueried, query, ast);
+ this.fact = fact;
+ this.queriedDimSet = dimsQueried;
+ }
+
+ SingleFactSingleStorageHQLContext(CandidateFact fact, String storageAlias, Map<Dimension, CandidateDim> dimsToQuery,
+ CubeQueryContext query, QueryAST ast) throws LensException {
+ this(fact, dimsToQuery, query, ast);
+ this.storageAlias = storageAlias;
+ }
+
+ @Override
+ protected String getFromTable() throws LensException {
+ if (getQuery().isAutoJoinResolved()) {
+ if (storageAlias != null) {
+ return storageAlias;
+ } else {
+ return fact.getStorageString(query.getAliasForTableName(query.getCube().getName()));
+ }
+ } else {
+ if (fact.getStorageTables().size() == 1) {
+ return getQuery().getQBFromString(fact, getDimsToQuery());
+ } else {
+ return storageAlias;
+ }
+ }
+ }
+
+ @Override
+ protected CandidateFact getQueriedFact() {
+ return fact;
+ }
+
+ @Override
+ public Set<Dimension> getQueriedDimSet() {
+ return queriedDimSet;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index 62cc071..14def15 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -19,14 +19,10 @@
package org.apache.lens.cube.parse;
import static org.apache.lens.cube.metadata.DateUtil.WSPACE;
-import static org.apache.lens.cube.metadata.MetastoreUtil.getFactOrDimtableStorageTableName;
-import static org.apache.lens.cube.metadata.MetastoreUtil.getStoragetableEndTimesKey;
-import static org.apache.lens.cube.metadata.MetastoreUtil.getStoragetableStartTimesKey;
+import static org.apache.lens.cube.metadata.MetastoreUtil.*;
import static org.apache.lens.cube.parse.CandidateTablePruneCause.*;
import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*;
-import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode.PART_COL_DOES_NOT_EXIST;
-import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode.RANGE_NOT_ANSWERABLE;
-import static org.apache.lens.cube.parse.StorageUtil.joinWithAnd;
+import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode.*;
import java.text.DateFormat;
import java.text.ParseException;
@@ -36,10 +32,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lens.cube.metadata.*;
-import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
-import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCause;
-import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode;
-import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipUpdatePeriodCode;
+import org.apache.lens.cube.parse.CandidateTablePruneCause.*;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.StringUtils;
@@ -49,7 +42,6 @@ import org.apache.hadoop.util.ReflectionUtils;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-
import lombok.extern.slf4j.Slf4j;
/**
@@ -361,9 +353,9 @@ class StorageTableResolver implements ContextRewriter {
private void resolveFactStoragePartitions(CubeQueryContext cubeql) throws LensException {
// Find candidate tables wrt supported storages
Iterator<CandidateFact> i = cubeql.getCandidateFacts().iterator();
- Map<TimeRange, String> whereClauseForFallback = new LinkedHashMap<TimeRange, String>();
while (i.hasNext()) {
CandidateFact cfact = i.next();
+ Map<TimeRange, String> whereClauseForFallback = new LinkedHashMap<TimeRange, String>();
List<FactPartition> answeringParts = new ArrayList<>();
Map<String, SkipStorageCause> skipStorageCauses = skipStorageCausesPerFact.get(cfact.fact);
if (skipStorageCauses == null) {
@@ -434,9 +426,6 @@ class StorageTableResolver implements ContextRewriter {
cfact.incrementPartsQueried(rangeParts.size());
answeringParts.addAll(rangeParts);
cfact.getPartsQueried().addAll(rangeParts);
- String rangeWhereClause = rangeWriter.getTimeRangeWhereClause(cubeql,
- cubeql.getAliasForTableName(cubeql.getCube().getName()), rangeParts);
- cfact.getRangeToWhereClause().put(range, joinWithAnd(rangeWhereClause, extraWhereClause.toString()));
}
if (!unsupportedTimeDims.isEmpty()) {
log.info("Not considering fact table:{} as it doesn't support time dimensions: {}", cfact.fact,
@@ -483,24 +472,27 @@ class StorageTableResolver implements ContextRewriter {
Set<String> storageTables = new LinkedHashSet<>();
storageTables.addAll(minimalStorageTables.keySet());
cfact.setStorageTables(storageTables);
-
// Update range->storage->partitions with time range where clause
for (TimeRange trange : cfact.getRangeToStoragePartMap().keySet()) {
- Map<String, String> rangeToWhere = new HashMap<String, String>();
+ Map<String, String> rangeToWhere = new HashMap<>();
for (Map.Entry<String, Set<FactPartition>> entry : minimalStorageTables.entrySet()) {
String table = entry.getKey();
Set<FactPartition> minimalParts = entry.getValue();
LinkedHashSet<FactPartition> rangeParts = cfact.getRangeToStoragePartMap().get(trange).get(table);
- LinkedHashSet<FactPartition> minimalPartsCopy = new LinkedHashSet<FactPartition>(minimalParts);
- minimalPartsCopy.retainAll(rangeParts);
+ LinkedHashSet<FactPartition> minimalPartsCopy = Sets.newLinkedHashSet();
+
+ if (rangeParts != null) {
+ minimalPartsCopy.addAll(minimalParts);
+ minimalPartsCopy.retainAll(rangeParts);
+ }
if (!StringUtils.isEmpty(whereClauseForFallback.get(trange))) {
- rangeToWhere.put(
- rangeWriter.getTimeRangeWhereClause(cubeql, cubeql.getAliasForTableName(cubeql.getCube().getName()),
- minimalPartsCopy) + " and " + whereClauseForFallback.get(trange), table);
+ rangeToWhere.put(table, "(("
+ + rangeWriter.getTimeRangeWhereClause(cubeql, cubeql.getAliasForTableName(cubeql.getCube().getName()),
+ minimalPartsCopy) + ") and (" + whereClauseForFallback.get(trange) + "))");
} else {
- rangeToWhere.put(rangeWriter.getTimeRangeWhereClause(cubeql,
- cubeql.getAliasForTableName(cubeql.getCube().getName()), minimalPartsCopy), table);
+ rangeToWhere.put(table, rangeWriter.getTimeRangeWhereClause(cubeql,
+ cubeql.getAliasForTableName(cubeql.getCube().getName()), minimalPartsCopy));
}
}
cfact.getRangeToStorageWhereMap().put(trange, rangeToWhere);
@@ -592,8 +584,7 @@ class StorageTableResolver implements ContextRewriter {
int lookAheadNumParts =
conf.getInt(CubeQueryConfUtil.getLookAheadPTPartsKey(interval), CubeQueryConfUtil.DEFAULT_LOOK_AHEAD_PT_PARTS);
- TimeRange.Iterable.Iterator iter = TimeRange.iterable(ceilFromDate, floorToDate, interval, 1)
- .iterator();
+ TimeRange.Iterable.Iterator iter = TimeRange.iterable(ceilFromDate, floorToDate, interval, 1).iterator();
// add partitions from ceilFrom to floorTo
while (iter.hasNext()) {
Date dt = iter.next();
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
index 87e4ce3..e7b9403 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
@@ -26,6 +26,10 @@ import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
+import org.apache.hadoop.util.StringUtils;
+
+import com.google.common.collect.Lists;
+
public class DateFactory {
private DateFactory() {
@@ -143,6 +147,7 @@ public class DateFactory {
// Time Ranges
public static final String LAST_HOUR_TIME_RANGE;
public static final String TWO_DAYS_RANGE;
+ public static final String TWO_DAYS_RANGE_SPLIT_OVER_UPDATE_PERIODS;
public static final String TWO_DAYS_RANGE_TTD;
public static final String TWO_DAYS_RANGE_TTD_BEFORE_4_DAYS;
public static final String TWO_DAYS_RANGE_TTD2;
@@ -192,5 +197,11 @@ public class DateFactory {
// calculate LAST_HOUR_TIME_RANGE
LAST_HOUR_TIME_RANGE = getTimeRangeString(HOURLY, -1, 0);
+
+ TWO_DAYS_RANGE_SPLIT_OVER_UPDATE_PERIODS = StringUtils.join(" OR ", Lists.newArrayList(
+ getTimeRangeString(getDateStringWithOffset(HOURLY, -48), getDateStringWithOffset(DAILY, -1)),
+ getTimeRangeString(getDateStringWithOffset(DAILY, 0), getDateStringWithOffset(HOURLY, 0)),
+ getTimeRangeString(getDateStringWithOffset(DAILY, -1), getDateStringWithOffset(DAILY, 0))
+ ));
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 3f01dbe..ad20ae1 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -512,7 +512,7 @@ public class CubeTestSetup {
"No aggregateMsr", null, null, null));
cubeMeasures.add(new ColumnMeasure(new FieldSchema("newmeasure", "bigint", "measure available from now"),
"New measure", null, null, null, NOW, null, 100.0));
- cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr15", "int", "first measure"), "Measure15", null, "SUM",
+ cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr15", "int", "fifteenth measure"), "Measure15", null, "SUM",
"RS"));
cubeDimensions = new HashSet<CubeDimAttribute>();
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
index a5886dc..5b44f95 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
@@ -50,6 +50,7 @@ import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.common.base.Splitter;
+import com.google.common.collect.Sets;
import lombok.Getter;
public class TestBaseCubeQueries extends TestQueryRewrite {
@@ -546,17 +547,24 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
assertEquals(ctx.getCandidateFactSets().size(), 1);
assertEquals(ctx.getCandidateFactSets().iterator().next().size(), 1);
CandidateFact cfact = ctx.getCandidateFactSets().iterator().next().iterator().next();
- assertEquals(cfact.getRangeToWhereClause().size(), 2);
- for(Map.Entry<TimeRange, String> entry: cfact.getRangeToWhereClause().entrySet()) {
+
+ assertEquals(cfact.getRangeToStoragePartMap().size(), 2);
+ Set<String> storages = Sets.newHashSet();
+ for(Map<String, String> entry: cfact.getRangeToStorageWhereMap().values()) {
+ storages.addAll(entry.keySet());
+ }
+ assertEquals(storages.size(), 1);
+ String storage = storages.iterator().next();
+ for(Map.Entry<TimeRange, Map<String, String>> entry: cfact.getRangeToStorageWhereMap().entrySet()) {
if (entry.getKey().getPartitionColumn().equals("dt")) {
- ASTNode parsed = HQLParser.parseExpr(entry.getValue());
+ ASTNode parsed = HQLParser.parseExpr(entry.getValue().get(storage));
assertEquals(parsed.getToken().getType(), KW_AND);
- assertTrue(entry.getValue().substring(((CommonToken) parsed.getToken()).getStopIndex() + 1).toLowerCase()
- .contains(dTimeWhereClause));
- assertFalse(entry.getValue().substring(0, ((CommonToken) parsed.getToken()).getStartIndex()).toLowerCase()
- .contains("and"));
+ assertTrue(entry.getValue().get(storage).substring(((CommonToken) parsed.getToken()).getStopIndex() + 1)
+ .toLowerCase().contains(dTimeWhereClause));
+ assertFalse(entry.getValue().get(storage).substring(0, ((CommonToken) parsed.getToken()).getStartIndex())
+ .toLowerCase().contains("and"));
} else if (entry.getKey().getPartitionColumn().equals("ttd")) {
- assertFalse(entry.getValue().toLowerCase().contains("and"));
+ assertFalse(entry.getValue().get(storage).toLowerCase().contains("and"));
} else {
throw new LensException("Unexpected");
}
http://git-wip-us.apache.org/repos/asf/lens/blob/04f5a822/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index 9a08735..f02cdb0 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -41,7 +41,6 @@ import org.apache.lens.server.api.LensServerAPITestUtil;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.time.DateUtils;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -58,7 +57,6 @@ import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-
import lombok.extern.slf4j.Slf4j;
@Slf4j
@@ -410,7 +408,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
return getWhereForMonthlyDailyAndHourly2monthsUnionQuery(storage);
}
};
- try{
+ try {
rewrite("select cityid as `City ID`, msr8, msr7 as `Third measure` "
+ "from testCube where " + TWO_MONTHS_RANGE_UPTO_HOURS, conf);
fail("Union feature is disabled, should have failed");
@@ -501,6 +499,35 @@ public class TestCubeRewriter extends TestQueryRewrite {
}
@Test
+ public void testMultiFactMultiStorage() throws ParseException, LensException {
+ Configuration conf = LensServerAPITestUtil.getConfiguration(
+ CubeQueryConfUtil.ENABLE_STORAGES_UNION, true,
+ CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C1,C2",
+ getValidStorageTablesKey("testfact"), "C1_testFact,C2_testFact",
+ getValidUpdatePeriodsKey("testfact", "C1"), "HOURLY",
+ getValidUpdatePeriodsKey("testfact", "C2"), "DAILY",
+ getValidUpdatePeriodsKey("testfact2_raw", "C1"), "YEARLY",
+ getValidUpdatePeriodsKey("testfact2_raw", "C2"), "YEARLY");
+ CubeTestSetup.getStorageToUpdatePeriodMap().put("c1_testfact", Lists.newArrayList(HOURLY));
+ CubeTestSetup.getStorageToUpdatePeriodMap().put("c2_testfact", Lists.newArrayList(DAILY));
+ String whereCond = "zipcode = 'a' and cityid = 'b' and (" + TWO_DAYS_RANGE_SPLIT_OVER_UPDATE_PERIODS + ")";
+ String hqlQuery = rewrite("cube select zipcode, count(msr4), sum(msr15) from testCube where " + whereCond, conf);
+ System.out.println(hqlQuery);
+ String possibleStart1 = "SELECT COALESCE(mq1.zipcode, mq2.zipcode) zipcode, mq1.msr4 msr4, mq2.msr15 msr15 FROM ";
+ String possibleStart2 = "SELECT COALESCE(mq1.zipcode, mq2.zipcode) zipcode, mq2.msr4 msr4, mq1.msr15 msr15 FROM ";
+
+ assertTrue(hqlQuery.startsWith(possibleStart1) || hqlQuery.startsWith(possibleStart2));
+ compareContains(rewrite("cube select zipcode as `zipcode`, sum(msr15) as `msr15` from testcube where " + whereCond,
+ conf), hqlQuery);
+ compareContains(rewrite("cube select zipcode as `zipcode`, count(msr4) as `msr4` from testcube where " + whereCond,
+ conf), hqlQuery);
+ assertTrue(hqlQuery.endsWith("on mq1.zipcode <=> mq2.zipcode"));
+ // No time_range_in should be remaining
+ assertFalse(hqlQuery.contains("time_range_in"));
+ //TODO: handle having after LENS-813, also handle for order by and limit
+ }
+
+ @Test
public void testCubeWhereQueryWithMultipleTables() throws Exception {
Configuration conf = getConf();
conf.setBoolean(CubeQueryConfUtil.ENABLE_STORAGES_UNION, true);
@@ -1120,18 +1147,18 @@ public class TestCubeRewriter extends TestQueryRewrite {
new HashMap<String, List<CandidateTablePruneCause>>() {
{
put("statetable", Arrays.asList(CandidateTablePruneCause.noCandidateStorages(
- new HashMap<String, SkipStorageCause>() {
- {
- put("c1_statetable", new SkipStorageCause(SkipStorageCode.NO_PARTITIONS));
- }
- }))
+ new HashMap<String, SkipStorageCause>() {
+ {
+ put("c1_statetable", new SkipStorageCause(SkipStorageCode.NO_PARTITIONS));
+ }
+ }))
);
put("statetable_partitioned", Arrays.asList(CandidateTablePruneCause.noCandidateStorages(
- new HashMap<String, SkipStorageCause>() {
- {
- put("C3_statetable_partitioned", new SkipStorageCause(SkipStorageCode.UNSUPPORTED));
- }
- }))
+ new HashMap<String, SkipStorageCause>() {
+ {
+ put("C3_statetable_partitioned", new SkipStorageCause(SkipStorageCode.UNSUPPORTED));
+ }
+ }))
);
}
}