You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by jd...@apache.org on 2015/06/23 12:25:07 UTC
[19/51] [abbrv] incubator-lens git commit: LENS-174: (incremental)
Cube rewriter changes to support multiple expressions
LENS-174: (incremental) Cube rewriter changes to support multiple expressions
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/099d19e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/099d19e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/099d19e8
Branch: refs/heads/current-release-line
Commit: 099d19e881332b90eadc1fb36b443d2954c11bd2
Parents: ad471b2
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Tue Jun 9 12:16:01 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Tue Jun 9 12:16:01 2015 +0530
----------------------------------------------------------------------
.../lens/cube/parse/AggregateResolver.java | 6 +-
.../apache/lens/cube/parse/AliasReplacer.java | 6 +-
.../lens/cube/parse/CandidateTableResolver.java | 75 ++++++++++++--------
.../lens/cube/parse/CubeQueryContext.java | 15 ++--
.../lens/cube/parse/ExpressionResolver.java | 33 ++++-----
5 files changed, 75 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/099d19e8/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
index 2ad2b7c..edf0f74 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
@@ -218,7 +218,7 @@ class AggregateResolver implements ContextRewriter {
if (node.getChild(0).getType() == HiveParser.Identifier) {
function = BaseSemanticAnalyzer.unescapeIdentifier(node.getChild(0).getText());
}
- } else if (cubeql.isMeasure(node)) {
+ } else if (cubeql.isCubeMeasure(node)) {
// Exit for the recursion
String colname;
@@ -291,7 +291,7 @@ class AggregateResolver implements ContextRewriter {
boolean isDistinct = hasDistinct;
if (exprTokenType == HiveParser.TOK_FUNCTIONDI || exprTokenType == HiveParser.TOK_SELECTDI) {
isDistinct = true;
- } else if (cubeql.isMeasure(node) && isDistinct) {
+ } else if (cubeql.isCubeMeasure(node) && isDistinct) {
// Exit for the recursion
return true;
}
@@ -310,7 +310,7 @@ class AggregateResolver implements ContextRewriter {
return false;
}
- if (cubeql.isMeasure(node)) {
+ if (cubeql.isCubeMeasure(node)) {
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/099d19e8/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
index 4d3443c..9309307 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
@@ -120,11 +120,11 @@ class AliasReplacer implements ContextRewriter {
}
private void extractTabAliasForCol(CubeQueryContext cubeql) throws SemanticException {
- extractTabAliasForCol(cubeql.getColToTableAlias(), cubeql, cubeql);
+ extractTabAliasForCol(cubeql, cubeql);
}
- static void extractTabAliasForCol(Map<String, String> colToTableAlias, CubeQueryContext cubeql,
- TrackQueriedColumns tqc) throws SemanticException {
+ static void extractTabAliasForCol(CubeQueryContext cubeql, TrackQueriedColumns tqc) throws SemanticException {
+ Map<String, String> colToTableAlias = cubeql.getColToTableAlias();
Set<String> columns = tqc.getTblAliasToColumns().get(CubeQueryContext.DEFAULT_TABLE);
if (columns == null) {
return;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/099d19e8/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index d56fb80..53fb11f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -284,23 +284,35 @@ class CandidateTableResolver implements ContextRewriter {
(!queriedDimAttrs.isEmpty() ? queriedDimAttrs.toString() : "")
+ (!dimExprs.isEmpty() ? dimExprs.toString() : ""));
}
- // Find out candidate fact table sets which contain all the measures
- // queried
- List<CandidateFact> cfacts = new ArrayList<CandidateFact>(cubeql.getCandidateFacts());
- Set<Set<CandidateFact>> cfactset = findCoveringSets(cubeql, cfacts, queriedMsrs,
- cubeql.getQueriedExprsWithMeasures());
- LOG.info("Measure covering fact sets :" + cfactset);
- String msrString = (!queriedMsrs.isEmpty() ? queriedMsrs.toString() : "")
- + (!cubeql.getQueriedExprsWithMeasures().isEmpty() ? cubeql.getQueriedExprsWithMeasures().toString() : "");
- if (cfactset.isEmpty()) {
- throw new SemanticException(ErrorMsg.NO_FACT_HAS_COLUMN, msrString);
- }
- cubeql.getCandidateFactSets().addAll(cfactset);
- cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCause.columnNotFound(queriedMsrs,
- cubeql.getQueriedExprsWithMeasures()));
+ Set<Set<CandidateFact>> cfactset;
+ if (queriedMsrs.isEmpty() && cubeql.getQueriedExprsWithMeasures().isEmpty()) {
+ // if no measures are queried, add all facts individually as single covering sets
+ cfactset = new HashSet<Set<CandidateFact>>();
+ for (CandidateFact cfact : cubeql.getCandidateFacts()) {
+ Set<CandidateFact> one = new LinkedHashSet<CandidateFact>();
+ one.add(cfact);
+ cfactset.add(one);
+ }
+ cubeql.getCandidateFactSets().addAll(cfactset);
+ } else {
+ // Find out candidate fact table sets which contain all the measures
+ // queried
+ List<CandidateFact> cfacts = new ArrayList<CandidateFact>(cubeql.getCandidateFacts());
+ cfactset = findCoveringSets(cubeql, cfacts, queriedMsrs,
+ cubeql.getQueriedExprsWithMeasures());
+ LOG.info("Measure covering fact sets :" + cfactset);
+ String msrString = (!queriedMsrs.isEmpty() ? queriedMsrs.toString() : "")
+ + (!cubeql.getQueriedExprsWithMeasures().isEmpty() ? cubeql.getQueriedExprsWithMeasures().toString() : "");
+ if (cfactset.isEmpty()) {
+ throw new SemanticException(ErrorMsg.NO_FACT_HAS_COLUMN, msrString);
+ }
+ cubeql.getCandidateFactSets().addAll(cfactset);
+ cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCause.columnNotFound(queriedMsrs,
+ cubeql.getQueriedExprsWithMeasures()));
- if (cubeql.getCandidateFacts().size() == 0) {
- throw new SemanticException(ErrorMsg.NO_FACT_HAS_COLUMN, msrString);
+ if (cubeql.getCandidateFacts().size() == 0) {
+ throw new SemanticException(ErrorMsg.NO_FACT_HAS_COLUMN, msrString);
+ }
}
}
}
@@ -312,10 +324,10 @@ class CandidateTableResolver implements ContextRewriter {
for (Iterator<CandidateFact> i = cfacts.iterator(); i.hasNext();) {
CandidateFact cfact = i.next();
i.remove();
- if (!checkForColumnExists(cfact, msrs)
+ // cfact does not contain any of msrs and none of exprsWithMeasures are evaluable.
+ if ((msrs.isEmpty() || !checkForColumnExists(cfact, msrs))
&& (exprsWithMeasures.isEmpty() || cubeql.getExprCtx().allNotEvaluable(exprsWithMeasures, cfact))) {
- // check if fact contains any of the maeasures
- // if not ignore the fact
+ // ignore the fact
continue;
} else if (cfact.getColumns().containsAll(msrs) && cubeql.getExprCtx().allEvaluable(cfact, exprsWithMeasures)) {
// return single set
@@ -324,18 +336,21 @@ class CandidateTableResolver implements ContextRewriter {
cfactset.add(one);
} else {
// find the remaining measures in other facts
- Set<String> remainingMsrs = new HashSet<String>(msrs);
- Set<String> remainingExprs = new HashSet<String>(exprsWithMeasures);
- remainingMsrs.removeAll(cfact.getColumns());
- remainingExprs.removeAll(cubeql.getExprCtx().coveringExpressions(exprsWithMeasures, cfact));
- Set<Set<CandidateFact>> coveringSets = findCoveringSets(cubeql, cfacts, remainingMsrs, remainingExprs);
- if (!coveringSets.isEmpty()) {
- for (Set<CandidateFact> set : coveringSets) {
- set.add(cfact);
- cfactset.add(set);
+ if (i.hasNext()) {
+ Set<String> remainingMsrs = new HashSet<String>(msrs);
+ Set<String> remainingExprs = new HashSet<String>(exprsWithMeasures);
+ remainingMsrs.removeAll(cfact.getColumns());
+ remainingExprs.removeAll(cubeql.getExprCtx().coveringExpressions(exprsWithMeasures, cfact));
+ Set<Set<CandidateFact>> coveringSets = findCoveringSets(cubeql, cfacts, remainingMsrs, remainingExprs);
+ if (!coveringSets.isEmpty()) {
+ for (Set<CandidateFact> set : coveringSets) {
+ set.add(cfact);
+ cfactset.add(set);
+ }
+ } else {
+ LOG.info("Couldnt find any set containing remaining measures:" + remainingMsrs + " " + remainingExprs
+ + " in " + cfactsPassed);
}
- } else {
- LOG.info("Couldnt find any set containing remaining measures:" + remainingMsrs + " " + remainingExprs);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/099d19e8/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 0409af3..c5d8d91 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -21,7 +21,6 @@ package org.apache.lens.cube.parse;
import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TABLE_OR_COL;
-
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TMP_FILE;
import java.io.ByteArrayOutputStream;
@@ -803,20 +802,20 @@ public class CubeQueryContext implements TrackQueriedColumns {
}
// pick dimension tables required during expression expansion for the picked fact and dimensions
- Set<Dimension> exprDimTables = new HashSet<Dimension>();
+ Set<Dimension> exprDimensions = new HashSet<Dimension>();
if (cfacts != null) {
for (CandidateFact cfact : cfacts) {
Set<Dimension> factExprDimTables = exprCtx.rewriteExprCtx(cfact, dimsToQuery, cfacts.size() > 1);
- exprDimTables.addAll(factExprDimTables);
+ exprDimensions.addAll(factExprDimTables);
if (cfacts.size() > 1) {
factDimMap.get(cfact).addAll(factExprDimTables);
}
}
} else {
// dim only query
- exprDimTables.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, false));
+ exprDimensions.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, false));
}
- dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimTables));
+ dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimensions));
// pick denorm tables for the picked fact and dimensions
Set<Dimension> denormTables = new HashSet<Dimension>();
@@ -927,7 +926,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
if (split.length <= 1) {
col = col.trim().toLowerCase();
if (queriedExprs.contains(col)) {
- return exprCtx.getExpressionContext(col, getAliasForTableName(cube.getName())).isHasMeasures();
+ return exprCtx.getExpressionContext(col, getAliasForTableName(cube.getName())).hasMeasures();
} else {
return cube.getMeasureNames().contains(col);
}
@@ -936,7 +935,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
String colName = split[1].trim().toLowerCase();
if (cubeName.equalsIgnoreCase(cube.getName()) || cubeName.equals(getAliasForTableName(cube.getName()))) {
if (queriedExprs.contains(colName)) {
- return exprCtx.getExpressionContext(colName, cubeName).isHasMeasures();
+ return exprCtx.getExpressionContext(colName, cubeName).hasMeasures();
} else {
return cube.getMeasureNames().contains(colName.toLowerCase());
}
@@ -946,7 +945,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
}
}
- boolean isMeasure(ASTNode node) {
+ boolean isCubeMeasure(ASTNode node) {
String tabname = null;
String colname;
int nodeType = node.getToken().getType();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/099d19e8/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index f2a7039..539badb 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -70,9 +70,12 @@ class ExpressionResolver implements ContextRewriter {
private Set<CandidateTable> directlyAvailableIn = new HashSet<CandidateTable>();
private Map<CandidateTable, Set<ExprSpecContext>> evaluableExpressions =
new HashMap<CandidateTable, Set<ExprSpecContext>>();
- @Getter
private boolean hasMeasures = false;
+ public boolean hasMeasures() {
+ return hasMeasures;
+ }
+
ExpressionContext(CubeQueryContext cubeql, ExprColumn exprCol, AbstractBaseTable srcTable, String srcAlias)
throws SemanticException {
this.srcTable = srcTable;
@@ -82,13 +85,12 @@ class ExpressionResolver implements ContextRewriter {
allExprs.add(new ExprSpecContext(es, cubeql));
}
resolveColumnsAndAlias(cubeql);
+ log.debug("All exprs for {} are {}", exprCol.getName(), allExprs);
}
private void resolveColumnsAndAlias(CubeQueryContext cubeql) throws SemanticException {
for (ExprSpecContext esc : allExprs) {
esc.resolveColumns(cubeql);
- esc.replaceAliasInAST(cubeql, cubeql.getColToTableAlias());
- }
- for (ExprSpecContext esc : allExprs) {
+ esc.replaceAliasInAST(cubeql);
for (String table : esc.getTblAliasToColumns().keySet()) {
try {
if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) {
@@ -101,21 +103,18 @@ class ExpressionResolver implements ContextRewriter {
}
}
}
- resolveColumnsAndReplaceAlias(cubeql, allExprs, cubeql.getColToTableAlias());
+ resolveColumnsAndReplaceAlias(cubeql, allExprs);
}
- private void resolveColumnsAndReplaceAlias(CubeQueryContext cubeql, Set<ExprSpecContext> exprs,
- Map<String, String> colToTableAlias) throws SemanticException {
+ private void resolveColumnsAndReplaceAlias(CubeQueryContext cubeql, Set<ExprSpecContext> exprs)
+ throws SemanticException {
Set<ExprSpecContext> nestedExpressions = new LinkedHashSet<ExprSpecContext>();
for (ExprSpecContext esc : exprs) {
- for (Map.Entry<String, Set<String>> entry : esc.tblAliasToColumns.entrySet()) {
+ for (Map.Entry<String, Set<String>> entry : esc.getTblAliasToColumns().entrySet()) {
if (entry.getKey().equals(CubeQueryContext.DEFAULT_TABLE)) {
continue;
}
AbstractBaseTable baseTable = (AbstractBaseTable)cubeql.getCubeTableForAlias(entry.getKey());
- // if (baseTable == null) {
- // continue;
- // }
Set<String> exprCols = new HashSet<String>();
for (String col : entry.getValue()) {
// col is an expression
@@ -129,11 +128,11 @@ class ExpressionResolver implements ContextRewriter {
}
for (ExprSpecContext esc : nestedExpressions) {
esc.resolveColumns(cubeql);
- esc.replaceAliasInAST(cubeql, colToTableAlias);
+ esc.replaceAliasInAST(cubeql);
for (String table : esc.getTblAliasToColumns().keySet()) {
try {
if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) {
- cubeql.addOptionalDimTable(table, null, true,
+ cubeql.addOptionalDimTable(table, null, false,
esc.getTblAliasToColumns().get(table).toArray(new String[0]));
esc.exprDims.add((Dimension) cubeql.getCubeTableForAlias(table));
}
@@ -217,10 +216,10 @@ class ExpressionResolver implements ContextRewriter {
exprSpecs.add(current);
finalAST = replaceAlias(node, cubeql);
}
- public void replaceAliasInAST(CubeQueryContext cubeql, Map<String, String> colToTableAlias)
+ public void replaceAliasInAST(CubeQueryContext cubeql)
throws SemanticException {
- AliasReplacer.extractTabAliasForCol(colToTableAlias, cubeql, this);
- AliasReplacer.replaceAliases(finalAST, 0, colToTableAlias);
+ AliasReplacer.extractTabAliasForCol(cubeql, this);
+ AliasReplacer.replaceAliases(finalAST, 0, cubeql.getColToTableAlias());
}
public void addColumnsQueried(String alias, String column) {
Set<String> cols = tblAliasToColumns.get(alias.toLowerCase());
@@ -368,12 +367,14 @@ class ExpressionResolver implements ContextRewriter {
if (!cTable.getColumns().contains(col.toLowerCase())) {
if (!cubeql.getDeNormCtx().addRefUsage(cTable, col, cTable.getBaseTable().getName())) {
// check if it is available as reference, if not expression is not evaluable
+ log.debug("{} = {} is not evaluable in {}", expr, esc, cTable);
isEvaluable = false;
break;
}
}
}
if (isEvaluable) {
+ log.debug("{} = {} is evaluable in {}", expr, esc, cTable);
ec.addEvaluable(cubeql, cTable, esc);
}
}