You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by de...@apache.org on 2016/02/03 13:10:50 UTC
[30/51] [abbrv] lens git commit: LENS-917 : Fixs table pruning for
multiple chains for same destination table
LENS-917 : Fixs table pruning for multiple chains for same destination table
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c7451f8e
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c7451f8e
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c7451f8e
Branch: refs/heads/current-release-line
Commit: c7451f8e8e8f429fc55458b03dbb10a2b7428be9
Parents: 9c03c76
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Tue Jan 12 11:09:24 2016 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Tue Jan 12 11:09:24 2016 +0530
----------------------------------------------------------------------
.../org/apache/lens/cube/parse/Aliased.java | 4 +
.../lens/cube/parse/CandidateTableResolver.java | 150 +++++++++----------
.../lens/cube/parse/CubeQueryContext.java | 43 +++---
.../lens/cube/parse/ExpressionResolver.java | 3 +
.../lens/cube/parse/StorageTableResolver.java | 4 +-
.../lens/cube/parse/join/AutoJoinContext.java | 12 +-
.../apache/lens/cube/parse/CubeTestSetup.java | 28 ++++
.../cube/parse/TestDenormalizationResolver.java | 16 +-
8 files changed, 152 insertions(+), 108 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/c7451f8e/lens-cube/src/main/java/org/apache/lens/cube/parse/Aliased.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/Aliased.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/Aliased.java
index 56fe9fc..160a9c6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/Aliased.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/Aliased.java
@@ -36,4 +36,8 @@ public class Aliased<T extends Named> {
public static <K extends Named> Aliased<K> create(K obj, String alias) {
return new Aliased<K>(obj, alias);
}
+
+ public String getName() {
+ return object.getName();
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/c7451f8e/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index 38ff5a4..00ccf36 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -67,9 +67,9 @@ class CandidateTableResolver implements ContextRewriter {
checkForQueriedColumns = false;
} else {
// populate optional tables
- for (Dimension dim : cubeql.getOptionalDimensions()) {
+ for (Aliased<Dimension> dim : cubeql.getOptionalDimensions()) {
log.info("Populating optional dim:{}", dim);
- populateDimTables(dim, cubeql, true);
+ populateDimTables(dim.getObject(), cubeql, true);
}
if (cubeql.getAutoJoinCtx() != null) {
// Before checking for candidate table columns, prune join paths containing non existing columns
@@ -117,7 +117,7 @@ class CandidateTableResolver implements ContextRewriter {
return;
}
try {
- Set<CandidateDim> candidates = new HashSet<CandidateDim>();
+ Set<CandidateDim> candidates = new HashSet<>();
cubeql.getCandidateDimTables().put(dim, candidates);
List<CubeDimensionTable> dimtables = cubeql.getMetastoreClient().getAllDimensionTables(dim);
if (dimtables.isEmpty()) {
@@ -126,7 +126,7 @@ class CandidateTableResolver implements ContextRewriter {
"Dimension tables do not exist");
} else {
log.info("Not considering optional dimension {} as, No dimension tables exist", dim);
- removeOptionalDim(cubeql, dim);
+ removeOptionalDimWithoutAlias(cubeql, dim);
}
}
for (CubeDimensionTable dimtable : dimtables) {
@@ -139,44 +139,30 @@ class CandidateTableResolver implements ContextRewriter {
}
}
- private void pruneOptionalDims(CubeQueryContext cubeql) {
- Set<Dimension> tobeRemoved = new HashSet<Dimension>();
- Set<CandidateTable> allCandidates = new HashSet<CandidateTable>();
- allCandidates.addAll(cubeql.getCandidateFacts());
- for (Set<CandidateDim> cdims : cubeql.getCandidateDimTables().values()) {
- allCandidates.addAll(cdims);
- }
- Set<CandidateTable> removedCandidates = new HashSet<CandidateTable>();
- for (Map.Entry<Dimension, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) {
- Dimension dim = optdimEntry.getKey();
- OptionalDimCtx optdim = optdimEntry.getValue();
- Iterator<CandidateTable> iter = optdim.requiredForCandidates.iterator();
- while (iter.hasNext()) {
- CandidateTable candidate = iter.next();
- if (!allCandidates.contains(candidate)) {
- log.info("Removing candidate {} from requiredForCandidates of {}, as it is no more candidate", candidate,
- dim);
- iter.remove();
- removedCandidates.add(candidate);
- }
+ private void removeOptionalDimWithoutAlias(CubeQueryContext cubeql, Dimension dim) {
+ for (Aliased<Dimension> aDim : cubeql.getOptionalDimensions()) {
+ if (aDim.getName().equals(dim.getName())) {
+ removeOptionalDim(cubeql, aDim);
}
}
- Set<CandidateTable> candidatesReachableThroughRefs = new HashSet<CandidateTable>();
- for (Map.Entry<Dimension, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) {
- Dimension dim = optdimEntry.getKey();
+ }
+
+ private void pruneOptionalDims(CubeQueryContext cubeql) {
+ Set<Aliased<Dimension>> tobeRemoved = new HashSet<>();
+ for (Map.Entry<Aliased<Dimension>, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) {
+ Aliased<Dimension> dim = optdimEntry.getKey();
OptionalDimCtx optdim = optdimEntry.getValue();
- candidatesReachableThroughRefs.addAll(optdim.requiredForCandidates);
if ((!optdim.colQueried.isEmpty() && optdim.requiredForCandidates.isEmpty()) && !optdim.isRequiredInJoinChain) {
log.info("Not considering optional dimension {} as all requiredForCandidates are removed", dim);
tobeRemoved.add(dim);
}
}
- for (Dimension dim : tobeRemoved) {
+ for (Aliased<Dimension> dim : tobeRemoved) {
removeOptionalDim(cubeql, dim);
}
}
- private void removeOptionalDim(CubeQueryContext cubeql, Dimension dim) {
+ private void removeOptionalDim(CubeQueryContext cubeql, Aliased<Dimension> dim) {
OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().remove(dim);
// remove all the depending candidate table as well
for (CandidateTable candidate : optdim.requiredForCandidates) {
@@ -240,7 +226,8 @@ class CandidateTableResolver implements ContextRewriter {
// go over join chains and prune facts that dont have any of the columns in each chain
for (JoinChain chain : cubeql.getJoinchains().values()) {
- OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(cubeql.getCubeTbls().get(chain.getName()));
+ OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(Aliased.create((Dimension)cubeql.getCubeTbls()
+ .get(chain.getName()), chain.getName()));
if (!checkForColumnExists(cfact, chain.getSourceColumns())) {
// check if chain is optional or not
if (optdim == null) {
@@ -282,7 +269,7 @@ class CandidateTableResolver implements ContextRewriter {
i.remove();
}
}
- Set<String> dimExprs = new HashSet<String>(cubeql.getQueriedExprs());
+ Set<String> dimExprs = new HashSet<>(cubeql.getQueriedExprs());
dimExprs.removeAll(cubeql.getQueriedExprsWithMeasures());
if (cubeql.getCandidateFacts().size() == 0) {
throw new LensException(LensCubeErrorCode.NO_FACT_HAS_COLUMN.getLensErrorInfo(),
@@ -292,9 +279,9 @@ class CandidateTableResolver implements ContextRewriter {
Set<Set<CandidateFact>> cfactset;
if (queriedMsrs.isEmpty() && cubeql.getQueriedExprsWithMeasures().isEmpty()) {
// if no measures are queried, add all facts individually as single covering sets
- cfactset = new HashSet<Set<CandidateFact>>();
+ cfactset = new HashSet<>();
for (CandidateFact cfact : cubeql.getCandidateFacts()) {
- Set<CandidateFact> one = new LinkedHashSet<CandidateFact>();
+ Set<CandidateFact> one = new LinkedHashSet<>();
one.add(cfact);
cfactset.add(one);
}
@@ -302,7 +289,7 @@ class CandidateTableResolver implements ContextRewriter {
} else {
// Find out candidate fact table sets which contain all the measures
// queried
- List<CandidateFact> cfacts = new ArrayList<CandidateFact>(cubeql.getCandidateFacts());
+ List<CandidateFact> cfacts = new ArrayList<>(cubeql.getCandidateFacts());
cfactset = findCoveringSets(cubeql, cfacts, queriedMsrs,
cubeql.getQueriedExprsWithMeasures());
log.info("Measure covering fact sets :{}", cfactset);
@@ -324,8 +311,8 @@ class CandidateTableResolver implements ContextRewriter {
static Set<Set<CandidateFact>> findCoveringSets(CubeQueryContext cubeql, List<CandidateFact> cfactsPassed,
Set<String> msrs, Set<String> exprsWithMeasures) {
- Set<Set<CandidateFact>> cfactset = new HashSet<Set<CandidateFact>>();
- List<CandidateFact> cfacts = new ArrayList<CandidateFact>(cfactsPassed);
+ Set<Set<CandidateFact>> cfactset = new HashSet<>();
+ List<CandidateFact> cfacts = new ArrayList<>(cfactsPassed);
for (Iterator<CandidateFact> i = cfacts.iterator(); i.hasNext();) {
CandidateFact cfact = i.next();
i.remove();
@@ -336,14 +323,14 @@ class CandidateTableResolver implements ContextRewriter {
continue;
} else if (cfact.getColumns().containsAll(msrs) && cubeql.getExprCtx().allEvaluable(cfact, exprsWithMeasures)) {
// return single set
- Set<CandidateFact> one = new LinkedHashSet<CandidateFact>();
+ Set<CandidateFact> one = new LinkedHashSet<>();
one.add(cfact);
cfactset.add(one);
} else {
// find the remaining measures in other facts
if (i.hasNext()) {
- Set<String> remainingMsrs = new HashSet<String>(msrs);
- Set<String> remainingExprs = new HashSet<String>(exprsWithMeasures);
+ Set<String> remainingMsrs = new HashSet<>(msrs);
+ Set<String> remainingExprs = new HashSet<>(exprsWithMeasures);
remainingMsrs.removeAll(cfact.getColumns());
remainingExprs.removeAll(cubeql.getExprCtx().coveringExpressions(exprsWithMeasures, cfact));
Set<Set<CandidateFact>> coveringSets = findCoveringSets(cubeql, cfacts, remainingMsrs, remainingExprs);
@@ -366,9 +353,13 @@ class CandidateTableResolver implements ContextRewriter {
if (cubeql.getAutoJoinCtx() == null) {
return;
}
- Set<Dimension> allDims = new HashSet<Dimension>(cubeql.getDimensions());
+ Set<Aliased<Dimension>> allDims = new HashSet<>();
+ for (Dimension dim : cubeql.getDimensions()) {
+ allDims.add(Aliased.create(dim));
+ }
allDims.addAll(cubeql.getOptionalDimensions());
- for (Dimension dim : allDims) {
+ for (Aliased<Dimension> aliasedDim : allDims) {
+ Dimension dim = aliasedDim.getObject();
if (cubeql.getCandidateDimTables().get(dim) != null && !cubeql.getCandidateDimTables().get(dim).isEmpty()) {
for (Iterator<CandidateDim> i = cubeql.getCandidateDimTables().get(dim).iterator(); i.hasNext();) {
CandidateDim cdim = i.next();
@@ -377,11 +368,11 @@ class CandidateTableResolver implements ContextRewriter {
// can participate in join
// for each join path check for columns involved in path
boolean removed = false;
- for (Map.Entry<Dimension, Map<AbstractCubeTable, List<String>>> joincolumnsEntry : cubeql.getAutoJoinCtx()
- .getJoinPathFromColumns().entrySet()) {
- Dimension reachableDim = joincolumnsEntry.getKey();
+ for (Map.Entry<Aliased<Dimension>, Map<AbstractCubeTable, List<String>>> joincolumnsEntry : cubeql
+ .getAutoJoinCtx().getJoinPathFromColumns().entrySet()) {
+ Aliased<Dimension> reachableDim = joincolumnsEntry.getKey();
OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(reachableDim);
- Collection<String> colSet = joincolumnsEntry.getValue().get((AbstractCubeTable) dim);
+ Collection<String> colSet = joincolumnsEntry.getValue().get(dim);
if (!checkForColumnExists(cdim, colSet)) {
if (optdim == null || optdim.isRequiredInJoinChain
@@ -397,11 +388,11 @@ class CandidateTableResolver implements ContextRewriter {
}
if (!removed) {
// check for to columns
- for (Map.Entry<Dimension, Map<AbstractCubeTable, List<String>>> joincolumnsEntry : cubeql.getAutoJoinCtx()
- .getJoinPathToColumns().entrySet()) {
- Dimension reachableDim = joincolumnsEntry.getKey();
+ for (Map.Entry<Aliased<Dimension>, Map<AbstractCubeTable, List<String>>> joincolumnsEntry : cubeql
+ .getAutoJoinCtx().getJoinPathToColumns().entrySet()) {
+ Aliased<Dimension> reachableDim = joincolumnsEntry.getKey();
OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(reachableDim);
- Collection<String> colSet = joincolumnsEntry.getValue().get((AbstractCubeTable) dim);
+ Collection<String> colSet = joincolumnsEntry.getValue().get(dim);
if (!checkForColumnExists(cdim, colSet)) {
if (optdim == null || optdim.isRequiredInJoinChain
@@ -418,18 +409,18 @@ class CandidateTableResolver implements ContextRewriter {
}
if (!removed) {
// go over the referenced columns accessed in the query and find out which tables can participate
- if (cubeql.getOptionalDimensionMap().get(dim) != null
- && !checkForColumnExists(cdim, cubeql.getOptionalDimensionMap().get(dim).colQueried)) {
+ if (cubeql.getOptionalDimensionMap().get(aliasedDim) != null
+ && !checkForColumnExists(cdim, cubeql.getOptionalDimensionMap().get(aliasedDim).colQueried)) {
i.remove();
log.info("Not considering optional dimtable:{} as its denorm fields do not exist. Denorm fields:{}",
- dimtable, cubeql.getOptionalDimensionMap().get(dim).colQueried);
- cubeql.addDimPruningMsgs(dim, dimtable,
- CandidateTablePruneCause.noColumnPartOfAJoinPath(cubeql.getOptionalDimensionMap().get(dim).colQueried));
+ dimtable, cubeql.getOptionalDimensionMap().get(aliasedDim).colQueried);
+ cubeql.addDimPruningMsgs(dim, dimtable, CandidateTablePruneCause
+ .noColumnPartOfAJoinPath(cubeql.getOptionalDimensionMap().get(aliasedDim).colQueried));
}
}
}
if (cubeql.getCandidateDimTables().get(dim).size() == 0) {
- OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(dim);
+ OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(aliasedDim);
if ((cubeql.getDimensions() != null && cubeql.getDimensions().contains(dim))
|| (optdim != null && optdim.isRequiredInJoinChain)) {
throw new LensException(LensCubeErrorCode.NO_DIM_HAS_COLUMN.getLensErrorInfo(), dim.getName(),
@@ -438,7 +429,7 @@ class CandidateTableResolver implements ContextRewriter {
// remove it from optional tables
log.info("Not considering optional dimension {} as, No dimension table has the queried columns:{}"
+ " Clearing the required for candidates:{}", dim, optdim.colQueried, optdim.requiredForCandidates);
- removeOptionalDim(cubeql, dim);
+ removeOptionalDim(cubeql, aliasedDim);
}
}
}
@@ -456,11 +447,12 @@ class CandidateTableResolver implements ContextRewriter {
CubeFactTable fact = cfact.fact;
// for each join path check for columns involved in path
- for (Map.Entry<Dimension, Map<AbstractCubeTable, List<String>>> joincolumnsEntry : cubeql.getAutoJoinCtx()
+ for (Map.Entry<Aliased<Dimension>, Map<AbstractCubeTable, List<String>>> joincolumnsEntry : cubeql
+ .getAutoJoinCtx()
.getJoinPathFromColumns().entrySet()) {
- Dimension reachableDim = joincolumnsEntry.getKey();
+ Aliased<Dimension> reachableDim = joincolumnsEntry.getKey();
OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(reachableDim);
- colSet = joincolumnsEntry.getValue().get((AbstractCubeTable) cubeql.getCube());
+ colSet = joincolumnsEntry.getValue().get(cubeql.getCube());
if (!checkForColumnExists(cfact, colSet)) {
if (optdim == null || optdim.isRequiredInJoinChain
@@ -486,21 +478,21 @@ class CandidateTableResolver implements ContextRewriter {
* available in candidate tables that want to use references
*/
private void checkForSourceReachabilityForDenormCandidates(CubeQueryContext cubeql) {
- if (cubeql.getOptionalDimensionMap().isEmpty()) {
+ if (cubeql.getOptionalDimensions().isEmpty()) {
return;
}
if (cubeql.getAutoJoinCtx() == null) {
- Set<Dimension> optionaldims = new HashSet<Dimension>(cubeql.getOptionalDimensions());
- for (Dimension dim : optionaldims) {
+ Set<Aliased<Dimension>> optionaldims = new HashSet<>(cubeql.getOptionalDimensions());
+ for (Aliased<Dimension> dim : optionaldims) {
log.info("Not considering optional dimension {} as, automatic join resolver is disbled ", dim);
removeOptionalDim(cubeql, dim);
}
return;
}
// check for source columns for denorm columns
- Map<Dimension, Set<CandidateTable>> removedCandidates = new HashMap<Dimension, Set<CandidateTable>>();
- for (Map.Entry<Dimension, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) {
- Dimension dim = optdimEntry.getKey();
+ Map<Aliased<Dimension>, Set<CandidateTable>> removedCandidates = new HashMap<>();
+ for (Map.Entry<Aliased<Dimension>, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) {
+ Aliased<Dimension> dim = optdimEntry.getKey();
OptionalDimCtx optdim = optdimEntry.getValue();
Iterator<CandidateTable> iter = optdim.requiredForCandidates.iterator();
// remove candidates from each optional dim if the dimension is not reachable from candidate
@@ -537,17 +529,17 @@ class CandidateTableResolver implements ContextRewriter {
// F5 | Directly available | Directly available
// F6 | Directly available | Not reachable
// F3 and F4 will get pruned while iterating over col1 and F1, F6 will get pruned while iterating over col2.
- for (Map.Entry<String, Set<Dimension>> dimColEntry : cubeql.getRefColToDim().entrySet()) {
- Set<CandidateTable> candidatesReachableThroughRefs = new HashSet<CandidateTable>();
+ for (Map.Entry<String, Set<Aliased<Dimension>>> dimColEntry : cubeql.getRefColToDim().entrySet()) {
+ Set<CandidateTable> candidatesReachableThroughRefs = new HashSet<>();
String col = dimColEntry.getKey();
- Set<Dimension> dimSet = dimColEntry.getValue();
- for (Dimension dim : dimSet) {
+ Set<Aliased<Dimension>> dimSet = dimColEntry.getValue();
+ for (Aliased<Dimension> dim : dimSet) {
OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(dim);
if (optdim != null) {
candidatesReachableThroughRefs.addAll(optdim.requiredForCandidates);
}
}
- for (Dimension dim : dimSet) {
+ for (Aliased<Dimension> dim : dimSet) {
if (removedCandidates.get(dim) != null) {
for (CandidateTable candidate : removedCandidates.get(dim)) {
if (!candidatesReachableThroughRefs.contains(candidate)) {
@@ -582,11 +574,11 @@ class CandidateTableResolver implements ContextRewriter {
// F4 | Not evaluable | evaluable through D6
// F5 | Directly available | Directly available
// F6 | Directly available | Not evaluable
- for (Map.Entry<QueriedExprColumn, Set<Dimension>> exprColEntry : cubeql.getExprColToDim().entrySet()) {
+ for (Map.Entry<QueriedExprColumn, Set<Aliased<Dimension>>> exprColEntry : cubeql.getExprColToDim().entrySet()) {
QueriedExprColumn col = exprColEntry.getKey();
- Set<Dimension> dimSet = exprColEntry.getValue();
+ Set<Aliased<Dimension>> dimSet = exprColEntry.getValue();
ExpressionContext ec = cubeql.getExprCtx().getExpressionContext(col.getExprCol(), col.getAlias());
- for (Dimension dim : dimSet) {
+ for (Aliased<Dimension> dim : dimSet) {
if (removedCandidates.get(dim) != null) {
for (CandidateTable candidate : removedCandidates.get(dim)) {
// check if evaluable expressions of this candidate are no more evaluable because dimension is not reachable
@@ -595,7 +587,7 @@ class CandidateTableResolver implements ContextRewriter {
Iterator<ExprSpecContext> escIter = ec.getEvaluableExpressions().get(candidate).iterator();
while (escIter.hasNext()) {
ExprSpecContext esc = escIter.next();
- if (esc.getExprDims().contains(dim)) {
+ if (esc.getExprDims().contains(dim.getObject())) {
escIter.remove();
}
}
@@ -623,16 +615,16 @@ class CandidateTableResolver implements ContextRewriter {
}
// remove optional dims which are not required any more.
- Set<Dimension> tobeRemoved = new HashSet<Dimension>();
- for (Map.Entry<Dimension, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) {
- Dimension dim = optdimEntry.getKey();
+ Set<Aliased<Dimension>> tobeRemoved = new HashSet<>();
+ for (Map.Entry<Aliased<Dimension>, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) {
+ Aliased<Dimension> dim = optdimEntry.getKey();
OptionalDimCtx optdim = optdimEntry.getValue();
if ((!optdim.colQueried.isEmpty() && optdim.requiredForCandidates.isEmpty()) && !optdim.isRequiredInJoinChain) {
log.info("Not considering optional dimension {} as all requiredForCandidates are removed", dim);
tobeRemoved.add(dim);
}
}
- for (Dimension dim : tobeRemoved) {
+ for (Aliased<Dimension> dim : tobeRemoved) {
removeOptionalDim(cubeql, dim);
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/c7451f8e/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 06c2a0b..3e930de 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -96,13 +96,14 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST {
@Getter
// Mapping of a qualified column name to its table alias
- private final Map<String, String> colToTableAlias = new HashMap<String, String>();
+ private final Map<String, String> colToTableAlias = new HashMap<>();
- @Getter()
- private final Set<Set<CandidateFact>> candidateFactSets = new HashSet<Set<CandidateFact>>();
+ @Getter
+ private final Set<Set<CandidateFact>> candidateFactSets = new HashSet<>();
+ @Getter
// would be added through join chains and de-normalized resolver
- protected Map<Dimension, OptionalDimCtx> optionalDimensions = new HashMap<Dimension, OptionalDimCtx>();
+ protected Map<Aliased<Dimension>, OptionalDimCtx> optionalDimensionMap = new HashMap<>();
// Alias to table object mapping of tables accessed in this query
@Getter
@@ -351,10 +352,10 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST {
// map of ref column in query to set of Dimension that have the column - which are added as optional dims
@Getter
- private Map<String, Set<Dimension>> refColToDim = Maps.newHashMap();
+ private Map<String, Set<Aliased<Dimension>>> refColToDim = Maps.newHashMap();
- public void updateRefColDim(String col, Dimension dim) {
- Set<Dimension> refDims = refColToDim.get(col.toLowerCase());
+ public void updateRefColDim(String col, Aliased<Dimension> dim) {
+ Set<Aliased<Dimension>> refDims = refColToDim.get(col.toLowerCase());
if (refDims == null) {
refDims = Sets.newHashSet();
refColToDim.put(col.toLowerCase(), refDims);
@@ -371,12 +372,12 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST {
// map of expression column in query to set of Dimension that are accessed in the expression column - which are added
// as optional dims
@Getter
- private Map<QueriedExprColumn, Set<Dimension>> exprColToDim = Maps.newHashMap();
+ private Map<QueriedExprColumn, Set<Aliased<Dimension>>> exprColToDim = Maps.newHashMap();
- public void updateExprColDim(String tblAlias, String col, Dimension dim) {
+ public void updateExprColDim(String tblAlias, String col, Aliased<Dimension> dim) {
QueriedExprColumn qexpr = new QueriedExprColumn(col, tblAlias);
- Set<Dimension> exprDims = exprColToDim.get(qexpr);
+ Set<Aliased<Dimension>> exprDims = exprColToDim.get(qexpr);
if (exprDims == null) {
exprDims = Sets.newHashSet();
exprColToDim.put(qexpr, exprDims);
@@ -419,10 +420,11 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST {
throw new LensException(LensCubeErrorCode.QUERIED_TABLE_NOT_FOUND.getLensErrorInfo(), alias);
}
Dimension dim = (Dimension) cubeTbls.get(alias);
- OptionalDimCtx optDim = optionalDimensions.get(dim);
+ Aliased<Dimension> aliasedDim = Aliased.create(dim, alias);
+ OptionalDimCtx optDim = optionalDimensionMap.get(aliasedDim);
if (optDim == null) {
optDim = new OptionalDimCtx();
- optionalDimensions.put(dim, optDim);
+ optionalDimensionMap.put(aliasedDim, optDim);
}
if (cols != null && candidate != null) {
for (String col : cols) {
@@ -432,16 +434,16 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST {
}
if (cubeCol != null) {
if (isRef) {
- updateRefColDim(cubeCol, dim);
+ updateRefColDim(cubeCol, aliasedDim);
} else {
- updateExprColDim(tableAlias, cubeCol, dim);
+ updateExprColDim(tableAlias, cubeCol, aliasedDim);
}
}
if (!optDim.isRequiredInJoinChain) {
optDim.isRequiredInJoinChain = isRequiredInJoin;
}
if (log.isDebugEnabled()) {
- log.debug("Adding optional dimension:{} optDim:{} {} isRef:{}", dim, optDim,
+ log.debug("Adding optional dimension:{} optDim:{} {} isRef:{}", aliasedDim, optDim,
(cubeCol == null ? "" : " for column:" + cubeCol), isRef);
}
}
@@ -872,6 +874,7 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST {
public String toHQL() throws LensException {
Set<CandidateFact> cfacts = pickCandidateFactToQuery();
Map<Dimension, CandidateDim> dimsToQuery = pickCandidateDimsToQuery(dimensions);
+ log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery);
if (autoJoinCtx != null) {
// prune join paths for picked fact and dimensions
autoJoinCtx.pruneAllPaths(cube, cfacts, dimsToQuery);
@@ -908,6 +911,7 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST {
exprDimensions.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, this));
}
dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimensions));
+ log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery);
// pick denorm tables for the picked fact and dimensions
Set<Dimension> denormTables = new HashSet<Dimension>();
@@ -923,6 +927,7 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST {
denormTables.addAll(deNormCtx.rewriteDenormctx(null, dimsToQuery, false));
}
dimsToQuery.putAll(pickCandidateDimsToQuery(denormTables));
+ log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery);
// Prune join paths once denorm tables are picked
if (autoJoinCtx != null) {
// prune join paths for picked fact and dimensions
@@ -1129,12 +1134,8 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST {
}
}
- public Set<Dimension> getOptionalDimensions() {
- return optionalDimensions.keySet();
- }
-
- public Map<Dimension, OptionalDimCtx> getOptionalDimensionMap() {
- return optionalDimensions;
+ public Set<Aliased<Dimension>> getOptionalDimensions() {
+ return optionalDimensionMap.keySet();
}
/**
http://git-wip-us.apache.org/repos/asf/lens/blob/c7451f8e/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 26514d8..5ff265d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -151,6 +151,7 @@ class ExpressionResolver implements ContextRewriter {
}
void addDirectlyAvailable(CandidateTable cTable) {
+ log.debug("Directly available in {}", cTable);
directlyAvailableIn.add(cTable);
}
@@ -447,6 +448,7 @@ class ExpressionResolver implements ContextRewriter {
}
// Replace picked expressions in all the base trees
replacePickedExpressions(queryAST);
+ log.debug("Picked expressions: {}", pickedExpressions);
for (Set<PickedExpression> peSet : pickedExpressions.values()) {
for (PickedExpression pe : peSet) {
exprDims.addAll(pe.pickedCtx.exprDims);
@@ -518,6 +520,7 @@ class ExpressionResolver implements ContextRewriter {
for (ExpressionContext ec : ecSet) {
if (ec.getSrcTable().getName().equals(cTable.getBaseTable().getName())) {
if (!ec.directlyAvailableIn.contains(cTable)) {
+ log.debug("{} is not directly evaluable in {}", ec, cTable);
if (ec.evaluableExpressions.get(cTable) != null && !ec.evaluableExpressions.get(cTable).isEmpty()) {
// pick first evaluable expression
Set<PickedExpression> peSet = pickedExpressions.get(ecEntry.getKey());
http://git-wip-us.apache.org/repos/asf/lens/blob/c7451f8e/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index de5f95e..46b6bb7 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -156,7 +156,9 @@ class StorageTableResolver implements ContextRewriter {
private void resolveDimStorageTablesAndPartitions(CubeQueryContext cubeql) throws LensException {
Set<Dimension> allDims = new HashSet<Dimension>(cubeql.getDimensions());
- allDims.addAll(cubeql.getOptionalDimensions());
+ for (Aliased<Dimension> dim : cubeql.getOptionalDimensions()) {
+ allDims.add(dim.getObject());
+ }
for (Dimension dim : allDims) {
Set<CandidateDim> dimTables = cubeql.getCandidateDimTables().get(dim);
if (dimTables == null || dimTables.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/lens/blob/c7451f8e/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java
index 993955a..4c30d3f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java
@@ -57,10 +57,10 @@ public class AutoJoinContext {
// Map of a joined table to its columns which are part of any of the join
// paths. This is used in candidate table resolver
@Getter
- private Map<Dimension, Map<AbstractCubeTable, List<String>>> joinPathFromColumns = new HashMap<>();
+ private Map<Aliased<Dimension>, Map<AbstractCubeTable, List<String>>> joinPathFromColumns = new HashMap<>();
@Getter
- private Map<Dimension, Map<AbstractCubeTable, List<String>>> joinPathToColumns = new HashMap<>();
+ private Map<Aliased<Dimension>, Map<AbstractCubeTable, List<String>>> joinPathToColumns = new HashMap<>();
// there can be separate join clause for each fact in-case of multi fact queries
@Getter
@@ -122,12 +122,12 @@ public class AutoJoinContext {
Map<AbstractCubeTable, List<String>> toColPaths = joinPathToColumns.get(joinPathEntry.getKey().getObject());
if (fromColPaths == null) {
fromColPaths = new HashMap<>();
- joinPathFromColumns.put(joinPathEntry.getKey().getObject(), fromColPaths);
+ joinPathFromColumns.put(joinPathEntry.getKey(), fromColPaths);
}
if (toColPaths == null) {
toColPaths = new HashMap<>();
- joinPathToColumns.put(joinPathEntry.getKey().getObject(), toColPaths);
+ joinPathToColumns.put(joinPathEntry.getKey(), toColPaths);
}
populateJoinPathCols(joinPaths, fromColPaths, toColPaths);
}
@@ -159,8 +159,8 @@ public class AutoJoinContext {
}
}
- public void removeJoinedTable(Dimension dim) {
- allPaths.remove(Aliased.create(dim));
+ public void removeJoinedTable(Aliased<Dimension> dim) {
+ allPaths.remove(dim);
joinPathFromColumns.remove(dim);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/c7451f8e/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 4366938..caea3af 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -563,6 +563,10 @@ public class CubeTestSetup {
chainRefs.add(new ChainRefCol("timedatechain2", "full_date"));
cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("test_time_dim2", "date", "chained dim"),
"Timedim full date", chainRefs, null, null, null, null));
+ cubeDimensions.add(new BaseDimAttribute(new FieldSchema("cityid1", "int", "id to city"),
+ "City1", null, null, null));
+ cubeDimensions.add(new BaseDimAttribute(new FieldSchema("cityid2", "int", "id to city"),
+ "City2", null, null, null));
Map<String, JoinChain> joinChains = new HashMap<>();
addCubeChains(joinChains, TEST_CUBE_NAME);
@@ -717,6 +721,26 @@ public class CubeTestSetup {
});
}
});
+ joinChains.put("cubeCity1", new JoinChain("cubeCity1", "cube-city", "city thru cube") {
+ {
+ addPath(new ArrayList<TableReference>() {
+ {
+ add(new TableReference(cubeName, "cityid1"));
+ add(new TableReference("citydim", "id"));
+ }
+ });
+ }
+ });
+ joinChains.put("cubeCity2", new JoinChain("cubeCity2", "cube-city", "city thru cube") {
+ {
+ addPath(new ArrayList<TableReference>() {
+ {
+ add(new TableReference(cubeName, "cityid2"));
+ add(new TableReference("citydim", "id"));
+ }
+ });
+ }
+ });
joinChains.put("cubeState", new JoinChain("cubeState", "cube-state", "state thru cube") {
{
addPath(new ArrayList<TableReference>() {
@@ -1345,6 +1369,7 @@ public class CubeTestSetup {
// add dimensions of the cube
factColumns.add(new FieldSchema("zipcode", "int", "zip"));
factColumns.add(new FieldSchema("cityid", "int", "city id"));
+ factColumns.add(new FieldSchema("cityid1", "int", "city id"));
factColumns.add(new FieldSchema("stateid", "int", "city id"));
factColumns.add(new FieldSchema("test_time_dim_day_id", "int", "time id"));
factColumns.add(new FieldSchema("test_time_dim_day_id2", "int", "time id"));
@@ -1580,6 +1605,7 @@ public class CubeTestSetup {
// add dimensions of the cube
factColumns.add(new FieldSchema("zipcode", "int", "zip"));
factColumns.add(new FieldSchema("cityid", "int", "city id"));
+ factColumns.add(new FieldSchema("cityid2", "int", "city id"));
factColumns.add(new FieldSchema("test_time_dim_hour_id", "int", "time id"));
factColumns.add(new FieldSchema("test_time_dim_hour_id2", "int", "time id"));
factColumns.add(new FieldSchema("cdim2", "int", "cycledim id"));
@@ -1714,6 +1740,8 @@ public class CubeTestSetup {
// add dimensions of the cube
factColumns.add(new FieldSchema("zipcode", "int", "zip"));
factColumns.add(new FieldSchema("cityid", "int", "city id"));
+ factColumns.add(new FieldSchema("cityid1", "int", "city id"));
+ factColumns.add(new FieldSchema("cityid2", "int", "city id"));
factColumns.add(new FieldSchema("stateid", "int", "state id"));
factColumns.add(new FieldSchema("countryid", "int", "country id"));
factColumns.add(new FieldSchema("dim1", "string", "dim1"));
http://git-wip-us.apache.org/repos/asf/lens/blob/c7451f8e/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
index a8390ef..d7707a9 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
@@ -267,7 +267,7 @@ public class TestDenormalizationResolver extends TestQueryRewrite {
// candidate
Assert.assertEquals(getLensExceptionErrorMessageInRewrite(
"select citydim.name, citydim.statename, citydim.nocandidatecol " + "from citydim", conf),
- "No dimension table has the queried columns " + "for citydim, columns: [name, statename, nocandidatecol]");
+ "No dimension table has the queried columns " + "for citydim, columns: [name, statename, nocandidatecol]");
}
@Test
@@ -330,6 +330,20 @@ public class TestDenormalizationResolver extends TestQueryRewrite {
}
@Test
+ public void testTwoFieldsFromDifferentChainButSameTable() throws Exception {
+ String hqlQuery = rewrite("select cubecity1.name, cubecity2.name, msr2 from testcube where " + TWO_DAYS_RANGE,
+ conf);
+ String joinExpr = " join " + getDbName()
+ + "c1_citytable cubecity1 on testcube.cityid1 = cubecity1.id and (cubecity1.dt = 'latest') "
+ + " join " + getDbName()
+ + "c1_citytable cubecity2 on testcube.cityid2 = cubecity2.id and (cubecity2.dt = 'latest')";
+ String expected =
+ getExpectedQuery("testcube", "select cubecity1.name, cubecity2.name, sum(testcube.msr2) FROM ",
+ joinExpr, null, " group by cubecity1.name, cubecity2.name", null,
+ getWhereForHourly2days("testcube", "c1_testfact2_raw"));
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
+ }
+ @Test
public void testDimensionQueryWithTwoRefCols() throws Exception {
Configuration tConf = new Configuration(conf);
tConf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "");