You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2017/03/31 09:48:10 UTC
[02/16] lens git commit: LENS-1378 : Fix dimensions to query from
DenormalizationResolver for expression fields
LENS-1378 : Fix dimensions to query from DenormalizationResolver for expression fields
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/4fb25061
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/4fb25061
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/4fb25061
Branch: refs/heads/lens-1381
Commit: 4fb2506193acd3efa7d90bb640bd50e2aefe5d6a
Parents: 98990c3
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Wed Jan 18 17:46:58 2017 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Wed Jan 18 17:46:58 2017 +0530
----------------------------------------------------------------------
.../lens/cube/parse/CandidateTableResolver.java | 2 +-
.../lens/cube/parse/CubeQueryContext.java | 11 +-
.../cube/parse/DenormalizationResolver.java | 167 +++++++++++++------
.../lens/cube/parse/ExpressionResolver.java | 107 ++++++------
.../lens/cube/parse/QueriedPhraseContext.java | 2 +-
.../lens/cube/parse/TimeRangeChecker.java | 18 --
.../lens/cube/parse/TrackDenormContext.java | 37 ++++
.../apache/lens/cube/parse/CubeTestSetup.java | 11 +-
.../lens/cube/parse/TestExpressionResolver.java | 15 ++
.../lens/cube/parse/TestJoinResolver.java | 2 +-
tools/conf/server/logback.xml | 4 +-
11 files changed, 248 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index e7fc557..e9270ea 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -697,7 +697,7 @@ class CandidateTableResolver implements ContextRewriter {
i.remove();
break;
}
- } else if (!cubeql.getDeNormCtx().addRefUsage(cdim, col, dim.getName())) {
+ } else if (!cubeql.getDeNormCtx().addRefUsage(cubeql, cdim, col, dim.getName())) {
// check if it available as reference, if not remove the
// candidate
log.info("Not considering dimtable: {} as column {} is not available", cdim, col);
http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index e83ae76..125b432 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -59,7 +59,7 @@ import lombok.*;
import lombok.extern.slf4j.Slf4j;
@Slf4j
-public class CubeQueryContext extends TracksQueriedColumns implements QueryAST {
+public class CubeQueryContext extends TracksQueriedColumns implements QueryAST, TrackDenormContext {
public static final String TIME_RANGE_FUNC = "time_range_in";
public static final String NOW = "now";
public static final String DEFAULT_TABLE = "_default_";
@@ -922,7 +922,8 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST {
Set<Dimension> exprDimensions = new HashSet<>();
if (cfacts != null) {
for (CandidateFact cfact : cfacts) {
- Set<Dimension> factExprDimTables = exprCtx.rewriteExprCtx(cfact, dimsToQuery, cfacts.size() > 1 ? cfact : this);
+ Set<Dimension> factExprDimTables = exprCtx.rewriteExprCtx(this, cfact, dimsToQuery,
+ cfacts.size() > 1 ? cfact : this);
exprDimensions.addAll(factExprDimTables);
if (cfacts.size() > 1) {
factDimMap.get(cfact).addAll(factExprDimTables);
@@ -933,7 +934,7 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST {
}
} else {
// dim only query
- exprDimensions.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, this));
+ exprDimensions.addAll(exprCtx.rewriteExprCtx(this, null, dimsToQuery, this));
}
dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimensions));
log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery);
@@ -942,14 +943,14 @@ public class CubeQueryContext extends TracksQueriedColumns implements QueryAST {
Set<Dimension> denormTables = new HashSet<>();
if (cfacts != null) {
for (CandidateFact cfact : cfacts) {
- Set<Dimension> factDenormTables = deNormCtx.rewriteDenormctx(cfact, dimsToQuery, cfacts.size() > 1);
+ Set<Dimension> factDenormTables = deNormCtx.rewriteDenormctx(this, cfact, dimsToQuery, cfacts.size() > 1);
denormTables.addAll(factDenormTables);
if (cfacts.size() > 1) {
factDimMap.get(cfact).addAll(factDenormTables);
}
}
} else {
- denormTables.addAll(deNormCtx.rewriteDenormctx(null, dimsToQuery, false));
+ denormTables.addAll(deNormCtx.rewriteDenormctx(this, null, dimsToQuery, false));
}
dimsToQuery.putAll(pickCandidateDimsToQuery(denormTables));
log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery);
http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
index 40ed387..cb26878 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
@@ -87,23 +87,19 @@ public class DenormalizationResolver implements ContextRewriter {
public static class DenormalizationContext {
// map of column name to all references
+ @Getter
private Map<String, Set<ReferencedQueriedColumn>> referencedCols = new HashMap<>();
// candidate table name to all the references columns it needs
+ @Getter
private Map<String, Set<ReferencedQueriedColumn>> tableToRefCols = new HashMap<>();
- private CubeQueryContext cubeql;
-
// set of all picked references once all candidate tables are picked
private Set<PickedReference> pickedRefs = new HashSet<>();
// index on column name for picked references with map from column name to
// pickedrefs
private Map<String, Set<PickedReference>> pickedReferences = new HashMap<>();
- DenormalizationContext(CubeQueryContext cubeql) {
- this.cubeql = cubeql;
- }
-
void addReferencedCol(String col, ReferencedQueriedColumn refer) {
Set<ReferencedQueriedColumn> refCols = referencedCols.get(col);
if (refCols == null) {
@@ -116,7 +112,7 @@ public class DenormalizationResolver implements ContextRewriter {
// When candidate table does not have the field, this method checks
// if the field can be reached through reference,
// if yes adds the ref usage and returns to true, if not returns false.
- boolean addRefUsage(CandidateTable table, String col, String srcTbl) throws LensException {
+ boolean addRefUsage(CubeQueryContext cubeql, CandidateTable table, String col, String srcTbl) throws LensException {
// available as referenced col
if (referencedCols.containsKey(col)) {
for (ReferencedQueriedColumn refer : referencedCols.get(col)) {
@@ -144,10 +140,6 @@ public class DenormalizationResolver implements ContextRewriter {
return false;
}
- Map<String, Set<ReferencedQueriedColumn>> getReferencedCols() {
- return referencedCols;
- }
-
private void addPickedReference(String col, PickedReference refer) {
Set<PickedReference> refCols = pickedReferences.get(col);
if (refCols == null) {
@@ -169,23 +161,24 @@ public class DenormalizationResolver implements ContextRewriter {
return null;
}
- public Set<Dimension> rewriteDenormctx(CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery,
- boolean replaceFact) throws LensException {
+ public Set<Dimension> rewriteDenormctx(CubeQueryContext cubeql, CandidateFact cfact, Map<Dimension,
+ CandidateDim> dimsToQuery, boolean replaceFact) throws LensException {
Set<Dimension> refTbls = new HashSet<>();
+ log.info("Doing denorm changes for fact :{}", cfact);
if (!tableToRefCols.isEmpty()) {
// pick referenced columns for fact
if (cfact != null) {
- pickColumnsForTable(cfact.getName());
+ pickColumnsForTable(cubeql, cfact.getName());
}
// pick referenced columns for dimensions
if (dimsToQuery != null && !dimsToQuery.isEmpty()) {
for (CandidateDim cdim : dimsToQuery.values()) {
- pickColumnsForTable(cdim.getName());
+ pickColumnsForTable(cubeql, cdim.getName());
}
}
// Replace picked reference in all the base trees
- replaceReferencedColumns(cfact, replaceFact);
+ replaceReferencedColumns(cubeql, cfact, replaceFact);
// Add the picked references to dimsToQuery
for (PickedReference picked : pickedRefs) {
@@ -195,9 +188,45 @@ public class DenormalizationResolver implements ContextRewriter {
}
}
}
+ pickedReferences.clear();
+ pickedRefs.clear();
return refTbls;
}
+ public boolean hasReferences() {
+ return !tableToRefCols.isEmpty();
+ }
+ public Set<Dimension> rewriteDenormctxInExpression(CubeQueryContext cubeql, CandidateFact cfact, Map<Dimension,
+ CandidateDim> dimsToQuery, ASTNode exprAST) throws LensException {
+ Set<Dimension> refTbls = new HashSet<>();
+
+ log.info("Doing denorm changes for expressions in fact :{}", cfact);
+ if (!tableToRefCols.isEmpty()) {
+ // pick referenced columns for fact
+ if (cfact != null) {
+ pickColumnsForTable(cubeql, cfact.getName());
+ }
+ // pick referenced columns for dimensions
+ if (dimsToQuery != null && !dimsToQuery.isEmpty()) {
+ for (CandidateDim cdim : dimsToQuery.values()) {
+ pickColumnsForTable(cubeql, cdim.getName());
+ }
+ }
+ // Replace picked reference in expression ast
+ resolveClause(exprAST);
+
+ // Add the picked references to dimsToQuery
+ for (PickedReference picked : pickedRefs) {
+ if (isPickedFor(picked, cfact, dimsToQuery)) {
+ refTbls.add((Dimension) cubeql.getCubeTableForAlias(picked.getChainRef().getChainName()));
+ cubeql.addColumnsQueried(picked.getChainRef().getChainName(), picked.getChainRef().getRefColumn());
+ }
+ }
+ }
+ pickedReferences.clear();
+ pickedRefs.clear();
+ return refTbls;
+ }
// checks if the reference if picked for facts and dimsToQuery passed
private boolean isPickedFor(PickedReference picked, CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery) {
if (cfact != null && picked.pickedFor.equalsIgnoreCase(cfact.getName())) {
@@ -213,7 +242,7 @@ public class DenormalizationResolver implements ContextRewriter {
return false;
}
- private void pickColumnsForTable(String tbl) throws LensException {
+ private void pickColumnsForTable(CubeQueryContext cubeql, String tbl) throws LensException {
if (tableToRefCols.containsKey(tbl)) {
for (ReferencedQueriedColumn refered : tableToRefCols.get(tbl)) {
Iterator<ChainRefCol> iter = refered.chainRefCols.iterator();
@@ -237,27 +266,57 @@ public class DenormalizationResolver implements ContextRewriter {
}
}
- private void replaceReferencedColumns(CandidateFact cfact, boolean replaceFact) throws LensException {
+ public void pruneReferences(CubeQueryContext cubeql) {
+ for (Set<ReferencedQueriedColumn> referencedQueriedColumns : referencedCols.values()) {
+ for(Iterator<ReferencedQueriedColumn> iterator = referencedQueriedColumns.iterator(); iterator.hasNext();) {
+ ReferencedQueriedColumn rqc = iterator.next();
+ for (Iterator<ChainRefCol> iter = rqc.chainRefCols.iterator(); iter.hasNext();) {
+ // remove unreachable references
+ ChainRefCol reference = iter.next();
+ if (cubeql.getAutoJoinCtx() == null || !cubeql.getAutoJoinCtx().isReachableDim(
+ (Dimension) cubeql.getCubeTableForAlias(reference.getChainName()), reference.getChainName())) {
+ log.info("{} is not reachable", reference.getChainName());
+ iter.remove();
+ }
+ }
+ if (rqc.chainRefCols.isEmpty()) {
+ log.info("The referenced column: {} is not reachable", rqc.col.getName());
+ iterator.remove();
+ continue;
+ }
+ // do column life validation
+ for (TimeRange range : cubeql.getTimeRanges()) {
+ if (!rqc.col.isColumnAvailableInTimeRange(range)) {
+ log.info("The referenced column: {} is not in the range queried", rqc.col.getName());
+ iterator.remove();
+ break;
+ }
+ }
+ }
+ }
+ }
+ private void replaceReferencedColumns(CubeQueryContext cubeql, CandidateFact cfact, boolean replaceFact)
+ throws LensException {
QueryAST ast = cubeql;
boolean factRefExists = cfact != null && tableToRefCols.get(cfact.getName()) != null && !tableToRefCols.get(cfact
.getName()).isEmpty();
if (replaceFact && factRefExists) {
ast = cfact;
}
- resolveClause(cubeql, ast.getSelectAST());
+ resolveClause(ast.getSelectAST());
if (factRefExists) {
for (ASTNode storageWhereClauseAST : cfact.getStorgeWhereClauseMap().values()) {
- resolveClause(cubeql, storageWhereClauseAST);
+ resolveClause(storageWhereClauseAST);
}
} else {
- resolveClause(cubeql, ast.getWhereAST());
+ resolveClause(ast.getWhereAST());
}
- resolveClause(cubeql, ast.getGroupByAST());
- resolveClause(cubeql, ast.getHavingAST());
- resolveClause(cubeql, cubeql.getOrderByAST());
+ resolveClause(ast.getGroupByAST());
+ resolveClause(ast.getHavingAST());
+ resolveClause(cubeql.getOrderByAST());
}
- private void resolveClause(CubeQueryContext query, ASTNode node) throws LensException {
+ private void resolveClause(ASTNode node) throws LensException {
if (node == null) {
return;
}
@@ -288,9 +347,22 @@ public class DenormalizationResolver implements ContextRewriter {
// recurse down
for (int i = 0; i < node.getChildCount(); i++) {
ASTNode child = (ASTNode) node.getChild(i);
- resolveClause(query, child);
+ resolveClause(child);
+ }
+ }
+ }
+
+ public Set<String> getNonReachableReferenceFields(String table) {
+ Set<String> nonReachableFields = new HashSet<>();
+ if (tableToRefCols.containsKey(table)) {
+ for (ReferencedQueriedColumn refcol : tableToRefCols.get(table)) {
+ if (getReferencedCols().get(refcol.col.getName()).isEmpty()) {
+ log.info("For table:{}, the column {} is not available", table, refcol.col);
+ nonReachableFields.add(refcol.col.getName());
+ }
}
}
+ return nonReachableFields;
}
}
@@ -320,6 +392,14 @@ public class DenormalizationResolver implements ContextRewriter {
}
}
}
+ private static DenormalizationContext getOrCreateDeNormCtx(TrackDenormContext tdc) {
+ DenormalizationContext denormCtx = tdc.getDeNormCtx();
+ if (denormCtx == null) {
+ denormCtx = new DenormalizationContext();
+ tdc.setDeNormCtx(denormCtx);
+ }
+ return denormCtx;
+ }
/**
* Find all de-normalized columns, if these columns are not directly available in candidate tables, query will be
* replaced with the corresponding table reference
@@ -329,33 +409,29 @@ public class DenormalizationResolver implements ContextRewriter {
DenormalizationContext denormCtx = cubeql.getDeNormCtx();
if (denormCtx == null) {
// Adds all the reference dimensions as eligible for denorm fields
- denormCtx = new DenormalizationContext(cubeql);
- cubeql.setDeNormCtx(denormCtx);
// add ref columns in cube
- addRefColsQueried(cubeql, cubeql, denormCtx);
+ addRefColsQueried(cubeql, cubeql, getOrCreateDeNormCtx(cubeql));
// add ref columns from expressions
for (Set<ExpressionContext> ecSet : cubeql.getExprCtx().getAllExprsQueried().values()) {
for (ExpressionContext ec : ecSet) {
for (ExprSpecContext esc : ec.getAllExprs()) {
- addRefColsQueried(cubeql, esc, denormCtx);
+ addRefColsQueried(cubeql, esc, getOrCreateDeNormCtx(esc));
}
}
}
} else if (!denormCtx.tableToRefCols.isEmpty()) {
+ denormCtx.pruneReferences(cubeql);
// In the second iteration of denorm resolver
// candidate tables which require denorm fields and the refernces are no
// more valid will be pruned
if (cubeql.getCube() != null && !cubeql.getCandidateFacts().isEmpty()) {
for (Iterator<CandidateFact> i = cubeql.getCandidateFacts().iterator(); i.hasNext();) {
CandidateFact cfact = i.next();
- if (denormCtx.tableToRefCols.containsKey(cfact.getName())) {
- for (ReferencedQueriedColumn refcol : denormCtx.tableToRefCols.get(cfact.getName())) {
- if (denormCtx.getReferencedCols().get(refcol.col.getName()).isEmpty()) {
- log.info("Not considering fact table:{} as column {} is not available", cfact, refcol.col);
- cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(refcol.col.getName()));
- i.remove();
- }
- }
+ Set<String> nonReachableFields = denormCtx.getNonReachableReferenceFields(cfact.getName());
+ if (!nonReachableFields.isEmpty()) {
+ log.info("Not considering fact table:{} as columns {} are not available", cfact, nonReachableFields);
+ cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(nonReachableFields));
+ i.remove();
}
}
if (cubeql.getCandidateFacts().size() == 0) {
@@ -368,15 +444,12 @@ public class DenormalizationResolver implements ContextRewriter {
for (Dimension dim : cubeql.getDimensions()) {
for (Iterator<CandidateDim> i = cubeql.getCandidateDimTables().get(dim).iterator(); i.hasNext();) {
CandidateDim cdim = i.next();
- if (denormCtx.tableToRefCols.containsKey(cdim.getName())) {
- for (ReferencedQueriedColumn refcol : denormCtx.tableToRefCols.get(cdim.getName())) {
- if (denormCtx.getReferencedCols().get(refcol.col.getName()).isEmpty()) {
- log.info("Not considering dim table:{} as column {} is not available", cdim, refcol.col);
- cubeql.addDimPruningMsgs(dim, cdim.dimtable,
- CandidateTablePruneCause.columnNotFound(refcol.col.getName()));
- i.remove();
- }
- }
+ Set<String> nonReachableFields = denormCtx.getNonReachableReferenceFields(cdim.getName());
+ if (!nonReachableFields.isEmpty()) {
+ log.info("Not considering dim table:{} as column {} is not available", cdim, nonReachableFields);
+ cubeql.addDimPruningMsgs(dim, cdim.dimtable,
+ CandidateTablePruneCause.columnNotFound(nonReachableFields));
+ i.remove();
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 60dacdb..0ea0b1c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -154,7 +154,7 @@ class ExpressionResolver implements ContextRewriter {
void addEvaluable(CubeQueryContext cubeql, CandidateTable cTable, ExprSpecContext esc) throws LensException {
Set<ExprSpecContext> evalSet = evaluableExpressions.get(cTable);
if (evalSet == null) {
- evalSet = new LinkedHashSet<ExprSpecContext>();
+ evalSet = new LinkedHashSet<>();
evaluableExpressions.put(cTable, evalSet);
}
// add optional dimensions involved in expressions
@@ -196,13 +196,16 @@ class ExpressionResolver implements ContextRewriter {
}
}
- static class ExprSpecContext extends TracksQueriedColumns {
+ static class ExprSpecContext extends TracksQueriedColumns implements TrackDenormContext {
private Set<ExprSpec> exprSpecs = new LinkedHashSet<>();
@Getter
@Setter
private ASTNode finalAST;
@Getter
private Set<Dimension> exprDims = new HashSet<>();
+ @Getter
+ @Setter
+ private DenormalizationResolver.DenormalizationContext deNormCtx;
ExprSpecContext(ExprSpec exprSpec, CubeQueryContext cubeql) throws LensException {
// replaces table names in expression with aliases in the query
@@ -221,6 +224,7 @@ class ExpressionResolver implements ContextRewriter {
finalAST = AliasReplacer.replaceAliases(finalAST, 0, cubeql.getColToTableAlias());
}
+
void resolveColumns(CubeQueryContext cubeql) throws LensException {
// finds all columns and table aliases in the expression
ColumnResolver.getColsForTree(cubeql, finalAST, this, false);
@@ -267,13 +271,31 @@ class ExpressionResolver implements ContextRewriter {
public String toString() {
return HQLParser.getString(finalAST);
}
+
}
- @AllArgsConstructor
+ @RequiredArgsConstructor
@ToString
private static class PickedExpression {
- private String srcAlias;
- private ExprSpecContext pickedCtx;
+ private final String srcAlias;
+ private final ExprSpecContext pickedCtx;
+ private transient ASTNode reWrittenAST = null;
+
+ /*
+ Initialized rewrittenAST as copy of final AST if boolean is passed. Copy would be required if finalAST gets
+ modified because of denormalization context.
+ Otherwise, it is final AST reference, without any copy.
+ */
+ void initRewrittenAST(boolean copyFinal) {
+ if (copyFinal) {
+ reWrittenAST = MetastoreUtil.copyAST(pickedCtx.getFinalAST());
+ } else {
+ reWrittenAST = pickedCtx.getFinalAST();
+ }
+ }
+ ASTNode getRewrittenAST() {
+ return reWrittenAST;
+ }
}
static class ExpressionResolverContext {
@@ -357,7 +379,7 @@ class ExpressionResolver implements ContextRewriter {
boolean isEvaluable = true;
for (String col : columns) {
if (!cTable.getColumns().contains(col.toLowerCase())) {
- if (!cubeql.getDeNormCtx().addRefUsage(cTable, col, cTable.getBaseTable().getName())) {
+ if (!esc.getDeNormCtx().addRefUsage(cubeql, cTable, col, cTable.getBaseTable().getName())) {
// check if it is available as reference, if not expression is not evaluable
log.debug("{} = {} is not evaluable in {}", expr, esc, cTable);
isEvaluable = false;
@@ -379,49 +401,10 @@ class ExpressionResolver implements ContextRewriter {
return ec.isEvaluable(cTable);
}
- /**
- *
- * @param exprs
- * @return
- */
- public boolean allNotEvaluable(Set<String> exprs, CandidateTable cTable) {
- for (String expr : exprs) {
- if (isEvaluable(expr, cTable)) {
- return false;
- }
- }
- return true;
- }
-
- public Collection<String> coveringExpressions(Set<String> exprs, CandidateTable cTable) {
- Set<String> coveringSet = new HashSet<String>();
- for (String expr : exprs) {
- if (isEvaluable(expr, cTable)) {
- coveringSet.add(expr);
- }
- }
- return coveringSet;
- }
-
- /**
- * Returns true if all passed expressions are evaluable
- *
- * @param cTable
- * @param exprs
- * @return
- */
- public boolean allEvaluable(CandidateTable cTable, Set<String> exprs) {
- for (String expr : exprs) {
- if (!isEvaluable(expr, cTable)) {
- return false;
- }
- }
- return true;
- }
-
- public Set<Dimension> rewriteExprCtx(CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery,
- QueryAST queryAST) throws LensException {
+ public Set<Dimension> rewriteExprCtx(CubeQueryContext cubeql, CandidateFact cfact, Map<Dimension,
+ CandidateDim> dimsToQuery, QueryAST queryAST) throws LensException {
Set<Dimension> exprDims = new HashSet<Dimension>();
+ log.info("Picking expressions for fact {} ", cfact);
if (!allExprsQueried.isEmpty()) {
// pick expressions for fact
if (cfact != null) {
@@ -433,16 +416,21 @@ class ExpressionResolver implements ContextRewriter {
pickExpressionsForTable(cdim);
}
}
- // Replace picked expressions in all the base trees
- replacePickedExpressions(cfact, queryAST);
- log.debug("Picked expressions: {}", pickedExpressions);
+ log.info("Picked expressions: {}", pickedExpressions);
for (Set<PickedExpression> peSet : pickedExpressions.values()) {
for (PickedExpression pe : peSet) {
exprDims.addAll(pe.pickedCtx.exprDims);
+ pe.initRewrittenAST(pe.pickedCtx.deNormCtx.hasReferences());
+ exprDims.addAll(pe.pickedCtx.deNormCtx.rewriteDenormctxInExpression(cubeql, cfact, dimsToQuery,
+ pe.getRewrittenAST()));
}
}
+ // Replace picked expressions in all the base trees
+ replacePickedExpressions(cfact, queryAST);
}
+
pickedExpressions.clear();
+
return exprDims;
}
@@ -488,7 +476,7 @@ class ExpressionResolver implements ContextRewriter {
if (pickedExpressions.containsKey(column)) {
PickedExpression expr = getPickedExpression(column, tabident.getText().toLowerCase());
if (expr != null) {
- node.setChild(i, replaceAlias(expr.pickedCtx.finalAST, cubeql));
+ node.setChild(i, replaceAlias(expr.getRewrittenAST(), cubeql));
}
}
}
@@ -552,6 +540,21 @@ class ExpressionResolver implements ContextRewriter {
if (removed) {
continue;
}
+ // Remove expressions for which denormalized columns are no more reachable
+ esc.getDeNormCtx().pruneReferences(cubeql);
+ for (String table : esc.getDeNormCtx().getTableToRefCols().keySet()) {
+ Set<String> nonReachableFields = esc.getDeNormCtx().getNonReachableReferenceFields(table);
+ if (!nonReachableFields.isEmpty()) {
+ log.info("Removing expression {} as columns {} are not available", esc, nonReachableFields);
+ iterator.remove();
+ removedEsc.add(esc);
+ removed = true;
+ break;
+ }
+ }
+ if (removed) {
+ continue;
+ }
//remove expressions which are not valid in the timerange queried
// If an expression is defined as
// ex = a + b // from t1 to t2;
http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java
index 11eb8f7..34a562d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueriedPhraseContext.java
@@ -116,7 +116,7 @@ class QueriedPhraseContext extends TracksQueriedColumns implements TrackQueriedC
for (String col : queriedDimAttrs) {
if (!cfact.getColumns().contains(col.toLowerCase())) {
// check if it available as reference
- if (!cubeQl.getDeNormCtx().addRefUsage(cfact, col, cubeQl.getCube().getName())) {
+ if (!cubeQl.getDeNormCtx().addRefUsage(cubeQl, cfact, col, cubeQl.getCube().getName())) {
log.info("column {} is not available in fact table:{} ", col, cfact);
return false;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java
index 89b50f5..f18ae36 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/TimeRangeChecker.java
@@ -160,24 +160,6 @@ public class TimeRangeChecker implements ContextRewriter {
}
}
- // Look at referenced columns through denormalization resolver
- // and do column life validation
- Map<String, Set<DenormalizationResolver.ReferencedQueriedColumn>> refCols =
- cubeql.getDeNormCtx().getReferencedCols();
- for (String col : refCols.keySet()) {
- Iterator<DenormalizationResolver.ReferencedQueriedColumn> refColIter = refCols.get(col).iterator();
- while (refColIter.hasNext()) {
- DenormalizationResolver.ReferencedQueriedColumn refCol = refColIter.next();
- for (TimeRange range : cubeql.getTimeRanges()) {
- if (!refCol.col.isColumnAvailableInTimeRange(range)) {
- log.debug("The refernced column: {} is not in the range queried", refCol.col.getName());
- refColIter.remove();
- break;
- }
- }
- }
- }
-
// Remove join paths that have columns with invalid life span
AutoJoinContext joinContext = cubeql.getAutoJoinCtx();
if (joinContext == null) {
http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackDenormContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackDenormContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackDenormContext.java
new file mode 100644
index 0000000..5592f70
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/TrackDenormContext.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.parse;
+
+interface TrackDenormContext {
+
+ /**
+ * Get denormalization context
+ *
+ * @return DenormalizationContext
+ */
+ DenormalizationResolver.DenormalizationContext getDeNormCtx();
+
+ /**
+ * Set denormalization context
+ *
+ * @param deNormCtx DenormalizationContext
+ */
+ void setDeNormCtx(DenormalizationResolver.DenormalizationContext deNormCtx);
+
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 41ea83d..9b29083 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -578,6 +578,8 @@ public class CubeTestSetup {
"dim3 refer", "dim3chain", "id", null, null, 0.0));
cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("cityname", "string", "city name"),
"city name", "cubecity", "name", null, null, 0.0));
+ cubeDimensions.add(new ReferencedDimAttribute(new FieldSchema("statename_cube", "string", "state name"),
+ "state name", "cubestate", "name", null, null, 0.0));
List<ChainRefCol> references = new ArrayList<>();
references.add(new ChainRefCol("timedatechain1", "full_date"));
references.add(new ChainRefCol("timehourchain1", "full_hour"));
@@ -592,6 +594,8 @@ public class CubeTestSetup {
"City1", null, null, null));
cubeDimensions.add(new BaseDimAttribute(new FieldSchema("cityid2", "int", "id to city"),
"City2", null, null, null));
+ cubeDimensions.add(new BaseDimAttribute(new FieldSchema("concatedcitystate", "string", "citystate"),
+ "CityState", null, null, null));
Map<String, JoinChain> joinChains = new HashMap<>();
addCubeChains(joinChains, TEST_CUBE_NAME);
@@ -653,7 +657,11 @@ public class CubeTestSetup {
exprs.add(new ExprColumn(new FieldSchema("newexpr", "string", "expression which non existing colun"),
"new measure expr", "myfun(newmeasure)"));
exprs.add(new ExprColumn(new FieldSchema("cityAndState", "String", "city and state together"), "City and State",
- "concat(cubecity.name, \":\", cubestate.name)"));
+ new ExprSpec("concat(cityname, \":\", statename_cube)", null, null),
+ new ExprSpec("substr(concatedcitystate, 10)", null, null)));
+ exprs.add(new ExprColumn(new FieldSchema("cityAndStateNew", "String", "city and state together"), "City and State",
+ new ExprSpec("concat(cityname, \":\", statename_cube)", null, TWO_MONTHS_BACK),
+ new ExprSpec("substr(concatedcitystate, 10)", null, null)));
exprs.add(new ExprColumn(new FieldSchema("cityStateName", "String", "city state"), "City State",
"concat('CityState:', cubecity.statename)"));
exprs.add(new ExprColumn(new FieldSchema("isIndia", "String", "is indian city/state"), "Is Indian City/state",
@@ -1957,6 +1965,7 @@ public class CubeTestSetup {
factColumns.add(new FieldSchema("countryid", "int", "country id"));
factColumns.add(new FieldSchema("dim1", "string", "dim1"));
factColumns.add(new FieldSchema("dim2", "int", "dim2"));
+ factColumns.add(new FieldSchema("concatedCityState", "string", "citystate"));
Map<String, Set<UpdatePeriod>> storageAggregatePeriods = new HashMap<String, Set<UpdatePeriod>>();
Set<UpdatePeriod> updates = new HashSet<UpdatePeriod>();
http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java
index f2bb485..5d4e87f 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java
@@ -215,6 +215,21 @@ public class TestExpressionResolver extends TestQueryRewrite {
+ " group by concat(cubecity.name, \":\", cubestate.name)", null, getWhereForHourly2days("C1_testfact2_raw"));
TestCubeRewriter.compareQueries(hqlQuery, expected);
}
+
+ @Test
+ public void testExpressionToExcludeJoin() throws Exception {
+ // expression which results in join
+ String hqlQuery =
+ rewrite("select cityAndStateNew, avgmsr from testCube" + " where " + TWO_DAYS_RANGE + " and substrexpr != 'XYZ'",
+ conf);
+
+ String expected =
+ getExpectedQuery(cubeName, "select substr(testcube.concatedcitystate, 10)"
+ + " avg(testcube.msr1 + testcube.msr2) FROM ", null, null, " and substr(testcube.dim1, 3) != 'XYZ'"
+ + " group by substr(testcube.concatedcitystate, 10)", null, getWhereForHourly2days("C1_testfact2_raw"));
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
+ }
+
@Test
public void testExpressionInWhereWithJoinClausePassed() throws Exception {
assertLensExceptionInRewrite("select cityAndState, avgmsr from testCube tc join citydim cd join statedim sd where "
http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java
index 6430ed1..677d641 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java
@@ -545,7 +545,7 @@ public class TestJoinResolver extends TestQueryRewrite {
Configuration conf = new Configuration(hconf);
conf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C3, C4");
String failingQuery = "select testDim2.cityname, testDim2.cityStateCapital FROM testDim2 where " + TWO_DAYS_RANGE;
- assertLensExceptionInRewrite(failingQuery, conf, LensCubeErrorCode.NO_REF_COL_AVAILABLE);
+ assertLensExceptionInRewrite(failingQuery, conf, LensCubeErrorCode.NO_DIM_HAS_COLUMN);
}
@Test
http://git-wip-us.apache.org/repos/asf/lens/blob/4fb25061/tools/conf/server/logback.xml
----------------------------------------------------------------------
diff --git a/tools/conf/server/logback.xml b/tools/conf/server/logback.xml
index 63ab23c..5173321 100644
--- a/tools/conf/server/logback.xml
+++ b/tools/conf/server/logback.xml
@@ -41,7 +41,7 @@
<maxHistory>30</maxHistory>
</rollingPolicy>
<encoder>
- <pattern>%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c - %m%n</pattern>
+ <pattern>%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %L - %m%n</pattern>
</encoder>
</appender>
@@ -107,7 +107,7 @@
<file>${lens.log.dir}/${queryLogId}.log</file>
<append>true</append>
<layout class="ch.qos.logback.classic.PatternLayout">
- <pattern>%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c - %m%n</pattern>
+ <pattern>%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %L - %m%n</pattern>
</layout>
</appender>
</sift>