You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2017/07/20 10:43:19 UTC
[1/3] lens git commit: LENS-1455: Getting expression ast to generate
default expression should consider all expression contexts
Repository: lens
Updated Branches:
refs/heads/current-release-line 6dca44661 -> cdd7b0999
LENS-1455: Getting expression ast to generate default expression should consider all expression contexts
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/cdd7b099
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/cdd7b099
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/cdd7b099
Branch: refs/heads/current-release-line
Commit: cdd7b09995b765783867812e5b981c27f4938371
Parents: 8b41202
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Thu Jul 20 16:11:15 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 20 16:13:07 2017 +0530
----------------------------------------------------------------------
.../cube/parse/CandidateCoveringSetsResolver.java | 3 +++
.../apache/lens/cube/parse/ExpressionResolver.java | 17 ++++++++++++-----
.../lens/cube/parse/SegmentationCandidate.java | 5 +++++
3 files changed, 20 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/cdd7b099/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 4066cf7..de30703 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -255,6 +255,9 @@ public class CandidateCoveringSetsResolver implements ContextRewriter {
i.remove();
}
}
+ // sorting will make sure storage candidates come before complex candidates.
+ // ensuring maximum columns get selected from simpler candidates.
+ ucSet.sort(Comparator.comparing(Candidate::getChildrenCount));
// Sets that contain all measures or no measures are removed from iteration.
// find other facts
for (Iterator<Candidate> i = ucSet.iterator(); i.hasNext();) {
http://git-wip-us.apache.org/repos/asf/lens/blob/cdd7b099/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 66b043e..4680766 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.antlr.runtime.CommonToken;
+import com.google.common.collect.Sets;
import lombok.*;
import lombok.extern.slf4j.Slf4j;
@@ -501,15 +502,21 @@ class ExpressionResolver implements ContextRewriter {
}
}
- private ASTNode getExprAst(ExpressionContext ec) {
+ private ASTNode getExprAst(ExpressionContext expressionContext) {
Set<StorageCandidate> scSet = CandidateUtil.getStorageCandidates(cubeql.getCandidates());
- Set<String> storageTableNames = new HashSet<String>();
+ Set<String> storageTableNames = new HashSet<>();
+ Set<ExpressionContext> expressionContexts = Sets.newLinkedHashSet();
+ expressionContexts.add(expressionContext);
for (StorageCandidate sc : scSet) {
storageTableNames.add(sc.getStorageTable());
+ expressionContexts.add(sc.getCubeQueryContext().getExprCtx()
+ .getExpressionContext(expressionContext.getExprCol().getName(), expressionContext.getSrcAlias()));
}
- for (CandidateTable table : ec.evaluableExpressions.keySet()) {
- if (storageTableNames.contains(table.getStorageTable())) {
- return MetastoreUtil.copyAST(ec.evaluableExpressions.get(table).iterator().next().finalAST);
+ for (ExpressionContext ec : expressionContexts) {
+ for (CandidateTable table : ec.evaluableExpressions.keySet()) {
+ if (storageTableNames.contains(table.getStorageTable())) {
+ return MetastoreUtil.copyAST(ec.evaluableExpressions.get(table).iterator().next().finalAST);
+ }
}
}
return null;
http://git-wip-us.apache.org/repos/asf/lens/blob/cdd7b099/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
index 35638f3..055d6ef 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
@@ -273,6 +273,11 @@ public class SegmentationCandidate implements Candidate {
}
@Override
+ public int getChildrenCount() {
+ return segmentation.getSegments().size();
+ }
+
+ @Override
public boolean isTimeRangeCoverable(TimeRange timeRange) throws LensException {
return true;
}
[3/3] lens git commit: LENS-1453: Method metrics names duplicate for
segmentation inner rewrite
Posted by pr...@apache.org.
LENS-1453: Method metrics names duplicate for segmentation inner rewrite
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/65ca04b5
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/65ca04b5
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/65ca04b5
Branch: refs/heads/current-release-line
Commit: 65ca04b598bd29201a6670b7a51679f9ccb9ccc6
Parents: 6dca446
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Thu Jul 20 16:09:58 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 20 16:13:07 2017 +0530
----------------------------------------------------------------------
.../org/apache/lens/cube/parse/SegmentationCandidate.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/65ca04b5/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
index a2bd485..35638f3 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SegmentationCandidate.java
@@ -52,6 +52,7 @@ import org.apache.lens.cube.metadata.MetastoreUtil;
import org.apache.lens.cube.metadata.Segment;
import org.apache.lens.cube.metadata.Segmentation;
import org.apache.lens.cube.metadata.TimeRange;
+import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang3.tuple.Pair;
@@ -141,7 +142,13 @@ public class SegmentationCandidate implements Candidate {
});
addCubeNameAndAlias(ast, innerCube);
trimHavingAndOrderby(ast, innerCube);
- CubeQueryRewriter rewriter = new CubeQueryRewriter(conf, hconf);
+ Configuration innerConf = conf;
+ if (conf.get(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY) != null) {
+ innerConf = new Configuration(conf);
+ innerConf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY,
+ conf.get(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY) + "-" + segment.getName());
+ }
+ CubeQueryRewriter rewriter = new CubeQueryRewriter(innerConf, hconf);
CubeQueryContext ctx = rewriter.rewrite(ast);
cubeQueryContextMap.put(segment.getName(), ctx);
if (!ctx.getCandidates().isEmpty()) {
[2/3] lens git commit: LENS-1454: Time Covering set algorithm is
skipping some combinations
Posted by pr...@apache.org.
LENS-1454: Time Covering set algorithm is skipping some combinations
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/8b412023
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/8b412023
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/8b412023
Branch: refs/heads/current-release-line
Commit: 8b4120237af9aaca3a6934650ccfeac24bc62c59
Parents: 65ca04b
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Thu Jul 20 16:10:52 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 20 16:13:07 2017 +0530
----------------------------------------------------------------------
.../lens/cube/parse/CandidateCoveringSetsResolver.java | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/8b412023/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
index 69d9562..4066cf7 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java
@@ -209,21 +209,23 @@ public class CandidateCoveringSetsResolver implements ContextRewriter {
private List<UnionCandidate> getCombinationTailIterative(List<Candidate> candidates, CubeQueryContext cubeql) {
LinkedList<Candidate> candidateLinkedList = Lists.newLinkedList(candidates);
List<List<Candidate>> incompleteCombinations = Lists.newArrayList();
+ incompleteCombinations.add(Lists.newArrayList());
List<UnionCandidate> unionCandidates = Lists.newArrayList();
while(!candidateLinkedList.isEmpty()) {
+ List<List<Candidate>> moreIncomplete = Lists.newArrayList();
Candidate candidate = candidateLinkedList.remove();
- incompleteCombinations.add(Lists.newArrayList());
- Iterator<List<Candidate>> iter = incompleteCombinations.iterator();
- while(iter.hasNext()) {
- List<Candidate> incompleteCombination = iter.next();
+ for (List<Candidate> combination : incompleteCombinations) {
+ List<Candidate> incompleteCombination = Lists.newArrayList(combination);
incompleteCombination.add(candidate);
UnionCandidate unionCandidate = new UnionCandidate(incompleteCombination, cubeql);
if (isCandidateCoveringTimeRanges(unionCandidate, cubeql.getTimeRanges())) {
unionCandidates.add(unionCandidate);
- iter.remove();
+ } else {
+ moreIncomplete.add(incompleteCombination);
}
}
+ incompleteCombinations.addAll(moreIncomplete);
}
return unionCandidates;
}