You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2017/07/13 09:13:56 UTC
[03/12] lens git commit: LENS-1443: Fallback ranges not working for
virtual facts
LENS-1443: Fallback ranges not working for virtual facts
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c174583f
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c174583f
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c174583f
Branch: refs/heads/current-release-line
Commit: c174583ff7946d0347e0a0a87272f42c0023aecf
Parents: 9da5b40
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Wed Jun 21 13:20:09 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 13 14:42:50 2017 +0530
----------------------------------------------------------------------
.../apache/lens/cube/metadata/FactTable.java | 38 ++++++++++----------
.../apache/lens/cube/parse/JoinCandidate.java | 7 ++--
.../lens/cube/parse/LeastPartitionResolver.java | 21 +++++------
.../lens/cube/parse/StorageCandidate.java | 2 +-
.../lens/cube/parse/StorageTableResolver.java | 5 +--
5 files changed, 37 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
index f87cf44..a463c47 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
@@ -35,132 +35,132 @@ public interface FactTable extends Named {
*
* @return Map of storage to set of update periods
*/
- public Map<String, Set<UpdatePeriod>> getUpdatePeriods();
+ Map<String, Set<UpdatePeriod>> getUpdatePeriods();
/**
* Cube to which this fact belongs to
*
* @return the cube string
*/
- public String getCubeName();
+ String getCubeName();
/**
* The set of Storage names
*
* @return set of strings
*/
- public Set<String> getStorages();
+ Set<String> getStorages();
/**
*The type of the fact
*
* @return table type {@link CubeTableType}
*/
- public CubeTableType getTableType();
+ CubeTableType getTableType();
/**
* Config properties
*
* @return map of string, string
*/
- public Map<String, String> getProperties();
+ Map<String, String> getProperties();
/**
* Valid columns of the fact
*
* @return list of column names
*/
- public Set<String> getValidColumns();
+ Set<String> getValidColumns();
/**
* Weight of the fact
*
* @return weight of the fact in double
*/
- public double weight();
+ double weight();
/**
* Set of all the columns names of the fact
*
* @return set of column names
*/
- public Set<String> getAllFieldNames();
+ Set<String> getAllFieldNames();
/**
*tag for checking data completeness
*
* @return Tag String
*/
- public String getDataCompletenessTag();
+ String getDataCompletenessTag();
/**
* List of columns of the fact
*
* @return set of {@link FieldSchema}
*/
- public List<FieldSchema> getColumns();
+ List<FieldSchema> getColumns();
/**
* Is Aggregated Fact
*
* @return true if fact is Aggregated , false otherwise
*/
- public boolean isAggregated();
+ boolean isAggregated();
/**
* Absolute start time of the fact
*
* @return Absolute Start time of the fact {@link Date}
*/
- public Date getAbsoluteStartTime();
+ Date getAbsoluteStartTime();
/**
* Relative start time of the fact
*
* @return Relative Start time of the fact {@link Date}
*/
- public Date getRelativeStartTime();
+ Date getRelativeStartTime();
/**
* Start time of the fact
*
* @return Start time of the fact {@link Date}
*/
- public Date getStartTime();
+ Date getStartTime();
/**
* Absolute end time of the fact
*
* @return Absolute End time of the fact {@link Date}
*/
- public Date getAbsoluteEndTime();
+ Date getAbsoluteEndTime();
/**
* Relative End time of the Fact
*
* @return Relative end time of the fact {@link Date}
*/
- public Date getRelativeEndTime();
+ Date getRelativeEndTime();
/**
* End time of the fact
*
* @return End time of the fact {@link Date}
*/
- public Date getEndTime();
+ Date getEndTime();
/**
* Is Virtual Fact
*
* @return true if fact is a virtual fact, false otherwise
*/
- public boolean isVirtualFact();
+ boolean isVirtualFact();
/**
* Storage name of the fact
*
* @return Storage name of the fact
*/
- public String getSourceFactName();
+ String getSourceFactName();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java
index d9915f4..1c2b9f8 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java
@@ -91,12 +91,11 @@ public class JoinCandidate implements Candidate {
@Override
public boolean evaluateCompleteness(TimeRange timeRange, TimeRange parentTimeRange, boolean failOnPartialData)
throws LensException {
+ boolean complete = true;
for (Candidate child : children) {
- if (!child.evaluateCompleteness(timeRange, parentTimeRange, failOnPartialData)) {
- return false;
- }
+ complete &= child.evaluateCompleteness(timeRange, parentTimeRange, failOnPartialData);
}
- return true;
+ return complete;
}
/**
http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
index 22b1d03..a76e9b1 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
@@ -48,17 +48,18 @@ class LeastPartitionResolver implements ContextRewriter {
factPartCount.put(candidate, parts);
}
}
+ if (!factPartCount.isEmpty()) {
+ double minPartitions = Collections.min(factPartCount.values());
- double minPartitions = Collections.min(factPartCount.values());
-
- for (Iterator<Candidate> i = cubeql.getCandidates().iterator(); i.hasNext();) {
- Candidate candidate = i.next();
- if (factPartCount.containsKey(candidate) && factPartCount.get(candidate) > minPartitions) {
- log.info("Not considering Candidate:{} as it requires more partitions to be" + " queried:{} minimum:{}",
- candidate, factPartCount.get(candidate), minPartitions);
- i.remove();
- cubeql.addCandidatePruningMsg(candidate,
- new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.MORE_PARTITIONS));
+ for (Iterator<Candidate> i = cubeql.getCandidates().iterator(); i.hasNext();) {
+ Candidate candidate = i.next();
+ if (factPartCount.containsKey(candidate) && factPartCount.get(candidate) > minPartitions) {
+ log.info("Not considering Candidate:{} as it requires more partitions to be" + " queried:{} minimum:{}",
+ candidate, factPartCount.get(candidate), minPartitions);
+ i.remove();
+ cubeql.addCandidatePruningMsg(candidate,
+ new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.MORE_PARTITIONS));
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index 6e5aa4c..c6ef6d2 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -604,7 +604,7 @@ public class StorageCandidate implements Candidate, CandidateTable {
break;
}
}
- TimeRange fallBackRange = getFallbackRange(prevRange, this.getFact().getName(), cubeQueryContext);
+ TimeRange fallBackRange = getFallbackRange(prevRange, this.getFact().getSourceFactName(), cubeQueryContext);
log.info("No partitions for range:{}. fallback range: {}", timeRange, fallBackRange);
if (fallBackRange == null) {
break;
http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index d7da8cb..7f0f2d4 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -20,6 +20,7 @@ package org.apache.lens.cube.parse;
import static org.apache.lens.cube.parse.CandidateTablePruneCause.incompletePartitions;
import static org.apache.lens.cube.parse.CandidateTablePruneCause.partitionColumnsMissing;
+import static org.apache.lens.cube.parse.StorageUtil.getFallbackRange;
import java.util.*;
@@ -324,13 +325,13 @@ class StorageTableResolver implements ContextRewriter {
} else if (!sc.getValidUpdatePeriods().contains(UpdatePeriod.CONTINUOUS)) {
if (!client.partColExists(sc.getFact(), sc.getStorageName(), range.getPartitionColumn())) {
pruningCauseForThisTimeRange = partitionColumnsMissing(range.getPartitionColumn());
- TimeRange fallBackRange = StorageUtil.getFallbackRange(range, sc.getFact().getName(), cubeql);
+ TimeRange fallBackRange = getFallbackRange(range, sc.getFact().getSourceFactName(), cubeql);
while (fallBackRange != null) {
pruningCauseForThisTimeRange = null;
if (!client.partColExists(sc.getFact(), sc.getStorageName(),
fallBackRange.getPartitionColumn())) {
pruningCauseForThisTimeRange = partitionColumnsMissing(fallBackRange.getPartitionColumn());
- fallBackRange = StorageUtil.getFallbackRange(fallBackRange, sc.getFact().getName(), cubeql);
+ fallBackRange = getFallbackRange(fallBackRange, sc.getFact().getSourceFactName(), cubeql);
} else {
if (!sc.isPartiallyValidForTimeRange(fallBackRange)) {
pruningCauseForThisTimeRange =