You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2015/10/05 10:07:51 UTC
lens git commit: LENS-795: Look ahead is failing with partition not
found if any process time partitions are absent
Repository: lens
Updated Branches:
refs/heads/master ebb2c0366 -> aeaae6815
LENS-795: Look ahead is failing with partition not found if any process time partitions are absent
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/aeaae681
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/aeaae681
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/aeaae681
Branch: refs/heads/master
Commit: aeaae681523989c798c059bb4345aed106bd5fe1
Parents: ebb2c03
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Mon Oct 5 13:37:48 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Mon Oct 5 13:37:48 2015 +0530
----------------------------------------------------------------------
.../PartitionRangesForPartitionColumns.java | 6 ++++--
.../lens/cube/parse/StorageTableResolver.java | 19 +++++++++++++++----
2 files changed, 19 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/aeaae681/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java
index b389a8f..fa3f6ef 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java
@@ -41,10 +41,12 @@ public class PartitionRangesForPartitionColumns extends HashMap<String, RangesPa
.rangeUpto(partition.next().withUpdatePeriod(UpdatePeriod.values()[0])));
}
- public Set<String> toSet() {
+ public Set<String> toSet(Set<String> partColsQueried) {
Set<String> ret = Sets.newHashSet();
for (Map.Entry<String, RangesPartitionTimeline> entry : entrySet()) {
- ret.add(entry.getKey() + ":" + entry.getValue().getRanges());
+ if (partColsQueried.contains(entry.getKey())) {
+ ret.add(entry.getKey() + ":" + entry.getValue().getRanges());
+ }
}
return ret;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/aeaae681/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index 68ab5ab..f67fc26 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -363,7 +363,9 @@ class StorageTableResolver implements ContextRewriter {
PartitionRangesForPartitionColumns missingParts = new PartitionRangesForPartitionColumns();
boolean noPartsForRange = false;
Set<String> unsupportedTimeDims = Sets.newHashSet();
+ Set<String> partColsQueried = Sets.newHashSet();
for (TimeRange range : cubeql.getTimeRanges()) {
+ partColsQueried.add(range.getPartitionColumn());
StringBuilder extraWhereClause = new StringBuilder();
Set<FactPartition> rangeParts = getPartitions(cfact.fact, range, skipStorageCauses, missingParts);
// If no partitions were found, then we'll fallback.
@@ -389,6 +391,7 @@ class StorageTableResolver implements ContextRewriter {
if (fallBackRange == null) {
break;
}
+ partColsQueried.add(fallBackRange.getPartitionColumn());
rangeParts = getPartitions(cfact.fact, fallBackRange, skipStorageCauses, missingParts);
extraWhereClause.append(sep)
.append(prevRange.toTimeDimWhereClause(cubeql.getAliasForTableName(cubeql.getCube()), timeDim));
@@ -433,7 +436,7 @@ class StorageTableResolver implements ContextRewriter {
i.remove();
continue;
}
- Set<String> nonExistingParts = missingParts.toSet();
+ Set<String> nonExistingParts = missingParts.toSet(partColsQueried);
if (!nonExistingParts.isEmpty()) {
addNonExistingParts(cfact.fact.getName(), nonExistingParts);
}
@@ -637,8 +640,10 @@ class StorageTableResolver implements ContextRewriter {
for (Date date : timeRange.iterable(pPart.getPeriod(), 1)) {
FactPartition innerPart = new FactPartition(partCol, date, pPart.getPeriod(), pPart,
partWhereClauseFormat);
- updateFactPartitionStorageTablesFrom(fact, innerPart, pPart.getStorageTables());
- partitions.add(innerPart);
+ updateFactPartitionStorageTablesFrom(fact, innerPart, pPart);
+ if (innerPart.isFound()) {
+ partitions.add(innerPart);
+ }
}
log.debug("added all sub partitions blindly in pPart: {}", pPart);
}
@@ -678,7 +683,7 @@ class StorageTableResolver implements ContextRewriter {
updatePeriods, addNonExistingParts, failOnPartialData, skipStorageCauses, missingPartitions);
}
- void updateFactPartitionStorageTablesFrom(CubeFactTable fact,
+ private void updateFactPartitionStorageTablesFrom(CubeFactTable fact,
FactPartition part, Set<String> storageTableNames) throws LensException, HiveException, ParseException {
for (String storageTableName : storageTableNames) {
if (client.factPartitionExists(fact, part, storageTableName)) {
@@ -687,4 +692,10 @@ class StorageTableResolver implements ContextRewriter {
}
}
}
+
+ private void updateFactPartitionStorageTablesFrom(CubeFactTable fact,
+ FactPartition part, FactPartition pPart) throws LensException, HiveException, ParseException {
+ updateFactPartitionStorageTablesFrom(fact, part, pPart.getStorageTables());
+ part.setFound(part.isFound() && pPart.isFound());
+ }
}