You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by ra...@apache.org on 2015/10/09 06:17:26 UTC
[21/50] [abbrv] lens git commit: LENS-392 : Look ahead timerange
should not look for all finer granularity partitions to be present
LENS-392 : Look ahead timerange should not look for all finer granularity partitions to be present
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/08135aa6
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/08135aa6
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/08135aa6
Branch: refs/heads/current-release-line
Commit: 08135aa693658aa454e55a2c1e81c9e0fa19444b
Parents: 4e81ef4
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Wed Sep 16 09:50:55 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Wed Sep 16 09:50:55 2015 +0530
----------------------------------------------------------------------
.../lens/cube/metadata/FactPartition.java | 14 ++-
.../lens/cube/metadata/TimePartition.java | 9 ++
.../lens/cube/metadata/TimePartitionRange.java | 5 +
.../timeline/RangesPartitionTimeline.java | 4 +-
.../cube/parse/AbridgedTimeRangeWriter.java | 5 +-
.../lens/cube/parse/StorageTableResolver.java | 43 ++++-----
.../apache/lens/cube/parse/CubeTestSetup.java | 52 ++++++-----
.../lens/cube/parse/TestCubeRewriter.java | 97 ++++++++++----------
8 files changed, 130 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/08135aa6/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
index fc2d85b..f934ad3 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
@@ -42,6 +42,7 @@ public class FactPartition implements Comparable<FactPartition> {
@Getter
@Setter
private FactPartition containingPart;
+ @Getter
private final DateFormat partFormat;
@Getter
@Setter
@@ -59,7 +60,18 @@ public class FactPartition implements Comparable<FactPartition> {
public FactPartition(String partCol, Date partSpec, UpdatePeriod period, FactPartition containingPart,
DateFormat partFormat, Set<String> storageTables) {
this(partCol, partSpec, period, containingPart, partFormat);
- this.storageTables.addAll(storageTables);
+ if (storageTables != null) {
+ this.storageTables.addAll(storageTables);
+ }
+ }
+
+ public FactPartition(String partCol, TimePartition timePartition) {
+ this(partCol, timePartition, null, null);
+ }
+
+ public FactPartition(String partCol, TimePartition timePartition, FactPartition containingPart, Set<String>
+ storageTables) {
+ this(partCol, timePartition.getDate(), timePartition.getUpdatePeriod(), containingPart, null, storageTables);
}
public boolean hasContainingPart() {
http://git-wip-us.apache.org/repos/asf/lens/blob/08135aa6/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
index d52f168..0026262 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
@@ -121,6 +121,7 @@ public class TimePartition implements Comparable<TimePartition>, Named {
return rangeUpto(next());
}
+
@Override
public String getName() {
return getDateString();
@@ -129,4 +130,12 @@ public class TimePartition implements Comparable<TimePartition>, Named {
public TimePartitionRange emptyRange() throws LensException {
return this.rangeUpto(this);
}
+
+ public static TimePartition max(TimePartition p1, TimePartition p2) {
+ return p1.compareTo(p2) >= 0 ? p1 : p2;
+ }
+
+ public static TimePartition min(TimePartition p1, TimePartition p2) {
+ return p1.compareTo(p2) < 0 ? p1 : p2;
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/08135aa6/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java
index f5f8d4c..01069a5 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.cube.metadata;
+import java.util.Date;
import java.util.Iterator;
import org.apache.lens.cube.parse.DateUtil;
@@ -33,6 +34,10 @@ public class TimePartitionRange implements Iterable<TimePartition>, Named {
private TimePartition begin;
private TimePartition end;
+ public static TimePartitionRange between(Date from, Date to, UpdatePeriod period) throws LensException {
+ return TimePartition.of(period, from).rangeUpto(TimePartition.of(period, to));
+ }
+
public TimePartitionRange(TimePartition begin, TimePartition end) throws LensException {
if (end.before(begin)) {
throw new LensException("condition of creation of timepartition failed: end>=begin");
http://git-wip-us.apache.org/repos/asf/lens/blob/08135aa6/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java
index 1b9a44a..6c9eb7a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/RangesPartitionTimeline.java
@@ -132,9 +132,9 @@ public class RangesPartitionTimeline extends PartitionTimeline {
private void mergeRanges() {
for (int i = 0; i < ranges.size() - 1; i++) {
- if (ranges.get(i).getEnd().equals(ranges.get(i + 1).getBegin())) {
+ if (ranges.get(i).getEnd().compareTo(ranges.get(i + 1).getBegin()) >= 0) {
TimePartitionRange removed = ranges.remove(i + 1);
- ranges.get(i).setEnd(removed.getEnd());
+ ranges.get(i).setEnd(TimePartition.max(removed.getEnd(), ranges.get(i).getEnd()));
i--; // check again at same index
} else if (ranges.get(i).isEmpty()) {
ranges.remove(i);
http://git-wip-us.apache.org/repos/asf/lens/blob/08135aa6/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java
index 2caea56..8681e90 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AbridgedTimeRangeWriter.java
@@ -121,11 +121,12 @@ public class AbridgedTimeRangeWriter implements TimeRangeWriter {
Map<FactPartition, Set<FactPartition>> partitionSetMap = new HashMap<FactPartition, Set<FactPartition>>();
for (FactPartition part : parts) {
FactPartition key = part.getContainingPart();
- part.setContainingPart(null);
+ FactPartition part2 = new FactPartition(part.getPartCol(), part.getPartSpec(), part.getPeriod(), null, part
+ .getPartFormat(), part.getStorageTables());
if (partitionSetMap.get(key) == null) {
partitionSetMap.put(key, Sets.<FactPartition>newTreeSet());
}
- partitionSetMap.get(key).add(part);
+ partitionSetMap.get(key).add(part2);
}
Map<Set<FactPartition>, Set<FactPartition>> setSetOppositeMap = Maps.newHashMap();
for (Map.Entry<FactPartition, Set<FactPartition>> entry : partitionSetMap.entrySet()) {
http://git-wip-us.apache.org/repos/asf/lens/blob/08135aa6/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index 58d0fa7..68ab5ab 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -369,7 +369,7 @@ class StorageTableResolver implements ContextRewriter {
// If no partitions were found, then we'll fallback.
String partCol = range.getPartitionColumn();
boolean partColNotSupported = rangeParts.isEmpty();
- for(String storage: cfact.fact.getStorages()) {
+ for (String storage : cfact.fact.getStorages()) {
String storageTableName = getFactOrDimtableStorageTableName(cfact.fact.getName(), storage).toLowerCase();
partColNotSupported &= skipStorageCauses.containsKey(storageTableName)
&& skipStorageCauses.get(storageTableName).getCause().equals(PART_COL_DOES_NOT_EXIST)
@@ -484,11 +484,11 @@ class StorageTableResolver implements ContextRewriter {
minimalPartsCopy.retainAll(rangeParts);
if (!StringUtils.isEmpty(whereClauseForFallback.get(trange))) {
rangeToWhere.put(
- rangeWriter.getTimeRangeWhereClause(cubeql, cubeql.getAliasForTableName(cubeql.getCube().getName()),
- minimalPartsCopy) + " and " + whereClauseForFallback.get(trange), table);
+ rangeWriter.getTimeRangeWhereClause(cubeql, cubeql.getAliasForTableName(cubeql.getCube().getName()),
+ minimalPartsCopy) + " and " + whereClauseForFallback.get(trange), table);
} else {
rangeToWhere.put(rangeWriter.getTimeRangeWhereClause(cubeql,
- cubeql.getAliasForTableName(cubeql.getCube().getName()), minimalPartsCopy), table);
+ cubeql.getAliasForTableName(cubeql.getCube().getName()), minimalPartsCopy), table);
}
}
cfact.getRangeToStorageWhereMap().put(trange, rangeToWhere);
@@ -506,7 +506,7 @@ class StorageTableResolver implements ContextRewriter {
HashMap<String, SkipStorageCause> skipStorageCauses,
PartitionRangesForPartitionColumns missingPartitions) throws LensException {
try {
- return getPartitions(fact, range, getValidUpdatePeriods(fact), true, skipStorageCauses,
+ return getPartitions(fact, range, getValidUpdatePeriods(fact), true, failOnPartialData, skipStorageCauses,
missingPartitions);
} catch (Exception e) {
throw new LensException(e);
@@ -514,22 +514,22 @@ class StorageTableResolver implements ContextRewriter {
}
private Set<FactPartition> getPartitions(CubeFactTable fact, TimeRange range, TreeSet<UpdatePeriod> updatePeriods,
- boolean addNonExistingParts, Map<String, SkipStorageCause> skipStorageCauses,
+ boolean addNonExistingParts, boolean failOnPartialData, Map<String, SkipStorageCause> skipStorageCauses,
PartitionRangesForPartitionColumns missingPartitions)
throws Exception {
- Set<FactPartition> partitions = new TreeSet<FactPartition>();
+ Set<FactPartition> partitions = new TreeSet<>();
if (range != null && range.isCoverableBy(updatePeriods)
&& getPartitions(fact, range.getFromDate(), range.getToDate(), range.getPartitionColumn(), partitions,
- updatePeriods, addNonExistingParts, skipStorageCauses, missingPartitions)) {
+ updatePeriods, addNonExistingParts, failOnPartialData, skipStorageCauses, missingPartitions)) {
return partitions;
} else {
- return new TreeSet<FactPartition>();
+ return new TreeSet<>();
}
}
private boolean getPartitions(CubeFactTable fact, Date fromDate, Date toDate, String partCol,
Set<FactPartition> partitions, TreeSet<UpdatePeriod> updatePeriods,
- boolean addNonExistingParts, Map<String, SkipStorageCause> skipStorageCauses,
+ boolean addNonExistingParts, boolean failOnPartialData, Map<String, SkipStorageCause> skipStorageCauses,
PartitionRangesForPartitionColumns missingPartitions)
throws Exception {
log.info("getPartitions for {} from fromDate:{} toDate:{}", fact, fromDate, toDate);
@@ -629,21 +629,18 @@ class StorageTableResolver implements ContextRewriter {
log.debug("Looking for process time partitions between {} and {}", pdt, nextPdt);
Set<FactPartition> processTimeParts =
getPartitions(fact, TimeRange.getBuilder().fromDate(pdt).toDate(nextPdt).partitionColumn(
- processTimePartCol).build(), newset, false, skipStorageCauses, missingPartitions);
+ processTimePartCol).build(), newset, true, false, skipStorageCauses, missingPartitions);
log.debug("Look ahead partitions: {}", processTimeParts);
TimeRange timeRange = TimeRange.getBuilder().fromDate(dt).toDate(nextDt).build();
for (FactPartition pPart : processTimeParts) {
log.debug("Looking for finer partitions in pPart: {}", pPart);
for (Date date : timeRange.iterable(pPart.getPeriod(), 1)) {
- partitions.add(new FactPartition(partCol, date, pPart.getPeriod(), pPart,
- partWhereClauseFormat));
+ FactPartition innerPart = new FactPartition(partCol, date, pPart.getPeriod(), pPart,
+ partWhereClauseFormat);
+ updateFactPartitionStorageTablesFrom(fact, innerPart, pPart.getStorageTables());
+ partitions.add(innerPart);
}
log.debug("added all sub partitions blindly in pPart: {}", pPart);
- // if (!getPartitions(fact, dt, cal.getTime(), partCol, pPart, partitions,
- // newset, false,
- // skipStorageCauses, nonExistingParts)) {
- // log.info("No partitions found in look ahead range");
- // }
}
}
}
@@ -654,7 +651,7 @@ class StorageTableResolver implements ContextRewriter {
TreeSet<UpdatePeriod> newset = new TreeSet<UpdatePeriod>();
newset.addAll(updatePeriods);
newset.remove(interval);
- if (!getPartitions(fact, dt, nextDt, partCol, partitions, newset, false, skipStorageCauses,
+ if (!getPartitions(fact, dt, nextDt, partCol, partitions, newset, false, failOnPartialData, skipStorageCauses,
missingPartitions)) {
log.debug("Adding non existing partition {}", part);
@@ -676,13 +673,13 @@ class StorageTableResolver implements ContextRewriter {
}
}
return getPartitions(fact, fromDate, ceilFromDate, partCol, partitions,
- updatePeriods, addNonExistingParts, skipStorageCauses, missingPartitions)
+ updatePeriods, addNonExistingParts, failOnPartialData, skipStorageCauses, missingPartitions)
&& getPartitions(fact, floorToDate, toDate, partCol, partitions,
- updatePeriods, addNonExistingParts, skipStorageCauses, missingPartitions);
+ updatePeriods, addNonExistingParts, failOnPartialData, skipStorageCauses, missingPartitions);
}
- private void updateFactPartitionStorageTablesFrom(CubeFactTable fact, FactPartition part,
- Set<String> storageTableNames) throws LensException, HiveException, ParseException {
+ void updateFactPartitionStorageTablesFrom(CubeFactTable fact,
+ FactPartition part, Set<String> storageTableNames) throws LensException, HiveException, ParseException {
for (String storageTableName : storageTableNames) {
if (client.factPartitionExists(fact, part, storageTableName)) {
part.getStorageTables().add(storageTableName);
http://git-wip-us.apache.org/repos/asf/lens/blob/08135aa6/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 7f56292..67f7ab9 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -19,6 +19,10 @@
package org.apache.lens.cube.parse;
+import static java.util.Calendar.DAY_OF_MONTH;
+import static java.util.Calendar.HOUR_OF_DAY;
+import static java.util.Calendar.MONTH;
+
import static org.apache.lens.cube.metadata.UpdatePeriod.*;
import static org.testng.Assert.assertEquals;
@@ -142,31 +146,31 @@ public class CubeTestSetup {
log.debug("Test now:{}", NOW);
// Figure out if current hour is 0th hour
- zerothHour = (cal.get(Calendar.HOUR_OF_DAY) == 0);
+ zerothHour = (cal.get(HOUR_OF_DAY) == 0);
// Figure out last hour
- cal.add(Calendar.HOUR_OF_DAY, -1);
+ cal.add(HOUR_OF_DAY, -1);
LAST_HOUR = cal.getTime();
log.debug("LastHour:{}", LAST_HOUR);
cal.setTime(NOW);
- cal.add(Calendar.DAY_OF_MONTH, -1);
+ cal.add(DAY_OF_MONTH, -1);
ONE_DAY_BACK = cal.getTime();
- cal.add(Calendar.DAY_OF_MONTH, -1);
+ cal.add(DAY_OF_MONTH, -1);
TWODAYS_BACK = cal.getTime();
System.out.println("Test TWODAYS_BACK:" + TWODAYS_BACK);
// two months back
cal.setTime(NOW);
- cal.add(Calendar.MONTH, -2);
+ cal.add(MONTH, -2);
TWO_MONTHS_BACK = cal.getTime();
System.out.println("Test TWO_MONTHS_BACK:" + TWO_MONTHS_BACK);
// Before 4days
cal.setTime(NOW);
- cal.add(Calendar.DAY_OF_MONTH, -4);
+ cal.add(DAY_OF_MONTH, -4);
BEFORE_4_DAYS_END = cal.getTime();
- cal.add(Calendar.DAY_OF_MONTH, -2);
+ cal.add(DAY_OF_MONTH, -2);
BEFORE_4_DAYS_START = cal.getTime();
@@ -426,7 +430,7 @@ public class CubeTestSetup {
}
Calendar cal = new GregorianCalendar();
cal.setTime(dayStart);
- if (cal.get(Calendar.DAY_OF_MONTH) != 1) {
+ if (cal.get(DAY_OF_MONTH) != 1) {
addParts(dailyparts, DAILY, dayStart, DateUtil.getCeilDate(TWO_MONTHS_BACK, MONTHLY));
monthStart = DateUtil.getCeilDate(TWO_MONTHS_BACK, MONTHLY);
}
@@ -478,7 +482,7 @@ public class CubeTestSetup {
}
Calendar cal = new GregorianCalendar();
cal.setTime(dayStart);
- if (cal.get(Calendar.DAY_OF_MONTH) != 1) {
+ if (cal.get(DAY_OF_MONTH) != 1) {
addParts(dailyparts, DAILY, dayStart, DateUtil.getCeilDate(TWO_MONTHS_BACK, MONTHLY));
monthStart = DateUtil.getCeilDate(TWO_MONTHS_BACK, MONTHLY);
}
@@ -1277,7 +1281,7 @@ public class CubeTestSetup {
timeParts.put("ttd2", temp);
StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY);
client.addPartition(sPartSpec, c99);
- cal.add(Calendar.HOUR_OF_DAY, 1);
+ cal.add(HOUR_OF_DAY, 1);
temp = cal.getTime();
}
@@ -1290,7 +1294,7 @@ public class CubeTestSetup {
timeParts.put("ttd2", temp);
StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY);
client.addPartition(sPartSpec, c99);
- cal.add(Calendar.HOUR_OF_DAY, 1);
+ cal.add(HOUR_OF_DAY, 1);
temp = cal.getTime();
}
}
@@ -1392,7 +1396,7 @@ public class CubeTestSetup {
} catch (LensException e) {
log.error("Encountered Lens exception.", e);
}
- cal.add(Calendar.HOUR_OF_DAY, 1);
+ cal.add(HOUR_OF_DAY, 1);
temp = cal.getTime();
}
@@ -1404,7 +1408,7 @@ public class CubeTestSetup {
timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), temp);
StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY);
client.addPartition(sPartSpec, c1);
- cal.add(Calendar.HOUR_OF_DAY, 1);
+ cal.add(HOUR_OF_DAY, 1);
temp = cal.getTime();
}
client.clearHiveTableCache();
@@ -1438,7 +1442,7 @@ public class CubeTestSetup {
partitions.add(HOURLY.format().format(temp));
StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY);
storagePartitionDescs.add(sPartSpec);
- cal.add(Calendar.HOUR_OF_DAY, 1);
+ cal.add(HOUR_OF_DAY, 1);
temp = cal.getTime();
}
client.addPartitions(storagePartitionDescs, c4);
@@ -1457,7 +1461,7 @@ public class CubeTestSetup {
timeParts.put("ttd2", temp);
StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY);
client.addPartition(sPartSpec, c4);
- cal.add(Calendar.HOUR_OF_DAY, 1);
+ cal.add(HOUR_OF_DAY, 1);
temp = cal.getTime();
}
}
@@ -1516,7 +1520,7 @@ public class CubeTestSetup {
timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), temp);
StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact2.getName(), timeParts, null, HOURLY);
client.addPartition(sPartSpec, c3);
- cal.add(Calendar.HOUR_OF_DAY, 1);
+ cal.add(HOUR_OF_DAY, 1);
temp = cal.getTime();
}
}
@@ -2309,10 +2313,10 @@ public class CubeTestSetup {
// Add partitions in PIE storage
Calendar pcal = Calendar.getInstance();
pcal.setTime(TWODAYS_BACK);
- pcal.set(Calendar.HOUR, 0);
+ pcal.set(HOUR_OF_DAY, 0);
Calendar ical = Calendar.getInstance();
ical.setTime(TWODAYS_BACK);
- ical.set(Calendar.HOUR, 0);
+ ical.set(HOUR_OF_DAY, 0);
Map<UpdatePeriod, TreeSet<Date>> pTimes = Maps.newHashMap();
pTimes.put(DAILY, Sets.<Date>newTreeSet());
@@ -2343,8 +2347,8 @@ public class CubeTestSetup {
pTimes.get(DAILY).add(ptime);
iTimes.get(DAILY).add(itime);
client.addPartition(sPartSpec, storageName);
- pcal.add(Calendar.DAY_OF_MONTH, 1);
- ical.add(Calendar.HOUR_OF_DAY, 20);
+ pcal.add(DAY_OF_MONTH, 1);
+ ical.add(HOUR_OF_DAY, 20);
} else if (p == 2) { // day2
// pt=day2-hour[0-3] it = day1-hour[20-23]
// pt=day2 and it=day1
@@ -2372,8 +2376,8 @@ public class CubeTestSetup {
pTimes.get(HOURLY).add(ptime);
iTimes.get(HOURLY).add(itime);
client.addPartition(sPartSpec, storageName);
- pcal.add(Calendar.HOUR_OF_DAY, 1);
- ical.add(Calendar.HOUR_OF_DAY, 1);
+ pcal.add(HOUR_OF_DAY, 1);
+ ical.add(HOUR_OF_DAY, 1);
}
// pt=day2 and it=day2
sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, DAILY);
@@ -2394,8 +2398,8 @@ public class CubeTestSetup {
pTimes.get(HOURLY).add(ptime);
iTimes.get(HOURLY).add(itime);
client.addPartition(sPartSpec, storageName);
- pcal.add(Calendar.HOUR_OF_DAY, 1);
- ical.add(Calendar.HOUR_OF_DAY, 1);
+ pcal.add(HOUR_OF_DAY, 1);
+ ical.add(HOUR_OF_DAY, 1);
}
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/08135aa6/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index 7e5184c..a58f5fe 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -19,6 +19,8 @@
package org.apache.lens.cube.parse;
+import static org.apache.lens.cube.metadata.UpdatePeriod.DAILY;
+import static org.apache.lens.cube.metadata.UpdatePeriod.HOURLY;
import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*;
import static org.apache.lens.cube.parse.CubeTestSetup.*;
@@ -46,6 +48,8 @@ import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@@ -91,14 +95,14 @@ public class TestCubeRewriter extends TestQueryRewrite {
String qFrom = qFmt.format(from2DaysBackDate);
CubeQueryContext rewrittenQuery = rewriteCtx("select SUM(msr15) from testCube where"
- + " time_range_in(d_time, '"+ qFrom + "', '" + qTo + "')", conf);
+ + " time_range_in(d_time, '" + qFrom + "', '" + qTo + "')", conf);
DateFormat fmt = UpdatePeriod.CONTINUOUS.format();
String to = fmt.format(toDate);
String from = fmt.format(from2DaysBackDate);
String expected = "select SUM((testCube.msr15)) from TestQueryRewrite.c0_testFact_CONTINUOUS testcube"
- + " WHERE ((( testcube . dt ) between '" + from + "' and '" + to + "' ))";
+ + " WHERE ((( testcube . dt ) between '" + from + "' and '" + to + "' ))";
System.out.println("rewrittenQuery.toHQL() " + rewrittenQuery.toHQL());
System.out.println("expected " + expected);
compareQueries(expected, rewrittenQuery.toHQL());
@@ -107,7 +111,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
rewrittenQuery = rewriteCtx("select SUM(msr2) from testCube where" + " time_range_in(d_time, '"
+ qFrom + "', '" + qTo + "')", conf);
expected = "select SUM((testCube.msr2)) from TestQueryRewrite.c0_testFact testcube"
- + " WHERE ((( testcube . dt ) between '" + from + "' and '" + to + "' ))";
+ + " WHERE ((( testcube . dt ) between '" + from + "' and '" + to + "' ))";
System.out.println("rewrittenQuery.toHQL() " + rewrittenQuery.toHQL());
System.out.println("expected " + expected);
compareQueries(expected, rewrittenQuery.toHQL());
@@ -117,7 +121,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
Date from4DaysBackDate = qCal.getTime();
String qFrom4DaysBackDate = qFmt.format(from4DaysBackDate);
LensException th = getLensExceptionInRewrite("select SUM(msr15) from testCube where"
- + " time_range_in(d_time, '"+ qFrom4DaysBackDate + "', '" + qTo + "')", getConf());
+ + " time_range_in(d_time, '" + qFrom4DaysBackDate + "', '" + qTo + "')", getConf());
assertEquals(th.getErrorCode(), LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo().getErrorCode());
}
@@ -427,9 +431,9 @@ public class TestCubeRewriter extends TestQueryRewrite {
System.out.println("HQL: " + hqlQuery);
String expected1 = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null,
- getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c1_testfact"));
+ getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c1_testfact"));
String expected2 = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null,
- getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c2_testfact"));
+ getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c2_testfact"));
System.out.println("Expected1 : " + expected1);
System.out.println("Expected2 : " + expected2);
@@ -460,9 +464,9 @@ public class TestCubeRewriter extends TestQueryRewrite {
System.out.println("HQL:" + hqlQuery);
String expected1 = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null,
- getWhereForDailyAndHourly2days(cubeName, "c1_testfact"));
+ getWhereForDailyAndHourly2days(cubeName, "c1_testfact"));
String expected2 = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null,
- getWhereForDailyAndHourly2days(cubeName, "c2_testfact"));
+ getWhereForDailyAndHourly2days(cubeName, "c2_testfact"));
System.out.println("Expected1 : " + expected1);
System.out.println("Expected2 : " + expected2);
@@ -496,11 +500,11 @@ public class TestCubeRewriter extends TestQueryRewrite {
System.out.println("HQL:" + hqlQuery);
String expected1 = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null,
- getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c1_testfact"));
+ getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c1_testfact"));
String expected2 = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null,
- getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c2_testfact"));
+ getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c2_testfact"));
String expected3 = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null,
- getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c3_testfact"));
+ getWhereForMonthlyDailyAndHourly2monthsUnionQuery("c3_testfact"));
System.out.println("Expected1 : " + expected1);
System.out.println("Expected2 : " + expected2);
@@ -1325,7 +1329,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
}
@Test
- public void testFactsWithTimedDimensionWithProcessTimeCol() throws Exception {
+ public void testLookAhead() throws Exception {
String twoDaysITRange =
"time_range_in(it, '" + CubeTestSetup.getDateUptoHours(TWODAYS_BACK) + "','"
+ CubeTestSetup.getDateUptoHours(NOW) + "')";
@@ -1333,41 +1337,40 @@ public class TestCubeRewriter extends TestQueryRewrite {
Configuration conf = getConf();
conf.set(CubeQueryConfUtil.PROCESS_TIME_PART_COL, "pt");
conf.setClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, AbridgedTimeRangeWriter.class, TimeRangeWriter.class);
- String hqlQuery = rewrite("select dim1, max(msr3)," + " msr2 from testCube" + " where " + twoDaysITRange, conf);
- System.out.println("Query With process time col:" + hqlQuery);
- String expected = getExpectedQuery(cubeName, "select testcube.dim1, max(testcube.msr3), sum(testcube.msr2) FROM ",
- null, " GROUP BY ( testcube . dim1 )",
- getWhereForDailyAndHourly2daysWithTimeDim(cubeName, "it", "C2_summary1"),
- null);
- // TODO compare queries
- // compareQueries(expected, hqlQuery);
- hqlQuery =
- rewrite("select dim1, dim2, COUNT(msr4)," + " SUM(msr2), msr3 from testCube" + " where " + twoDaysITRange, conf);
- System.out.println("Query With process time col:" + hqlQuery);
- // TODO compare queries
- // compareQueries(expected, hqlQuery);
- hqlQuery =
- rewrite("select dim1, dim2, cityid, count(msr4)," + " SUM(msr2), msr3 from testCube" + " where "
- + twoDaysITRange, conf);
- System.out.println("Query With process time col:" + hqlQuery);
- // TODO compare queries
- // compareQueries(expected, hqlQuery);
- conf.setInt(CubeQueryConfUtil.getLookAheadPTPartsKey(UpdatePeriod.DAILY), 3);
- hqlQuery = rewrite("select dim1, max(msr3)," + " msr2 from testCube" + " where " + twoDaysITRange, conf);
- System.out.println("Query With process time col:" + hqlQuery);
- // TODO compare queries
- // compareQueries(expected, hqlQuery);
- hqlQuery =
- rewrite("select dim1, dim2, COUNT(msr4)," + " SUM(msr2), msr3 from testCube" + " where " + twoDaysITRange, conf);
- System.out.println("Query With process time col:" + hqlQuery);
- // TODO compare queries
- // compareQueries(expected, hqlQuery);
- hqlQuery =
- rewrite("select dim1, dim2, cityid, count(msr4)," + " SUM(msr2), msr3 from testCube" + " where "
- + twoDaysITRange, conf);
- System.out.println("Query With process time col:" + hqlQuery);
- // TODO compare queries
- // compareQueries(expected, hqlQuery);
+ CubeQueryContext ctx = rewriteCtx("select dim1, max(msr3)," + " msr2 from testCube" + " where " + twoDaysITRange,
+ conf);
+ assertEquals(ctx.candidateFacts.size(), 1);
+ CandidateFact candidateFact = ctx.candidateFacts.iterator().next();
+ Set<FactPartition> partsQueried = new TreeSet<>(candidateFact.getPartsQueried());
+ Date ceilDay = DateUtil.getCeilDate(TWODAYS_BACK, DAILY);
+ Date nextDay = DateUtils.addDays(ceilDay, 1);
+ Date nextToNextDay = DateUtils.addDays(nextDay, 1);
+ HashSet<String> storageTables = Sets.newHashSet();
+ for (String storageTable : candidateFact.getStorageTables()) {
+ storageTables.add(storageTable.split("\\.")[1]);
+ }
+ TreeSet<FactPartition> expectedPartsQueried = Sets.newTreeSet();
+ for (TimePartition p : Iterables.concat(
+ TimePartition.of(HOURLY, TWODAYS_BACK).rangeUpto(TimePartition.of(HOURLY, ceilDay)),
+ TimePartition.of(DAILY, ceilDay).rangeUpto(TimePartition.of(DAILY, nextDay)),
+ TimePartition.of(HOURLY, nextDay).rangeUpto(TimePartition.of(HOURLY, NOW)))) {
+ FactPartition fp = new FactPartition("it", p, null, storageTables);
+ expectedPartsQueried.add(fp);
+ }
+ for (TimePartition it : TimePartition.of(HOURLY, ceilDay).rangeUpto(TimePartition.of(HOURLY, nextDay))) {
+ for (TimePartition pt : TimePartition.of(HOURLY, nextDay).rangeUpto(TimePartition.of(HOURLY, nextToNextDay))) {
+ FactPartition ptPartition = new FactPartition("pt", pt, null, storageTables);
+ FactPartition itPartition = new FactPartition("it", it, ptPartition, storageTables);
+ expectedPartsQueried.add(itPartition);
+ }
+ }
+ assertEquals(partsQueried, expectedPartsQueried);
+ conf.setInt(CubeQueryConfUtil.LOOK_AHEAD_PT_PARTS_PFX, 3);
+ ctx = rewriteCtx("select dim1, max(msr3)," + " msr2 from testCube" + " where " + twoDaysITRange,
+ conf);
+ partsQueried = new TreeSet<>(ctx.candidateFacts.iterator().next().getPartsQueried());
+ // pt does not exist beyond 1 day. So in this test, max look ahead possible is 3
+ assertEquals(partsQueried, expectedPartsQueried);
}
@Test