You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by su...@apache.org on 2017/12/27 08:13:22 UTC
lens git commit: LENS-1496 : Look ahead Time Partitions to be made
configurable
Repository: lens
Updated Branches:
refs/heads/master ae23e9c97 -> 53c342bd2
LENS-1496 : Look ahead Time Partitions to be made configurable
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/53c342bd
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/53c342bd
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/53c342bd
Branch: refs/heads/master
Commit: 53c342bd20dc35f75b80da78ac799c48cc2e04b1
Parents: ae23e9c
Author: Rajitha R <ra...@gmail.com>
Authored: Wed Dec 27 13:43:09 2017 +0530
Committer: sushilmohanty <su...@apache.org>
Committed: Wed Dec 27 13:43:09 2017 +0530
----------------------------------------------------------------------
.../lens/cube/parse/CubeQueryConfUtil.java | 6 +++
.../lens/cube/parse/StorageCandidate.java | 27 +++++++++---
.../src/main/resources/olap-query-conf.xml | 8 ++++
.../apache/lens/cube/metadata/DateFactory.java | 2 +
.../lens/cube/parse/TestCubeRewriter.java | 43 ++++++++++++++++++++
5 files changed, 80 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/53c342bd/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
index eeaa3af..4768fb8 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java
@@ -48,6 +48,7 @@ public final class CubeQueryConfUtil {
public static final String PROCESS_TIME_PART_COL = "lens.cube.query.process.time" + ".partition.column";
public static final String COMPLETENESS_CHECK_PART_COL = "lens.cube.query.completeness.check.partition.column";
public static final String LOOK_AHEAD_PT_PARTS_PFX = "lens.cube.query.lookahead.ptparts.forinterval.";
+ public static final String LOOK_AHEAD_TIME_PARTS_PFX = "lens.cube.query.lookahead.timeparts.forinterval.";
public static final String ENABLE_GROUP_BY_TO_SELECT = "lens.cube.query.promote.groupby.toselect";
public static final String ENABLE_SELECT_TO_GROUPBY = "lens.cube.query.promote.select.togroupby";
public static final String ENABLE_ATTRFIELDS_ADD_DISTINCT = "lens.cube.query.enable.attrfields.add.distinct";
@@ -58,6 +59,7 @@ public final class CubeQueryConfUtil {
public static final String REPLACE_TIMEDIM_WITH_PART_COL = "lens.cube.query.replace.timedim";
public static final boolean DEFAULT_MULTI_TABLE_SELECT = true;
public static final int DEFAULT_LOOK_AHEAD_PT_PARTS = 1;
+ public static final int DEFAULT_LOOK_AHEAD_TIME_PARTS = 1;
public static final boolean DEFAULT_ENABLE_GROUP_BY_TO_SELECT = false;
public static final boolean DEFAULT_ENABLE_SELECT_TO_GROUPBY = false;
public static final boolean DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL = true;
@@ -66,6 +68,10 @@ public final class CubeQueryConfUtil {
return LOOK_AHEAD_PT_PARTS_PFX + interval.name().toLowerCase();
}
+ public static String getLookAheadTimePartsKey(UpdatePeriod interval) {
+ return LOOK_AHEAD_TIME_PARTS_PFX + interval.name().toLowerCase();
+ }
+
private static String getValidKeyCubePFX(String cubeName) {
return VALID_PFX + cubeName.toLowerCase();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/53c342bd/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index 3ee66ee..b48be7f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -456,6 +456,10 @@ public class StorageCandidate implements Candidate, CandidateTable {
int lookAheadNumParts = getConf()
.getInt(CubeQueryConfUtil.getLookAheadPTPartsKey(maxInterval), CubeQueryConfUtil.DEFAULT_LOOK_AHEAD_PT_PARTS);
+
+ int lookAheadNumTimeParts = getConf()
+ .getInt(CubeQueryConfUtil.getLookAheadTimePartsKey(maxInterval), CubeQueryConfUtil.DEFAULT_LOOK_AHEAD_TIME_PARTS);
+
TimeRange.Iterable.Iterator iter = TimeRange.iterable(ceilFromDate, floorToDate, maxInterval, 1).iterator();
// add partitions from ceilFrom to floorTo
while (iter.hasNext()) {
@@ -475,7 +479,7 @@ public class StorageCandidate implements Candidate, CandidateTable {
log.debug("part column is process time col");
} else if (updatePeriods.first().equals(maxInterval)) {
log.debug("Update period is the least update period");
- } else if ((iter.getNumIters() - iter.getCounter()) > lookAheadNumParts) {
+ } else if ((iter.getNumIters() - iter.getCounter()) > lookAheadNumTimeParts) {
// see if this is the part of the last-n look ahead partitions
log.debug("Not a look ahead partition");
} else {
@@ -486,16 +490,27 @@ public class StorageCandidate implements Candidate, CandidateTable {
// process time are present
TimeRange.Iterable.Iterator processTimeIter = TimeRange.iterable(nextDt, lookAheadNumParts, maxInterval, 1)
.iterator();
+ TimeRange.Iterable.Iterator timeIter = TimeRange.iterable(nextDt, lookAheadNumTimeParts, maxInterval, 1)
+ .iterator();
while (processTimeIter.hasNext()) {
Date pdt = processTimeIter.next();
Date nextPdt = processTimeIter.peekNext();
- FactPartition processTimePartition = new FactPartition(processTimePartCol, pdt, maxInterval, null,
- partWhereClauseFormat);
- updatePartitionStorage(processTimePartition);
- if (processTimePartition.isFound()) {
+ FactPartition currFactPartition;
+ boolean allProcessTimePartitionsFound = true;
+ while (timeIter.hasNext()){
+ Date date = timeIter.next();
+ currFactPartition = new FactPartition(processTimePartCol, date, maxInterval, null,
+ partWhereClauseFormat);
+ updatePartitionStorage(currFactPartition);
+ if (!currFactPartition.isFound()) {
+ log.debug("Looked ahead process time partition {} is not found : " + currFactPartition);
+ allProcessTimePartitionsFound = false;
+ break;
+ }
+ }
+ if (allProcessTimePartitionsFound) {
log.debug("Finer parts not required for look-ahead partition :{}", part);
} else {
- log.debug("Looked ahead process time partition {} is not found", processTimePartition);
TreeSet<UpdatePeriod> newset = new TreeSet<UpdatePeriod>();
newset.addAll(updatePeriods);
newset.remove(maxInterval);
http://git-wip-us.apache.org/repos/asf/lens/blob/53c342bd/lens-cube/src/main/resources/olap-query-conf.xml
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/resources/olap-query-conf.xml b/lens-cube/src/main/resources/olap-query-conf.xml
index b389d6a..d0e5eb4 100644
--- a/lens-cube/src/main/resources/olap-query-conf.xml
+++ b/lens-cube/src/main/resources/olap-query-conf.xml
@@ -153,6 +153,14 @@
</property>
<property>
+ <name>lens.cube.query.lookahead.timeparts.forinterval.${interval}</name>
+ <value>1</value>
+ <description>The number of time partitions for interval specified where look ahead is applied.
+ Interval can be any Update period.
+ </description>
+ </property>
+
+ <property>
<name>lens.cube.query.replace.timedim</name>
<value>true</value>
<description>Tells whether timedim attribute queried in the time range should be replaced with its corresponding
http://git-wip-us.apache.org/repos/asf/lens/blob/53c342bd/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
index 75f2abd..c4404f1 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/DateFactory.java
@@ -168,6 +168,7 @@ public class DateFactory {
public static final String TWO_DAYS_RANGE_TTD_BEFORE_4_DAYS;
public static final String TWO_DAYS_RANGE_TTD2;
public static final String TWO_DAYS_RANGE_TTD2_BEFORE_4_DAYS;
+ public static final String THREE_DAYS_RANGE_IT;
public static final String TWO_DAYS_RANGE_IT;
public static final String ONE_DAY_RANGE_IT;
public static final String THIS_YEAR_RANGE;
@@ -218,6 +219,7 @@ public class DateFactory {
TWO_DAYS_RANGE_TTD_BEFORE_4_DAYS = getTimeRangeString("test_time_dim", DAILY, -6, -4, HOURLY);
TWO_DAYS_RANGE_TTD2 = getTimeRangeString("test_time_dim2", DAILY, -2, 0, HOURLY);
TWO_DAYS_RANGE_TTD2_BEFORE_4_DAYS = getTimeRangeString("test_time_dim2", DAILY, -6, -4, HOURLY);
+ THREE_DAYS_RANGE_IT = getTimeRangeString("it", DAILY, -3, 0, HOURLY);
TWO_DAYS_RANGE_IT = getTimeRangeString("it", DAILY, -2, 0, HOURLY);
ONE_DAY_RANGE_IT = getTimeRangeString("it", DAILY, -1, 0, DAILY);
THIS_YEAR_RANGE = getTimeRangeString(YEARLY, 0, 1);
http://git-wip-us.apache.org/repos/asf/lens/blob/53c342bd/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index 9b54957..e83eacb 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -1467,6 +1467,49 @@ public class TestCubeRewriter extends TestQueryRewrite {
.getParticipatingPartitions());
// pt does not exist beyond 1 day. So in this test, max look ahead possible is 3
assertEquals(partsQueried, expectedPartsQueried);
+
+ //New test case for testing look ahead time parts
+ expectedPartsQueried = Sets.newTreeSet();
+ conf.setInt(CubeQueryConfUtil.LOOK_AHEAD_TIME_PARTS_PFX + "daily", 2);
+
+ ceilDay = DAILY.getCeilDate(getDateWithOffset(DAILY, -3));
+ nextDay = DateUtils.addDays(ceilDay, 1);
+ nextToNextDay = DateUtils.addDays(nextDay, 1);
+
+ for (TimePartition p : Iterables.concat(
+ TimePartition.of(HOURLY, getDateWithOffset(DAILY, -3)).rangeUpto(TimePartition.of(HOURLY, ceilDay)),
+ TimePartition.of(DAILY, ceilDay).rangeUpto(TimePartition.of(DAILY, nextDay)),
+ TimePartition.of(DAILY, nextDay).rangeUpto(TimePartition.of(DAILY, nextToNextDay)),
+ TimePartition.of(HOURLY, nextToNextDay).rangeUpto(TimePartition.of(HOURLY, NOW)))) {
+ FactPartition fp = new FactPartition("it", p, null, storageTables);
+ expectedPartsQueried.add(fp);
+ }
+ for (TimePartition it : TimePartition.of(HOURLY, ceilDay).rangeUpto(TimePartition.of(HOURLY, nextDay))) {
+ for (TimePartition pt : TimePartition.of(HOURLY, nextDay).rangeUpto(TimePartition.of(HOURLY, nextToNextDay))) {
+ FactPartition ptPartition = new FactPartition("pt", pt, null, storageTables);
+ FactPartition itPartition = new FactPartition("it", it, ptPartition, storageTables);
+ expectedPartsQueried.add(itPartition);
+ }
+ }
+
+ ceilDay = DateUtils.addDays(ceilDay, 1);
+ nextDay = DateUtils.addDays(nextDay, 1);
+ nextToNextDay = DateUtils.addDays(nextToNextDay, 1);
+
+ for (TimePartition it : TimePartition.of(HOURLY, ceilDay).rangeUpto(TimePartition.of(HOURLY, nextDay))) {
+ for (TimePartition pt : TimePartition.of(HOURLY, nextDay).rangeUpto(TimePartition.of(HOURLY, nextToNextDay))) {
+ FactPartition ptPartition = new FactPartition("pt", pt, null, storageTables);
+ FactPartition itPartition = new FactPartition("it", it, ptPartition, storageTables);
+ expectedPartsQueried.add(itPartition);
+ }
+ }
+
+ ctx = rewriteCtx("select dim1, max(msr3)," + " msr2 from testCube" + " where " + THREE_DAYS_RANGE_IT,
+ conf);
+ partsQueried = new TreeSet<>(((StorageCandidate)ctx.getCandidates().iterator().next())
+ .getParticipatingPartitions());
+ // pt does not exist beyond 1 day. So in this test, max look ahead possible is 3
+ assertEquals(partsQueried, expectedPartsQueried);
}
@Test