You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2021/07/09 03:24:11 UTC
[iotdb] branch rel/0.12 updated: [To rel/0.12] Improve group by
month performance (#3518)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12 by this push:
new eaf27f5 [To rel/0.12] Improve group by month performance (#3518)
eaf27f5 is described below
commit eaf27f500d2b65d30990c964aa32a6b37b738ecc
Author: Xiangwei Wei <34...@users.noreply.github.com>
AuthorDate: Fri Jul 9 11:23:39 2021 +0800
[To rel/0.12] Improve group by month performance (#3518)
---
.../tsfile/read/filter/GroupByMonthFilter.java | 108 +++++++++++++--------
.../tsfile/read/filter/GroupByMonthFilterTest.java | 22 +++++
2 files changed, 87 insertions(+), 43 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java
index 00a4d83..df64437 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java
@@ -35,7 +35,6 @@ public class GroupByMonthFilter extends GroupByFilter {
private int intervalInMo;
private Calendar calendar = Calendar.getInstance();
private static final long MS_TO_MONTH = 30 * 86400_000L;
- private int intervalCnt = 0;
/** 10.31 -> 11.30 -> 12.31, not 10.31 -> 11.30 -> 12.30 */
private final long initialStartTime;
@@ -58,7 +57,7 @@ public class GroupByMonthFilter extends GroupByFilter {
if (isSlidingStepByMonth) {
slidingStepsInMo = (int) (slidingStep / MS_TO_MONTH);
}
- getNextIntervalAndSlidingStep();
+ getNthTimeInterval(0);
}
public GroupByMonthFilter(GroupByMonthFilter filter) {
@@ -67,7 +66,6 @@ public class GroupByMonthFilter extends GroupByFilter {
isSlidingStepByMonth = filter.isSlidingStepByMonth;
intervalInMo = filter.intervalInMo;
slidingStepsInMo = filter.slidingStepsInMo;
- intervalCnt = filter.intervalCnt;
initialStartTime = filter.initialStartTime;
calendar = Calendar.getInstance();
calendar.setTimeInMillis(filter.calendar.getTimeInMillis());
@@ -76,36 +74,32 @@ public class GroupByMonthFilter extends GroupByFilter {
// TODO: time descending order
@Override
public boolean satisfy(long time, Object value) {
- if (time < startTime || time >= endTime) {
+ if (time < initialStartTime || time >= endTime) {
return false;
- } else if (time - startTime < interval) {
- return true;
+ } else if (time >= startTime && time < startTime + slidingStep) {
+ return time - startTime < interval;
} else {
- this.startTime = calendar.getTimeInMillis();
- getNextIntervalAndSlidingStep();
- return satisfy(time, value);
+ long count = getTimePointPosition(time);
+ getNthTimeInterval(count);
+ return time - startTime < interval;
}
}
@Override
public boolean satisfyStartEndTime(long startTime, long endTime) {
- boolean isSatisfy = satisfyCurrentInterval(startTime, endTime);
- if (isSatisfy) {
+ if (satisfyCurrentInterval(startTime, endTime)) {
return true;
} else {
- long beforeStartTime = this.startTime;
- int beforeIntervalCnt = this.intervalCnt;
- // TODO: optimize to jump but not one by one
- while (endTime >= this.startTime && !isSatisfy) {
- this.startTime = calendar.getTimeInMillis();
- getNextIntervalAndSlidingStep();
- isSatisfy = satisfyCurrentInterval(startTime, endTime);
+ // get the interval which contains the start time
+ long count = getTimePointPosition(startTime);
+ getNthTimeInterval(count);
+ // judge two adjacent intervals
+ if (satisfyCurrentInterval(startTime, endTime)) {
+ return true;
+ } else {
+ getNthTimeInterval(count + 1);
+ return satisfyCurrentInterval(startTime, endTime);
}
- // recover the initial state
- this.intervalCnt = beforeIntervalCnt - 1;
- this.startTime = beforeStartTime;
- getNextIntervalAndSlidingStep();
- return isSatisfy;
}
}
@@ -118,28 +112,20 @@ public class GroupByMonthFilter extends GroupByFilter {
if (endTime < this.startTime || startTime >= this.endTime) {
return false;
} else {
- return startTime <= this.startTime || startTime - this.startTime < interval;
+ return startTime - this.startTime < interval;
}
}
@Override
public boolean containStartEndTime(long startTime, long endTime) {
- boolean isContained = isContainedByCurrentInterval(startTime, endTime);
- if (isContained) {
+ if (isContainedByCurrentInterval(startTime, endTime)) {
return true;
} else {
- long beforeStartTime = this.startTime;
- int beforeIntervalCnt = this.intervalCnt;
- while (!isContained && startTime >= this.startTime) {
- this.startTime = calendar.getTimeInMillis();
- getNextIntervalAndSlidingStep();
- isContained = isContainedByCurrentInterval(startTime, endTime);
- }
- // recover the initial state
- this.intervalCnt = beforeIntervalCnt - 1;
- this.startTime = beforeStartTime;
- getNextIntervalAndSlidingStep();
- return isContained;
+ // get the interval which contains the start time
+ long count = getTimePointPosition(startTime);
+ getNthTimeInterval(count);
+ // judge single interval that contains start time
+ return isContainedByCurrentInterval(startTime, endTime);
}
}
@@ -171,16 +157,52 @@ public class GroupByMonthFilter extends GroupByFilter {
interval, slidingStep, startTime, endTime, isSlidingStepByMonth, isIntervalByMonth);
}
- private void getNextIntervalAndSlidingStep() {
- intervalCnt++;
- if (isIntervalByMonth) {
+ /** Get the interval that @param time belongs to. */
+ private long getTimePointPosition(long time) {
+ long count;
+ if (isSlidingStepByMonth) {
+ count = (time - this.initialStartTime) / (slidingStepsInMo * 31 * 86400_000L);
+ calendar.setTimeInMillis(initialStartTime);
+ calendar.add(Calendar.MONTH, (int) count * slidingStepsInMo);
+ while (calendar.getTimeInMillis() < time) {
+ calendar.setTimeInMillis(initialStartTime);
+ calendar.add(Calendar.MONTH, (int) (count + 1) * slidingStepsInMo);
+ if (calendar.getTimeInMillis() > time) {
+ break;
+ } else {
+ count++;
+ }
+ }
+ } else {
+ count = (time - this.initialStartTime) / slidingStep;
+ }
+ return count;
+ }
+
+ /** get the Nth time interval. */
+ private void getNthTimeInterval(long n) {
+ // get start time of time interval
+ if (isSlidingStepByMonth) {
calendar.setTimeInMillis(initialStartTime);
- calendar.add(Calendar.MONTH, slidingStepsInMo * (intervalCnt - 1) + intervalInMo);
+ calendar.add(Calendar.MONTH, (int) (slidingStepsInMo * n));
+ } else {
+ calendar.setTimeInMillis(initialStartTime + slidingStep * n);
+ }
+ this.startTime = calendar.getTimeInMillis();
+
+ // get interval and sliding step
+ if (isIntervalByMonth) {
+ if (isSlidingStepByMonth) {
+ calendar.setTimeInMillis(initialStartTime);
+ calendar.add(Calendar.MONTH, (int) (slidingStepsInMo * n) + intervalInMo);
+ } else {
+ calendar.add(Calendar.MONTH, intervalInMo);
+ }
this.interval = calendar.getTimeInMillis() - startTime;
}
if (isSlidingStepByMonth) {
calendar.setTimeInMillis(initialStartTime);
- calendar.add(Calendar.MONTH, slidingStepsInMo * intervalCnt);
+ calendar.add(Calendar.MONTH, (int) (slidingStepsInMo * (n + 1)));
this.slidingStep = calendar.getTimeInMillis() - startTime;
}
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java
index 7f11ebd..d816090 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java
@@ -92,6 +92,9 @@ public class GroupByMonthFilterTest {
// 1970-03-01 08:00:00
assertTrue(filter.satisfy(5097600000L, null));
+ // 1970-12-30 08:00:00
+ assertTrue(filter.satisfy(31363200000L, null));
+
// 1970-12-31 23:59:58
assertTrue(filter.satisfy(31507198000L, null));
@@ -130,6 +133,25 @@ public class GroupByMonthFilterTest {
assertFalse(filter.satisfy(31507199000L, null));
}
+ /** Test filter with slidingStep = 100 days, and timeInterval = 1 mo */
+ @Test
+ public void TestSatisfy4() {
+ GroupByMonthFilter filter =
+ new GroupByMonthFilter(MS_TO_MONTH, MS_TO_DAY * 100, 0, END_TIME, false, true);
+
+ // 1970-01-01 08:00:00, timezone = GMT+08:00
+ assertTrue(filter.satisfy(0, null));
+
+ // 1970-02-01 07:59:59
+ assertTrue(filter.satisfy(2678399000L, null));
+
+ // 1970-03-01 08:00:00
+ assertFalse(filter.satisfy(5097600000L, null));
+
+ // 1970-05-01 08:00:00
+ assertTrue(filter.satisfy(10368000000L, null));
+ }
+
/** Test filter with slidingStep = 1 month, and timeInterval = 1 day */
@Test
public void TestSatisfyStartEndTime() {