You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2021/07/06 13:03:17 UTC
[iotdb] 01/01: improve group by month performance
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch groupbybug
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit abec4457ef4966c8ac7c40045a30d8e428e96ef3
Author: Alima777 <wx...@gmail.com>
AuthorDate: Tue Jul 6 21:02:25 2021 +0800
improve group by month performance
---
.../tsfile/read/filter/GroupByMonthFilter.java | 101 ++++++++++++++-------
.../tsfile/read/filter/GroupByMonthFilterTest.java | 22 +++++
2 files changed, 88 insertions(+), 35 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java
index 00a4d83..be772c6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java
@@ -35,7 +35,6 @@ public class GroupByMonthFilter extends GroupByFilter {
private int intervalInMo;
private Calendar calendar = Calendar.getInstance();
private static final long MS_TO_MONTH = 30 * 86400_000L;
- private int intervalCnt = 0;
/** 10.31 -> 11.30 -> 12.31, not 10.31 -> 11.30 -> 12.30 */
private final long initialStartTime;
@@ -58,7 +57,7 @@ public class GroupByMonthFilter extends GroupByFilter {
if (isSlidingStepByMonth) {
slidingStepsInMo = (int) (slidingStep / MS_TO_MONTH);
}
- getNextIntervalAndSlidingStep();
+ getNthTimeInterval(0);
}
public GroupByMonthFilter(GroupByMonthFilter filter) {
@@ -67,7 +66,6 @@ public class GroupByMonthFilter extends GroupByFilter {
isSlidingStepByMonth = filter.isSlidingStepByMonth;
intervalInMo = filter.intervalInMo;
slidingStepsInMo = filter.slidingStepsInMo;
- intervalCnt = filter.intervalCnt;
initialStartTime = filter.initialStartTime;
calendar = Calendar.getInstance();
calendar.setTimeInMillis(filter.calendar.getTimeInMillis());
@@ -76,15 +74,15 @@ public class GroupByMonthFilter extends GroupByFilter {
// TODO: time descending order
@Override
public boolean satisfy(long time, Object value) {
+ boolean isSatisfy;
+ long count = getTimePointPosition(time);
+ getNthTimeInterval(count);
if (time < startTime || time >= endTime) {
- return false;
- } else if (time - startTime < interval) {
- return true;
+ isSatisfy = false;
} else {
- this.startTime = calendar.getTimeInMillis();
- getNextIntervalAndSlidingStep();
- return satisfy(time, value);
+ isSatisfy = time - startTime < interval;
}
+ return isSatisfy;
}
@Override
@@ -93,18 +91,18 @@ public class GroupByMonthFilter extends GroupByFilter {
if (isSatisfy) {
return true;
} else {
- long beforeStartTime = this.startTime;
- int beforeIntervalCnt = this.intervalCnt;
- // TODO: optimize to jump but not one by one
- while (endTime >= this.startTime && !isSatisfy) {
- this.startTime = calendar.getTimeInMillis();
- getNextIntervalAndSlidingStep();
- isSatisfy = satisfyCurrentInterval(startTime, endTime);
+ // get the interval which contains the start time
+ long count = getTimePointPosition(startTime);
+ getNthTimeInterval((int) count);
+ // judge two adjacent intervals
+ if (satisfyCurrentInterval(startTime, endTime)) {
+ isSatisfy = true;
+ } else {
+ getNthTimeInterval((int) count + 1);
+ if (satisfyCurrentInterval(startTime, endTime)) {
+ isSatisfy = true;
+ }
}
- // recover the initial state
- this.intervalCnt = beforeIntervalCnt - 1;
- this.startTime = beforeStartTime;
- getNextIntervalAndSlidingStep();
return isSatisfy;
}
}
@@ -118,7 +116,7 @@ public class GroupByMonthFilter extends GroupByFilter {
if (endTime < this.startTime || startTime >= this.endTime) {
return false;
} else {
- return startTime <= this.startTime || startTime - this.startTime < interval;
+ return startTime - this.startTime < interval;
}
}
@@ -128,17 +126,13 @@ public class GroupByMonthFilter extends GroupByFilter {
if (isContained) {
return true;
} else {
- long beforeStartTime = this.startTime;
- int beforeIntervalCnt = this.intervalCnt;
- while (!isContained && startTime >= this.startTime) {
- this.startTime = calendar.getTimeInMillis();
- getNextIntervalAndSlidingStep();
- isContained = isContainedByCurrentInterval(startTime, endTime);
+ // get the interval which contains the start time
+ long count = getTimePointPosition(startTime);
+ getNthTimeInterval((int) count);
+ // judge single interval that contains start time
+ if (isContainedByCurrentInterval(startTime, endTime)) {
+ isContained = true;
}
- // recover the initial state
- this.intervalCnt = beforeIntervalCnt - 1;
- this.startTime = beforeStartTime;
- getNextIntervalAndSlidingStep();
return isContained;
}
}
@@ -171,17 +165,54 @@ public class GroupByMonthFilter extends GroupByFilter {
interval, slidingStep, startTime, endTime, isSlidingStepByMonth, isIntervalByMonth);
}
- private void getNextIntervalAndSlidingStep() {
- intervalCnt++;
+ /** Get the interval that @param time belongs to */
+ private long getTimePointPosition(long time) {
+ long count;
+ if (isSlidingStepByMonth) {
+ count = (time - this.initialStartTime) / (slidingStepsInMo * 31 * 86400_000L);
+ calendar.setTimeInMillis(initialStartTime);
+ calendar.add(Calendar.MONTH, (int) count * slidingStepsInMo);
+ while (calendar.getTimeInMillis() < time) {
+ calendar.setTimeInMillis(initialStartTime);
+ calendar.add(Calendar.MONTH, (int) (count + 1) * slidingStepsInMo);
+ if (calendar.getTimeInMillis() > time) {
+ break;
+ } else {
+ count++;
+ }
+ }
+ } else {
+ count = (time - this.initialStartTime) / slidingStep;
+ }
+ return count;
+ }
+
+ /** get the Nth time interval */
+ private void getNthTimeInterval(long n) {
if (isIntervalByMonth) {
+ if (isSlidingStepByMonth) {
+ calendar.setTimeInMillis(initialStartTime);
+ calendar.add(Calendar.MONTH, (int) (slidingStepsInMo * n));
+ } else {
+ calendar.setTimeInMillis(initialStartTime + slidingStep * n);
+ }
+ this.startTime = calendar.getTimeInMillis();
calendar.setTimeInMillis(initialStartTime);
- calendar.add(Calendar.MONTH, slidingStepsInMo * (intervalCnt - 1) + intervalInMo);
+ calendar.add(Calendar.MONTH, (int) (slidingStepsInMo * n) + intervalInMo);
this.interval = calendar.getTimeInMillis() - startTime;
}
if (isSlidingStepByMonth) {
calendar.setTimeInMillis(initialStartTime);
- calendar.add(Calendar.MONTH, slidingStepsInMo * intervalCnt);
+ calendar.add(Calendar.MONTH, (int) (slidingStepsInMo * n));
+ this.startTime = calendar.getTimeInMillis();
+
+ calendar.setTimeInMillis(initialStartTime);
+ calendar.add(Calendar.MONTH, (int) (slidingStepsInMo * (n + 1)));
this.slidingStep = calendar.getTimeInMillis() - startTime;
+ } else {
+ // move calendar to the startTime of interval anyway
+ calendar.setTimeInMillis(initialStartTime + n * slidingStep);
+ this.startTime = calendar.getTimeInMillis();
}
}
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java
index 7f11ebd..d816090 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java
@@ -92,6 +92,9 @@ public class GroupByMonthFilterTest {
// 1970-03-01 08:00:00
assertTrue(filter.satisfy(5097600000L, null));
+ // 1970-12-30 08:00:00
+ assertTrue(filter.satisfy(31363200000L, null));
+
// 1970-12-31 23:59:58
assertTrue(filter.satisfy(31507198000L, null));
@@ -130,6 +133,25 @@ public class GroupByMonthFilterTest {
assertFalse(filter.satisfy(31507199000L, null));
}
+ /** Test filter with slidingStep = 100 days, and timeInterval = 1 mo */
+ @Test
+ public void TestSatisfy4() {
+ GroupByMonthFilter filter =
+ new GroupByMonthFilter(MS_TO_MONTH, MS_TO_DAY * 100, 0, END_TIME, false, true);
+
+ // 1970-01-01 08:00:00, timezone = GMT+08:00
+ assertTrue(filter.satisfy(0, null));
+
+ // 1970-02-01 07:59:59
+ assertTrue(filter.satisfy(2678399000L, null));
+
+ // 1970-03-01 08:00:00
+ assertFalse(filter.satisfy(5097600000L, null));
+
+ // 1970-05-01 08:00:00
+ assertTrue(filter.satisfy(10368000000L, null));
+ }
+
/** Test filter with slidingStep = 1 month, and timeInterval = 1 day */
@Test
public void TestSatisfyStartEndTime() {