You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2021/07/06 13:03:16 UTC
[iotdb] branch groupbybug created (now abec445)
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a change to branch groupbybug
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at abec445 improve group by month performance
This branch includes the following new commits:
new abec445 improve group by month performance
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: improve group by month performance
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch groupbybug
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit abec4457ef4966c8ac7c40045a30d8e428e96ef3
Author: Alima777 <wx...@gmail.com>
AuthorDate: Tue Jul 6 21:02:25 2021 +0800
improve group by month performance
---
.../tsfile/read/filter/GroupByMonthFilter.java | 101 ++++++++++++++-------
.../tsfile/read/filter/GroupByMonthFilterTest.java | 22 +++++
2 files changed, 88 insertions(+), 35 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java
index 00a4d83..be772c6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilter.java
@@ -35,7 +35,6 @@ public class GroupByMonthFilter extends GroupByFilter {
private int intervalInMo;
private Calendar calendar = Calendar.getInstance();
private static final long MS_TO_MONTH = 30 * 86400_000L;
- private int intervalCnt = 0;
/** 10.31 -> 11.30 -> 12.31, not 10.31 -> 11.30 -> 12.30 */
private final long initialStartTime;
@@ -58,7 +57,7 @@ public class GroupByMonthFilter extends GroupByFilter {
if (isSlidingStepByMonth) {
slidingStepsInMo = (int) (slidingStep / MS_TO_MONTH);
}
- getNextIntervalAndSlidingStep();
+ getNthTimeInterval(0);
}
public GroupByMonthFilter(GroupByMonthFilter filter) {
@@ -67,7 +66,6 @@ public class GroupByMonthFilter extends GroupByFilter {
isSlidingStepByMonth = filter.isSlidingStepByMonth;
intervalInMo = filter.intervalInMo;
slidingStepsInMo = filter.slidingStepsInMo;
- intervalCnt = filter.intervalCnt;
initialStartTime = filter.initialStartTime;
calendar = Calendar.getInstance();
calendar.setTimeInMillis(filter.calendar.getTimeInMillis());
@@ -76,15 +74,15 @@ public class GroupByMonthFilter extends GroupByFilter {
// TODO: time descending order
@Override
public boolean satisfy(long time, Object value) {
+ boolean isSatisfy;
+ long count = getTimePointPosition(time);
+ getNthTimeInterval(count);
if (time < startTime || time >= endTime) {
- return false;
- } else if (time - startTime < interval) {
- return true;
+ isSatisfy = false;
} else {
- this.startTime = calendar.getTimeInMillis();
- getNextIntervalAndSlidingStep();
- return satisfy(time, value);
+ isSatisfy = time - startTime < interval;
}
+ return isSatisfy;
}
@Override
@@ -93,18 +91,18 @@ public class GroupByMonthFilter extends GroupByFilter {
if (isSatisfy) {
return true;
} else {
- long beforeStartTime = this.startTime;
- int beforeIntervalCnt = this.intervalCnt;
- // TODO: optimize to jump but not one by one
- while (endTime >= this.startTime && !isSatisfy) {
- this.startTime = calendar.getTimeInMillis();
- getNextIntervalAndSlidingStep();
- isSatisfy = satisfyCurrentInterval(startTime, endTime);
+ // get the interval which contains the start time
+ long count = getTimePointPosition(startTime);
+ getNthTimeInterval((int) count);
+ // judge two adjacent intervals
+ if (satisfyCurrentInterval(startTime, endTime)) {
+ isSatisfy = true;
+ } else {
+ getNthTimeInterval((int) count + 1);
+ if (satisfyCurrentInterval(startTime, endTime)) {
+ isSatisfy = true;
+ }
}
- // recover the initial state
- this.intervalCnt = beforeIntervalCnt - 1;
- this.startTime = beforeStartTime;
- getNextIntervalAndSlidingStep();
return isSatisfy;
}
}
@@ -118,7 +116,7 @@ public class GroupByMonthFilter extends GroupByFilter {
if (endTime < this.startTime || startTime >= this.endTime) {
return false;
} else {
- return startTime <= this.startTime || startTime - this.startTime < interval;
+ return startTime - this.startTime < interval;
}
}
@@ -128,17 +126,13 @@ public class GroupByMonthFilter extends GroupByFilter {
if (isContained) {
return true;
} else {
- long beforeStartTime = this.startTime;
- int beforeIntervalCnt = this.intervalCnt;
- while (!isContained && startTime >= this.startTime) {
- this.startTime = calendar.getTimeInMillis();
- getNextIntervalAndSlidingStep();
- isContained = isContainedByCurrentInterval(startTime, endTime);
+ // get the interval which contains the start time
+ long count = getTimePointPosition(startTime);
+ getNthTimeInterval((int) count);
+ // judge single interval that contains start time
+ if (isContainedByCurrentInterval(startTime, endTime)) {
+ isContained = true;
}
- // recover the initial state
- this.intervalCnt = beforeIntervalCnt - 1;
- this.startTime = beforeStartTime;
- getNextIntervalAndSlidingStep();
return isContained;
}
}
@@ -171,17 +165,54 @@ public class GroupByMonthFilter extends GroupByFilter {
interval, slidingStep, startTime, endTime, isSlidingStepByMonth, isIntervalByMonth);
}
- private void getNextIntervalAndSlidingStep() {
- intervalCnt++;
+ /** Get the interval that @param time belongs to */
+ private long getTimePointPosition(long time) {
+ long count;
+ if (isSlidingStepByMonth) {
+ count = (time - this.initialStartTime) / (slidingStepsInMo * 31 * 86400_000L);
+ calendar.setTimeInMillis(initialStartTime);
+ calendar.add(Calendar.MONTH, (int) count * slidingStepsInMo);
+ while (calendar.getTimeInMillis() < time) {
+ calendar.setTimeInMillis(initialStartTime);
+ calendar.add(Calendar.MONTH, (int) (count + 1) * slidingStepsInMo);
+ if (calendar.getTimeInMillis() > time) {
+ break;
+ } else {
+ count++;
+ }
+ }
+ } else {
+ count = (time - this.initialStartTime) / slidingStep;
+ }
+ return count;
+ }
+
+ /** get the Nth time interval */
+ private void getNthTimeInterval(long n) {
if (isIntervalByMonth) {
+ if (isSlidingStepByMonth) {
+ calendar.setTimeInMillis(initialStartTime);
+ calendar.add(Calendar.MONTH, (int) (slidingStepsInMo * n));
+ } else {
+ calendar.setTimeInMillis(initialStartTime + slidingStep * n);
+ }
+ this.startTime = calendar.getTimeInMillis();
calendar.setTimeInMillis(initialStartTime);
- calendar.add(Calendar.MONTH, slidingStepsInMo * (intervalCnt - 1) + intervalInMo);
+ calendar.add(Calendar.MONTH, (int) (slidingStepsInMo * n) + intervalInMo);
this.interval = calendar.getTimeInMillis() - startTime;
}
if (isSlidingStepByMonth) {
calendar.setTimeInMillis(initialStartTime);
- calendar.add(Calendar.MONTH, slidingStepsInMo * intervalCnt);
+ calendar.add(Calendar.MONTH, (int) (slidingStepsInMo * n));
+ this.startTime = calendar.getTimeInMillis();
+
+ calendar.setTimeInMillis(initialStartTime);
+ calendar.add(Calendar.MONTH, (int) (slidingStepsInMo * (n + 1)));
this.slidingStep = calendar.getTimeInMillis() - startTime;
+ } else {
+ // move calendar to the startTime of interval anyway
+ calendar.setTimeInMillis(initialStartTime + n * slidingStep);
+ this.startTime = calendar.getTimeInMillis();
}
}
}
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java
index 7f11ebd..d816090 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java
@@ -92,6 +92,9 @@ public class GroupByMonthFilterTest {
// 1970-03-01 08:00:00
assertTrue(filter.satisfy(5097600000L, null));
+ // 1970-12-30 08:00:00
+ assertTrue(filter.satisfy(31363200000L, null));
+
// 1970-12-31 23:59:58
assertTrue(filter.satisfy(31507198000L, null));
@@ -130,6 +133,25 @@ public class GroupByMonthFilterTest {
assertFalse(filter.satisfy(31507199000L, null));
}
+ /** Test filter with slidingStep = 100 days, and timeInterval = 1 mo */
+ @Test
+ public void TestSatisfy4() {
+ GroupByMonthFilter filter =
+ new GroupByMonthFilter(MS_TO_MONTH, MS_TO_DAY * 100, 0, END_TIME, false, true);
+
+ // 1970-01-01 08:00:00, timezone = GMT+08:00
+ assertTrue(filter.satisfy(0, null));
+
+ // 1970-02-01 07:59:59
+ assertTrue(filter.satisfy(2678399000L, null));
+
+ // 1970-03-01 08:00:00
+ assertFalse(filter.satisfy(5097600000L, null));
+
+ // 1970-05-01 08:00:00
+ assertTrue(filter.satisfy(10368000000L, null));
+ }
+
/** Test filter with slidingStep = 1 month, and timeInterval = 1 day */
@Test
public void TestSatisfyStartEndTime() {