You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2023/11/13 04:42:22 UTC
(iotdb) branch IOTDB-6243 updated: Add UT
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch IOTDB-6243
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/IOTDB-6243 by this push:
new e457adacdc2 Add UT
e457adacdc2 is described below
commit e457adacdc23eb2a8b0f00e07a931124c0a34529
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Mon Nov 13 12:42:12 2023 +0800
Add UT
---
.../filter/AbstractMonthIntervalFillFilter.java | 26 ++--
.../fill/filter/MonthIntervalMSFillFilter.java | 14 +-
.../fill/filter/MonthIntervalNSFillFilter.java | 14 +-
.../fill/filter/MonthIntervalUSFillFilter.java | 20 ++-
.../execution/operator/FillOperatorTest.java | 168 +++++++++++++++++++++
.../fill/filter/MonthIntervalMSFillFilterTest.java | 55 +++++++
6 files changed, 269 insertions(+), 28 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/AbstractMonthIntervalFillFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/AbstractMonthIntervalFillFilter.java
index 78b48d2fd89..99d5792eb26 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/AbstractMonthIntervalFillFilter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/AbstractMonthIntervalFillFilter.java
@@ -22,33 +22,25 @@ package org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter;
import org.apache.iotdb.db.queryengine.execution.operator.process.fill.IFillFilter;
import java.time.Instant;
-import java.time.LocalDateTime;
import java.time.ZoneId;
+import java.time.ZoneOffset;
public abstract class AbstractMonthIntervalFillFilter implements IFillFilter {
// month part of time duration
- public final int monthDuration;
+ protected final int monthDuration;
+
+ protected final ZoneId zone;
+
// non-month part of time duration, its precision is same as current time_precision
- public final long nonMonthDuration;
+ protected final long nonMonthDuration;
- public final ZoneId zone;
+ protected final ZoneOffset zoneOffset;
- public AbstractMonthIntervalFillFilter(int monthDuration, long nonMonthDuration, ZoneId zone) {
+ AbstractMonthIntervalFillFilter(int monthDuration, long nonMonthDuration, ZoneId zone) {
this.monthDuration = monthDuration;
this.nonMonthDuration = nonMonthDuration;
this.zone = zone;
+ this.zoneOffset = zone.getRules().getOffset(Instant.now());
}
-
- @Override
- public boolean needFill(long time, long previousTime) {
-
- long smaller = Math.min(time, previousTime);
- long greater = Math.max(time, previousTime);
- Instant instant = Instant.ofEpochSecond(smaller, smaller);
- LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, zone);
- return needFill(localDateTime, greater);
- }
-
- abstract boolean needFill(LocalDateTime smaller, long greater);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilter.java
index 4885152d1c3..ff601056914 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilter.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter;
+import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
@@ -29,9 +30,14 @@ public class MonthIntervalMSFillFilter extends AbstractMonthIntervalFillFilter {
}
@Override
- boolean needFill(LocalDateTime smaller, long greater) {
- LocalDateTime localDateTime =
- smaller.plusMonths(monthDuration).plusNanos(nonMonthDuration * 1_000_000L);
- return greater >= (localDateTime.getSecond() * 1_000L + localDateTime.getNano() / 1_000_000);
+ public boolean needFill(long time, long previousTime) {
+ long smaller = Math.min(time, previousTime);
+ long greater = Math.max(time, previousTime);
+ Instant instant = Instant.ofEpochMilli(smaller);
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, zone);
+ long epochMilli =
+ localDateTime.plusMonths(monthDuration).toInstant(zoneOffset).toEpochMilli()
+ + nonMonthDuration;
+ return epochMilli >= greater;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalNSFillFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalNSFillFilter.java
index ec376051dae..5fad98dcaa8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalNSFillFilter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalNSFillFilter.java
@@ -19,8 +19,10 @@
package org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter;
+import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
+import java.time.temporal.ChronoField;
public class MonthIntervalNSFillFilter extends AbstractMonthIntervalFillFilter {
@@ -29,8 +31,14 @@ public class MonthIntervalNSFillFilter extends AbstractMonthIntervalFillFilter {
}
@Override
- boolean needFill(LocalDateTime smaller, long greater) {
- LocalDateTime localDateTime = smaller.plusMonths(monthDuration).plusNanos(nonMonthDuration);
- return greater >= (localDateTime.getSecond() * 1_000_000_000L + localDateTime.getNano());
+ public boolean needFill(long time, long previousTime) {
+ long smaller = Math.min(time, previousTime);
+ long greater = Math.max(time, previousTime);
+ Instant instant = Instant.ofEpochSecond(smaller / 1_000_000_000L, smaller % 1_000_000_000L);
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, zone);
+ Instant upper =
+ localDateTime.plusMonths(monthDuration).plusNanos(nonMonthDuration).toInstant(zoneOffset);
+ long timeInNs = upper.getLong(ChronoField.NANO_OF_SECOND) + upper.getEpochSecond();
+ return timeInNs >= greater;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalUSFillFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalUSFillFilter.java
index 68afcda4ead..d5f0647d0b4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalUSFillFilter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalUSFillFilter.java
@@ -19,8 +19,12 @@
package org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter;
+import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
+import java.time.temporal.ChronoField;
+
+import static java.time.temporal.ChronoUnit.MICROS;
public class MonthIntervalUSFillFilter extends AbstractMonthIntervalFillFilter {
@@ -29,9 +33,17 @@ public class MonthIntervalUSFillFilter extends AbstractMonthIntervalFillFilter {
}
@Override
- boolean needFill(LocalDateTime smaller, long greater) {
- LocalDateTime localDateTime =
- smaller.plusMonths(monthDuration).plusNanos(nonMonthDuration * 1_000L);
- return greater >= (localDateTime.getSecond() * 1_000_000L + localDateTime.getNano() / 1_000);
+ public boolean needFill(long time, long previousTime) {
+ long smaller = Math.min(time, previousTime);
+ long greater = Math.max(time, previousTime);
+ Instant instant = Instant.ofEpochSecond(smaller / 1_000_000_000L, smaller % 1_000_000_000L);
+ LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, zone);
+ Instant upper =
+ localDateTime
+ .plusMonths(monthDuration)
+ .plus(nonMonthDuration, MICROS)
+ .toInstant(zoneOffset);
+ long timeInUs = upper.getLong(ChronoField.MICRO_OF_SECOND) + instant.getEpochSecond();
+ return timeInUs >= greater;
}
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/FillOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/FillOperatorTest.java
index 96a528eecda..94eaba5dd37 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/FillOperatorTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/FillOperatorTest.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.process.FillOperator;
import org.apache.iotdb.db.queryengine.execution.operator.process.fill.IFill;
import org.apache.iotdb.db.queryengine.execution.operator.process.fill.IFillFilter;
import org.apache.iotdb.db.queryengine.execution.operator.process.fill.constant.DoubleConstantFill;
+import org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter.FixedIntervalFillFilter;
import org.apache.iotdb.db.queryengine.execution.operator.process.fill.previous.IntPreviousFill;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.tsfile.enums.TSDataType;
@@ -386,4 +387,171 @@ public class FillOperatorTest {
instanceNotificationExecutor.shutdown();
}
}
+
+ @Test
+ public void batchPreviousFillWithThresholdTest() throws Exception {
+ ExecutorService instanceNotificationExecutor =
+ IoTDBThreadPoolFactory.newFixedThreadPool(1, "test-instance-notification");
+ try {
+ QueryId queryId = new QueryId("stub_query");
+ FragmentInstanceId instanceId =
+ new FragmentInstanceId(new PlanFragmentId(queryId, 0), "stub-instance");
+ FragmentInstanceStateMachine stateMachine =
+ new FragmentInstanceStateMachine(instanceId, instanceNotificationExecutor);
+ FragmentInstanceContext fragmentInstanceContext =
+ createFragmentInstanceContext(instanceId, stateMachine);
+ DriverContext driverContext = new DriverContext(fragmentInstanceContext, 0);
+ PlanNodeId planNodeId1 = new PlanNodeId("1");
+ driverContext.addOperatorContext(1, planNodeId1, FillOperator.class.getSimpleName());
+
+ IFill[] fillArray =
+ new IFill[] {
+ new IntPreviousFill(new FixedIntervalFillFilter(2)),
+ new IntPreviousFill(new FixedIntervalFillFilter(100)),
+ new IntPreviousFill(new FixedIntervalFillFilter(1))
+ };
+ FillOperator fillOperator =
+ new FillOperator(
+ driverContext.getOperatorContexts().get(0),
+ fillArray,
+ new Operator() {
+ private int index = 0;
+
+ @Override
+ public OperatorContext getOperatorContext() {
+ return driverContext.getOperatorContexts().get(0);
+ }
+
+ @Override
+ public TsBlock next() {
+ int delta = index * 10000;
+ TsBlockBuilder builder =
+ new TsBlockBuilder(
+ ImmutableList.of(TSDataType.INT32, TSDataType.INT32, TSDataType.INT32));
+ // 1 1, null, 100
+ builder.getTimeColumnBuilder().writeLong(1 + delta);
+ builder.getColumnBuilder(0).writeInt(1 + delta);
+ builder.getColumnBuilder(1).appendNull();
+ builder.getColumnBuilder(2).writeInt(100 + delta);
+ builder.declarePosition();
+ // 2 2, 20, 200
+ builder.getTimeColumnBuilder().writeLong(2 + delta);
+ builder.getColumnBuilder(0).writeInt(2 + delta);
+ builder.getColumnBuilder(1).writeInt(20 + delta);
+ builder.getColumnBuilder(2).writeInt(200 + delta);
+ builder.declarePosition();
+ // 3 3, 30, null
+ builder.getTimeColumnBuilder().writeLong(3 + delta);
+ builder.getColumnBuilder(0).writeInt(3 + delta);
+ builder.getColumnBuilder(1).writeInt(30 + delta);
+ builder.getColumnBuilder(2).appendNull();
+ builder.declarePosition();
+ // 4 null, 40, null
+ builder.getTimeColumnBuilder().writeLong(4 + delta);
+ builder.getColumnBuilder(0).appendNull();
+ builder.getColumnBuilder(1).writeInt(40 + delta);
+ builder.getColumnBuilder(2).appendNull();
+ builder.declarePosition();
+ // 5 null, null, 500
+ builder.getTimeColumnBuilder().writeLong(5 + delta);
+ builder.getColumnBuilder(0).appendNull();
+ builder.getColumnBuilder(1).appendNull();
+ builder.getColumnBuilder(2).writeInt(500 + delta);
+ builder.declarePosition();
+
+ index++;
+ return builder.build();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return index < 3;
+ }
+
+ @Override
+ public boolean isFinished() {
+ return index >= 3;
+ }
+
+ @Override
+ public long calculateMaxPeekMemory() {
+ return 0;
+ }
+
+ @Override
+ public long calculateMaxReturnSize() {
+ return 0;
+ }
+
+ @Override
+ public long calculateRetainedSizeAfterCallingNext() {
+ return 0;
+ }
+ });
+
+ int count = 0;
+ int[][][] res =
+ new int[][][] {
+ {{1, 0, 100}, {2, 20, 200}, {3, 30, 200}, {3, 40, 0}, {3, 40, 500}},
+ {
+ {10001, 0, 10100},
+ {10002, 10020, 10200},
+ {10003, 10030, 10200},
+ {10003, 10040, 0},
+ {10003, 10040, 10500}
+ },
+ {
+ {20001, 0, 20100},
+ {20002, 20020, 20200},
+ {20003, 20030, 20200},
+ {20003, 20040, 0},
+ {20003, 20040, 20500}
+ }
+ };
+ boolean[][][] isNull =
+ new boolean[][][] {
+ {
+ {false, true, false},
+ {false, false, false},
+ {false, false, false},
+ {false, false, true},
+ {false, false, false}
+ },
+ {
+ {false, true, false},
+ {false, false, false},
+ {false, false, false},
+ {false, false, true},
+ {false, false, false}
+ },
+ {
+ {false, true, false},
+ {false, false, false},
+ {false, false, false},
+ {false, false, true},
+ {false, false, false}
+ }
+ };
+ while (fillOperator.hasNext()) {
+ TsBlock block = fillOperator.next();
+ for (int i = 0; i < block.getPositionCount(); i++) {
+ long expectedTime = i + 1 + count * 10000L;
+ assertEquals(expectedTime, block.getTimeByIndex(i));
+ for (int j = 0; j < 3; j++) {
+ assertEquals(isNull[count][i][j], block.getColumn(j).isNull(i));
+ if (!isNull[count][i][j]) {
+ assertEquals(res[count][i][j], block.getColumn(j).getInt(i));
+ }
+ }
+ }
+ count++;
+ }
+
+ assertTrue(fillOperator.isFinished());
+ assertEquals(3, count);
+
+ } finally {
+ instanceNotificationExecutor.shutdown();
+ }
+ }
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilterTest.java
new file mode 100644
index 00000000000..f5dbc378123
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilterTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter;
+
+import org.junit.Test;
+
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class MonthIntervalMSFillFilterTest {
+
+ @Test
+ public void testMonthIntervalMSFillFilter() {
+ MonthIntervalMSFillFilter fillFilter =
+ new MonthIntervalMSFillFilter(1, 0, ZoneId.systemDefault());
+ DateTimeFormatter formatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME;
+ LocalDateTime localDateTime = LocalDateTime.parse("2023-02-01T11:47:30", formatter);
+
+ ZoneOffset zoneOffset = ZoneId.systemDefault().getRules().getOffset(localDateTime);
+
+ Instant instant = localDateTime.toInstant(zoneOffset);
+
+ long previousTime = instant.toEpochMilli();
+
+ assertTrue(
+ fillFilter.needFill(
+ localDateTime.plusMonths(1).toInstant(zoneOffset).toEpochMilli(), previousTime));
+ assertFalse(
+ fillFilter.needFill(
+ localDateTime.plusMonths(1).toInstant(zoneOffset).toEpochMilli() + 1, previousTime));
+ }
+}