You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2023/11/13 04:42:22 UTC

(iotdb) branch IOTDB-6243 updated: Add UT

This is an automated email from the ASF dual-hosted git repository.

jackietien pushed a commit to branch IOTDB-6243
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/IOTDB-6243 by this push:
     new e457adacdc2 Add UT
e457adacdc2 is described below

commit e457adacdc23eb2a8b0f00e07a931124c0a34529
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Mon Nov 13 12:42:12 2023 +0800

    Add UT
---
 .../filter/AbstractMonthIntervalFillFilter.java    |  26 ++--
 .../fill/filter/MonthIntervalMSFillFilter.java     |  14 +-
 .../fill/filter/MonthIntervalNSFillFilter.java     |  14 +-
 .../fill/filter/MonthIntervalUSFillFilter.java     |  20 ++-
 .../execution/operator/FillOperatorTest.java       | 168 +++++++++++++++++++++
 .../fill/filter/MonthIntervalMSFillFilterTest.java |  55 +++++++
 6 files changed, 269 insertions(+), 28 deletions(-)

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/AbstractMonthIntervalFillFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/AbstractMonthIntervalFillFilter.java
index 78b48d2fd89..99d5792eb26 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/AbstractMonthIntervalFillFilter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/AbstractMonthIntervalFillFilter.java
@@ -22,33 +22,25 @@ package org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter;
 import org.apache.iotdb.db.queryengine.execution.operator.process.fill.IFillFilter;
 
 import java.time.Instant;
-import java.time.LocalDateTime;
 import java.time.ZoneId;
+import java.time.ZoneOffset;
 
 public abstract class AbstractMonthIntervalFillFilter implements IFillFilter {
 
   // month part of time duration
-  public final int monthDuration;
+  protected final int monthDuration;
+
+  protected final ZoneId zone;
+
   // non-month part of time duration, its precision is same as current time_precision
-  public final long nonMonthDuration;
+  protected final long nonMonthDuration;
 
-  public final ZoneId zone;
+  protected final ZoneOffset zoneOffset;
 
-  public AbstractMonthIntervalFillFilter(int monthDuration, long nonMonthDuration, ZoneId zone) {
+  AbstractMonthIntervalFillFilter(int monthDuration, long nonMonthDuration, ZoneId zone) {
     this.monthDuration = monthDuration;
     this.nonMonthDuration = nonMonthDuration;
     this.zone = zone;
+    this.zoneOffset = zone.getRules().getOffset(Instant.now());
   }
-
-  @Override
-  public boolean needFill(long time, long previousTime) {
-
-    long smaller = Math.min(time, previousTime);
-    long greater = Math.max(time, previousTime);
-    Instant instant = Instant.ofEpochSecond(smaller, smaller);
-    LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, zone);
-    return needFill(localDateTime, greater);
-  }
-
-  abstract boolean needFill(LocalDateTime smaller, long greater);
 }
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilter.java
index 4885152d1c3..ff601056914 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilter.java
@@ -19,6 +19,7 @@
 
 package org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter;
 
+import java.time.Instant;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 
@@ -29,9 +30,14 @@ public class MonthIntervalMSFillFilter extends AbstractMonthIntervalFillFilter {
   }
 
   @Override
-  boolean needFill(LocalDateTime smaller, long greater) {
-    LocalDateTime localDateTime =
-        smaller.plusMonths(monthDuration).plusNanos(nonMonthDuration * 1_000_000L);
-    return greater >= (localDateTime.getSecond() * 1_000L + localDateTime.getNano() / 1_000_000);
+  public boolean needFill(long time, long previousTime) {
+    long smaller = Math.min(time, previousTime);
+    long greater = Math.max(time, previousTime);
+    Instant instant = Instant.ofEpochMilli(smaller);
+    LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, zone);
+    long epochMilli =
+        localDateTime.plusMonths(monthDuration).toInstant(zoneOffset).toEpochMilli()
+            + nonMonthDuration;
+    return epochMilli >= greater;
   }
 }
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalNSFillFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalNSFillFilter.java
index ec376051dae..5fad98dcaa8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalNSFillFilter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalNSFillFilter.java
@@ -19,8 +19,10 @@
 
 package org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter;
 
+import java.time.Instant;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
+import java.time.temporal.ChronoField;
 
 public class MonthIntervalNSFillFilter extends AbstractMonthIntervalFillFilter {
 
@@ -29,8 +31,14 @@ public class MonthIntervalNSFillFilter extends AbstractMonthIntervalFillFilter {
   }
 
   @Override
-  boolean needFill(LocalDateTime smaller, long greater) {
-    LocalDateTime localDateTime = smaller.plusMonths(monthDuration).plusNanos(nonMonthDuration);
-    return greater >= (localDateTime.getSecond() * 1_000_000_000L + localDateTime.getNano());
+  public boolean needFill(long time, long previousTime) {
+    long smaller = Math.min(time, previousTime);
+    long greater = Math.max(time, previousTime);
+    Instant instant = Instant.ofEpochSecond(smaller / 1_000_000_000L, smaller % 1_000_000_000L);
+    LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, zone);
+    Instant upper =
+        localDateTime.plusMonths(monthDuration).plusNanos(nonMonthDuration).toInstant(zoneOffset);
+    long timeInNs = upper.getLong(ChronoField.NANO_OF_SECOND) + upper.getEpochSecond();
+    return timeInNs >= greater;
   }
 }
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalUSFillFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalUSFillFilter.java
index 68afcda4ead..d5f0647d0b4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalUSFillFilter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalUSFillFilter.java
@@ -19,8 +19,12 @@
 
 package org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter;
 
+import java.time.Instant;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
+import java.time.temporal.ChronoField;
+
+import static java.time.temporal.ChronoUnit.MICROS;
 
 public class MonthIntervalUSFillFilter extends AbstractMonthIntervalFillFilter {
 
@@ -29,9 +33,17 @@ public class MonthIntervalUSFillFilter extends AbstractMonthIntervalFillFilter {
   }
 
   @Override
-  boolean needFill(LocalDateTime smaller, long greater) {
-    LocalDateTime localDateTime =
-        smaller.plusMonths(monthDuration).plusNanos(nonMonthDuration * 1_000L);
-    return greater >= (localDateTime.getSecond() * 1_000_000L + localDateTime.getNano() / 1_000);
+  public boolean needFill(long time, long previousTime) {
+    long smaller = Math.min(time, previousTime);
+    long greater = Math.max(time, previousTime);
+    Instant instant = Instant.ofEpochSecond(smaller / 1_000_000_000L, smaller % 1_000_000_000L);
+    LocalDateTime localDateTime = LocalDateTime.ofInstant(instant, zone);
+    Instant upper =
+        localDateTime
+            .plusMonths(monthDuration)
+            .plus(nonMonthDuration, MICROS)
+            .toInstant(zoneOffset);
+    long timeInUs = upper.getLong(ChronoField.MICRO_OF_SECOND) + instant.getEpochSecond();
+    return timeInUs >= greater;
   }
 }
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/FillOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/FillOperatorTest.java
index 96a528eecda..94eaba5dd37 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/FillOperatorTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/FillOperatorTest.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.process.FillOperator;
 import org.apache.iotdb.db.queryengine.execution.operator.process.fill.IFill;
 import org.apache.iotdb.db.queryengine.execution.operator.process.fill.IFillFilter;
 import org.apache.iotdb.db.queryengine.execution.operator.process.fill.constant.DoubleConstantFill;
+import org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter.FixedIntervalFillFilter;
 import org.apache.iotdb.db.queryengine.execution.operator.process.fill.previous.IntPreviousFill;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
 import org.apache.iotdb.tsfile.enums.TSDataType;
@@ -386,4 +387,171 @@ public class FillOperatorTest {
       instanceNotificationExecutor.shutdown();
     }
   }
+
+  @Test
+  public void batchPreviousFillWithThresholdTest() throws Exception {
+    ExecutorService instanceNotificationExecutor =
+        IoTDBThreadPoolFactory.newFixedThreadPool(1, "test-instance-notification");
+    try {
+      QueryId queryId = new QueryId("stub_query");
+      FragmentInstanceId instanceId =
+          new FragmentInstanceId(new PlanFragmentId(queryId, 0), "stub-instance");
+      FragmentInstanceStateMachine stateMachine =
+          new FragmentInstanceStateMachine(instanceId, instanceNotificationExecutor);
+      FragmentInstanceContext fragmentInstanceContext =
+          createFragmentInstanceContext(instanceId, stateMachine);
+      DriverContext driverContext = new DriverContext(fragmentInstanceContext, 0);
+      PlanNodeId planNodeId1 = new PlanNodeId("1");
+      driverContext.addOperatorContext(1, planNodeId1, FillOperator.class.getSimpleName());
+
+      IFill[] fillArray =
+          new IFill[] {
+            new IntPreviousFill(new FixedIntervalFillFilter(2)),
+            new IntPreviousFill(new FixedIntervalFillFilter(100)),
+            new IntPreviousFill(new FixedIntervalFillFilter(1))
+          };
+      FillOperator fillOperator =
+          new FillOperator(
+              driverContext.getOperatorContexts().get(0),
+              fillArray,
+              new Operator() {
+                private int index = 0;
+
+                @Override
+                public OperatorContext getOperatorContext() {
+                  return driverContext.getOperatorContexts().get(0);
+                }
+
+                @Override
+                public TsBlock next() {
+                  int delta = index * 10000;
+                  TsBlockBuilder builder =
+                      new TsBlockBuilder(
+                          ImmutableList.of(TSDataType.INT32, TSDataType.INT32, TSDataType.INT32));
+                  // 1  1, null, 100
+                  builder.getTimeColumnBuilder().writeLong(1 + delta);
+                  builder.getColumnBuilder(0).writeInt(1 + delta);
+                  builder.getColumnBuilder(1).appendNull();
+                  builder.getColumnBuilder(2).writeInt(100 + delta);
+                  builder.declarePosition();
+                  // 2  2, 20, 200
+                  builder.getTimeColumnBuilder().writeLong(2 + delta);
+                  builder.getColumnBuilder(0).writeInt(2 + delta);
+                  builder.getColumnBuilder(1).writeInt(20 + delta);
+                  builder.getColumnBuilder(2).writeInt(200 + delta);
+                  builder.declarePosition();
+                  // 3  3, 30, null
+                  builder.getTimeColumnBuilder().writeLong(3 + delta);
+                  builder.getColumnBuilder(0).writeInt(3 + delta);
+                  builder.getColumnBuilder(1).writeInt(30 + delta);
+                  builder.getColumnBuilder(2).appendNull();
+                  builder.declarePosition();
+                  // 4  null, 40, null
+                  builder.getTimeColumnBuilder().writeLong(4 + delta);
+                  builder.getColumnBuilder(0).appendNull();
+                  builder.getColumnBuilder(1).writeInt(40 + delta);
+                  builder.getColumnBuilder(2).appendNull();
+                  builder.declarePosition();
+                  // 5  null, null, 500
+                  builder.getTimeColumnBuilder().writeLong(5 + delta);
+                  builder.getColumnBuilder(0).appendNull();
+                  builder.getColumnBuilder(1).appendNull();
+                  builder.getColumnBuilder(2).writeInt(500 + delta);
+                  builder.declarePosition();
+
+                  index++;
+                  return builder.build();
+                }
+
+                @Override
+                public boolean hasNext() {
+                  return index < 3;
+                }
+
+                @Override
+                public boolean isFinished() {
+                  return index >= 3;
+                }
+
+                @Override
+                public long calculateMaxPeekMemory() {
+                  return 0;
+                }
+
+                @Override
+                public long calculateMaxReturnSize() {
+                  return 0;
+                }
+
+                @Override
+                public long calculateRetainedSizeAfterCallingNext() {
+                  return 0;
+                }
+              });
+
+      int count = 0;
+      int[][][] res =
+          new int[][][] {
+            {{1, 0, 100}, {2, 20, 200}, {3, 30, 200}, {3, 40, 0}, {3, 40, 500}},
+            {
+              {10001, 0, 10100},
+              {10002, 10020, 10200},
+              {10003, 10030, 10200},
+              {10003, 10040, 0},
+              {10003, 10040, 10500}
+            },
+            {
+              {20001, 0, 20100},
+              {20002, 20020, 20200},
+              {20003, 20030, 20200},
+              {20003, 20040, 0},
+              {20003, 20040, 20500}
+            }
+          };
+      boolean[][][] isNull =
+          new boolean[][][] {
+            {
+              {false, true, false},
+              {false, false, false},
+              {false, false, false},
+              {false, false, true},
+              {false, false, false}
+            },
+            {
+              {false, true, false},
+              {false, false, false},
+              {false, false, false},
+              {false, false, true},
+              {false, false, false}
+            },
+            {
+              {false, true, false},
+              {false, false, false},
+              {false, false, false},
+              {false, false, true},
+              {false, false, false}
+            }
+          };
+      while (fillOperator.hasNext()) {
+        TsBlock block = fillOperator.next();
+        for (int i = 0; i < block.getPositionCount(); i++) {
+          long expectedTime = i + 1 + count * 10000L;
+          assertEquals(expectedTime, block.getTimeByIndex(i));
+          for (int j = 0; j < 3; j++) {
+            assertEquals(isNull[count][i][j], block.getColumn(j).isNull(i));
+            if (!isNull[count][i][j]) {
+              assertEquals(res[count][i][j], block.getColumn(j).getInt(i));
+            }
+          }
+        }
+        count++;
+      }
+
+      assertTrue(fillOperator.isFinished());
+      assertEquals(3, count);
+
+    } finally {
+      instanceNotificationExecutor.shutdown();
+    }
+  }
 }
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilterTest.java
new file mode 100644
index 00000000000..f5dbc378123
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/fill/filter/MonthIntervalMSFillFilterTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.execution.operator.process.fill.filter;
+
+import org.junit.Test;
+
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class MonthIntervalMSFillFilterTest {
+
+  @Test
+  public void testMonthIntervalMSFillFilter() {
+    MonthIntervalMSFillFilter fillFilter =
+        new MonthIntervalMSFillFilter(1, 0, ZoneId.systemDefault());
+    DateTimeFormatter formatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME;
+    LocalDateTime localDateTime = LocalDateTime.parse("2023-02-01T11:47:30", formatter);
+
+    ZoneOffset zoneOffset = ZoneId.systemDefault().getRules().getOffset(localDateTime);
+
+    Instant instant = localDateTime.toInstant(zoneOffset);
+
+    long previousTime = instant.toEpochMilli();
+
+    assertTrue(
+        fillFilter.needFill(
+            localDateTime.plusMonths(1).toInstant(zoneOffset).toEpochMilli(), previousTime));
+    assertFalse(
+        fillFilter.needFill(
+            localDateTime.plusMonths(1).toInstant(zoneOffset).toEpochMilli() + 1, previousTime));
+  }
+}