This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new af8f6afaad [IOTDB-5683] Add special case process for aggregation function Mode
af8f6afaad is described below
commit af8f6afaad222640c69df887f7997f31f8a2aee9
Author: Weihao Li <60...@users.noreply.github.com>
AuthorDate: Wed Mar 29 16:49:58 2023 +0800
[IOTDB-5683] Add special case process for aggregation function Mode
---
docs/UserGuide/Operators-Functions/Aggregation.md | 30 ++++++-------
.../UserGuide/Operators-Functions/Aggregation.md | 30 ++++++-------
.../iotdb/db/it/aggregation/IoTDBModeIT.java | 24 ++++++++++-
.../src/main/codegen/templates/ModeAccumulator.ftl | 49 ++++++++++++++++------
4 files changed, 89 insertions(+), 44 deletions(-)
diff --git a/docs/UserGuide/Operators-Functions/Aggregation.md b/docs/UserGuide/Operators-Functions/Aggregation.md
index a6c0e07126..1a73578784 100644
--- a/docs/UserGuide/Operators-Functions/Aggregation.md
+++ b/docs/UserGuide/Operators-Functions/Aggregation.md
@@ -27,21 +27,21 @@ All aggregate functions except `COUNT()`, `COUNT_IF()` ignore null values and re
The aggregate functions supported by IoTDB are as follows:
-| Function Name | Function Description | Allowed Input Data Types | required attributes [...]
-|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------| ------------------------ |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- [...]
-| SUM | Summation. | INT32 INT64 FLOAT DOUBLE | No [...]
-| COUNT | Counts the number of data points. | All types | No [...]
-| AVG | Average. | INT32 INT64 FLOAT DOUBLE | No [...]
-| EXTREME | Finds the value with the largest absolute value. Returns a positive value if the maximum absolute value of positive and negative values is equal. | INT32 INT64 FLOAT DOUBLE | No [...]
-| MAX_VALUE | Find the maximum value. | INT32 INT64 FLOAT DOUBLE | No [...]
-| MIN_VALUE | Find the minimum value. | INT32 INT64 FLOAT DOUBLE | No [...]
-| FIRST_VALUE | Find the value with the smallest timestamp. | All data types | No [...]
-| LAST_VALUE | Find the value with the largest timestamp. | All data types | No [...]
-| MAX_TIME | Find the maximum timestamp. | All data Types | No [...]
-| MIN_TIME | Find the minimum timestamp. | All data Types | No [...]
-| COUNT_IF | Find the number of data points that continuously meet a given condition and the number of data points that meet the condition (represented by keep) meet the specified threshold. | BOOLEAN | `[keep >=/>/=/!=/</<=]threshold`：The specified threshold or threshold condition, it is equivalent to `keep >= threshold` if `threshold` is used alone, type of `threshold` is `INT64`<br/> `ignoreNull`：Optional, default value is `true`；If the value is `true`, null valu [...]
-| TIME_DURATION | Find the difference between the timestamp of the largest non-null value and the timestamp of the smallest non-null value in a column | All data Types | No [...]
-| MODE | Find the mode. Note: Having too many different values in the input series risks a memory exception. | All data Types | No [...]
+| Function Name | Function Description | Allowed Input Data Types | required attributes [...]
+|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ------------------------ |--------------------------------------------------------------------------------------------------------------------------------------------------------- [...]
+| SUM | Summation. | INT32 INT64 FLOAT DOUBLE | No [...]
+| COUNT | Counts the number of data points. | All types | No [...]
+| AVG | Average. | INT32 INT64 FLOAT DOUBLE | No [...]
+| EXTREME | Finds the value with the largest absolute value. Returns a positive value if the maximum absolute value of positive and negative values is equal. | INT32 INT64 FLOAT DOUBLE | No [...]
+| MAX_VALUE | Find the maximum value. | INT32 INT64 FLOAT DOUBLE | No [...]
+| MIN_VALUE | Find the minimum value. | INT32 INT64 FLOAT DOUBLE | No [...]
+| FIRST_VALUE | Find the value with the smallest timestamp. | All data types | No [...]
+| LAST_VALUE | Find the value with the largest timestamp. | All data types | No [...]
+| MAX_TIME | Find the maximum timestamp. | All data Types | No [...]
+| MIN_TIME | Find the minimum timestamp. | All data Types | No [...]
+| COUNT_IF | Find the number of data points that continuously meet a given condition and the number of data points that meet the condition (represented by keep) meet the specified threshold. | BOOLEAN | `[keep >=/>/=/!=/</<=]threshold`：The specified threshold or threshold condition, it is equivalent to `keep >= threshold` if `threshold` is used alone, t [...]
+| TIME_DURATION | Find the difference between the timestamp of the largest non-null value and the timestamp of the smallest non-null value in a column | All data Types | No [...]
+| MODE | Find the mode. Note: </br>1.Having too many different values in the input series risks a memory exception; </br>2.If all the elements have the same number of occurrences, that is no Mode, return the value with earliest time; </br>3.If there are many Modes, return the Mode with earliest time. | All data Types | No [...]
## COUNT
### example
diff --git a/docs/zh/UserGuide/Operators-Functions/Aggregation.md b/docs/zh/UserGuide/Operators-Functions/Aggregation.md
index 995aef81fa..87177d6145 100644
--- a/docs/zh/UserGuide/Operators-Functions/Aggregation.md
+++ b/docs/zh/UserGuide/Operators-Functions/Aggregation.md
@@ -27,21 +27,21 @@
IoTDB 支持的聚合函数如下：
-| 函数名 | 功能描述 | 允许的输入类型 | 必要的属性参数 | 输出类型 |
-|---------------|-----------------------------------------------|--------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------|
-| SUM | 求和。 | INT32 INT64 FLOAT DOUBLE | 无 | DOUBLE |
-| COUNT | 计算数据点数。 | 所有类型 | 无 | INT64 |
-| AVG | 求平均值。 | INT32 INT64 FLOAT DOUBLE | 无 | DOUBLE |
-| EXTREME | 求具有最大绝对值的值。如果正值和负值的最大绝对值相等，则返回正值。 | INT32 INT64 FLOAT DOUBLE | 无 | 与输入类型一致 |
-| MAX_VALUE | 求最大值。 | INT32 INT64 FLOAT DOUBLE | 无 | 与输入类型一致 |
-| MIN_VALUE | 求最小值。 | INT32 INT64 FLOAT DOUBLE | 无 | 与输入类型一致 |
-| FIRST_VALUE | 求时间戳最小的值。 | 所有类型 | 无 | 与输入类型一致 |
-| LAST_VALUE | 求时间戳最大的值。 | 所有类型 | 无 | 与输入类型一致 |
-| MAX_TIME | 求最大时间戳。 | 所有类型 | 无 | Timestamp |
-| MIN_TIME | 求最小时间戳。 | 所有类型 | 无 | Timestamp |
-| COUNT_IF | 求数据点连续满足某一给定条件，且满足条件的数据点个数（用keep表示）满足指定阈值的次数。 | BOOLEAN | `[keep >=/>/=/!=/</<=]threshold`：被指定的阈值或阈值条件，若只使用`threshold`则等价于`keep >= threshold`,`threshold`类型为`INT64`<br/> `ignoreNull`：可选，默认为`true`；为`true`表示忽略null值，即如果中间出现null值，直接忽略，不会打断连续性；为`false`表示不忽略null值，即如果中间出现null值，会打断连续性 | INT64 |
-| TIME_DURATION | 求某一列最大一个不为NULL的值所在时间戳与最小一个不为NULL的值所在时间戳的时间戳差 | 所有类型 | 无 | INT64 |
-| MODE | 求众数。注意：输入序列的不同值个数过多时会有内存异常风险。 | 所有类型 | 无 | INT64 |
+| 函数名 | 功能描述 | 允许的输入类型 | 必要的属性参数 | 输出类型 |
+|---------------|-------------------------------------------------------------------------------------------------------------|--------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------|
+| SUM | 求和。 | INT32 INT64 FLOAT DOUBLE | 无 | DOUBLE |
+| COUNT | 计算数据点数。 | 所有类型 | 无 | INT64 |
+| AVG | 求平均值。 | INT32 INT64 FLOAT DOUBLE | 无 | DOUBLE |
+| EXTREME | 求具有最大绝对值的值。如果正值和负值的最大绝对值相等，则返回正值。 | INT32 INT64 FLOAT DOUBLE | 无 | 与输入类型一致 |
+| MAX_VALUE | 求最大值。 | INT32 INT64 FLOAT DOUBLE | 无 | 与输入类型一致 |
+| MIN_VALUE | 求最小值。 | INT32 INT64 FLOAT DOUBLE | 无 | 与输入类型一致 |
+| FIRST_VALUE | 求时间戳最小的值。 | 所有类型 | 无 | 与输入类型一致 |
+| LAST_VALUE | 求时间戳最大的值。 | 所有类型 | 无 | 与输入类型一致 |
+| MAX_TIME | 求最大时间戳。 | 所有类型 | 无 | Timestamp |
+| MIN_TIME | 求最小时间戳。 | 所有类型 | 无 | Timestamp |
+| COUNT_IF | 求数据点连续满足某一给定条件，且满足条件的数据点个数（用keep表示）满足指定阈值的次数。 | BOOLEAN | `[keep >=/>/=/!=/</<=]threshold`：被指定的阈值或阈值条件，若只使用`threshold`则等价于`keep >= threshold`,`threshold`类型为`INT64`<br/> `ignoreNull`：可选，默认为`true`；为`true`表示忽略null值，即如果中间出现null值，直接忽略，不会打断连续性；为`false`表示不忽略null值，即如果中间出现null值，会打断连续性 | INT64 |
+| TIME_DURATION | 求某一列最大一个不为NULL的值所在时间戳与最小一个不为NULL的值所在时间戳的时间戳差 | 所有类型 | 无 | INT64 |
+| MODE | 求众数。注意：</br>1.输入序列的不同值个数过多时会有内存异常风险; </br>2.如果所有元素出现的频次相同，即没有众数，则返回对应时间戳最小的值; </br>3.如果有多个众数，则返回对应时间戳最小的众数。 | 所有类型 | 无 | INT64 |
### COUNT_IF
#### 语法
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBModeIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBModeIT.java
index 37e8883182..49bee98459 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBModeIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBModeIT.java
@@ -66,6 +66,13 @@ public class IoTDBModeIT {
"INSERT INTO root.db.d2(timestamp,s1,s2,s3,s4,s5,s6) values(10000000001, 2, 2, false, 2, 2, \"2\")",
"INSERT INTO root.db.d2(timestamp,s1,s2,s3,s4,s5,s6) values(10000000002, 2, 2, false, 2, 2, \"2\")",
"INSERT INTO root.db.d2(timestamp,s1,s2,s3,s4,s5,s6) values(10000000003, 2, 2, false, 2, 2, \"2\")",
+ "INSERT INTO root.test.d1(timestamp,s1) values(1, 5)",
+ "INSERT INTO root.test.d1(timestamp,s1) values(2, 5)",
+ "INSERT INTO root.test.d1(timestamp,s1) values(3, 2)",
+ "INSERT INTO root.test.d1(timestamp,s1) values(4, 2)",
+ "INSERT INTO root.test.d1(timestamp,s1) values(5, 8)",
+ "INSERT INTO root.test.d1(timestamp,s1) values(6, 8)",
+ "INSERT INTO root.test.d1(timestamp,s1) values(7, 1)",
"flush"
};
@@ -145,7 +152,7 @@ public class IoTDBModeIT {
};
String[] retArray = new String[] {"2,2,false,2.0,2.0,2,"};
resultSetEqualTest(
- "select mode(s1),mode(s2),mode(s3),mode(s4),mode(s5),mode(s6) from root.** group by level = 0",
+ "select mode(s1),mode(s2),mode(s3),mode(s4),mode(s5),mode(s6) from root.db.* group by level = 0",
expectedHeader,
retArray);
}
@@ -157,4 +164,19 @@ public class IoTDBModeIT {
TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()
+ ": MODE with slidingWindow is not supported now");
}
+
+ @Test
+ public void testNoMode() {
+ String[] expectedHeader = new String[] {mode("root.test.d1.s1")};
+ String[] retArray = new String[] {"5.0,"};
+ resultSetEqualTest(
+ "select mode(s1) from root.test.d1 where time <= 6", expectedHeader, retArray);
+ }
+
+ @Test
+ public void testManyModes() {
+ String[] expectedHeader = new String[] {mode("root.test.d1.s1")};
+ String[] retArray = new String[] {"5.0,"};
+ resultSetEqualTest("select mode(s1) from root.test.d1", expectedHeader, retArray);
+ }
}
diff --git a/server/src/main/codegen/templates/ModeAccumulator.ftl b/server/src/main/codegen/templates/ModeAccumulator.ftl
index 62175bf628..e81ae32d15 100644
--- a/server/src/main/codegen/templates/ModeAccumulator.ftl
+++ b/server/src/main/codegen/templates/ModeAccumulator.ftl
@@ -25,6 +25,8 @@
package org.apache.iotdb.db.mpp.aggregation;
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.collections4.comparators.ComparatorChain;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
@@ -32,6 +34,7 @@ import org.apache.iotdb.tsfile.read.common.block.column.Column;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.BitMap;
+import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import java.io.ByteArrayInputStream;
@@ -49,7 +52,8 @@ import static com.google.common.base.Preconditions.checkArgument;
*/
@SuppressWarnings("unused")
public class ${className} implements Accumulator {
- private final Map<${type.javaBoxName}, Long> countMap = new HashMap<>();
+ // pair left records count of element, pair right records min time of element
+ private final Map<${type.javaBoxName}, Pair<Long, Long>> countMap = new HashMap<>();
<#if type.dataType != "boolean">
private final int MAP_SIZE_THRESHOLD = IoTDBDescriptor.getInstance().getConfig().getModeMapSizeThreshold();
@@ -62,7 +66,11 @@ public class ${className} implements Accumulator {
continue;
}
if (!column[1].isNull(i)) {
- countMap.compute(column[1].get${type.dataType?cap_first}(i), (k, v) -> v == null ? 1 : v + 1);
+ final long time = column[0].getLong(i);
+ countMap.compute(
+ column[1].get${type.dataType?cap_first}(i),
+ (k, v) ->
+ v == null ? new Pair<>(1L, time) : new Pair<>(v.left + 1, Math.min(v.right, time)));
<#if type.dataType != "boolean">
if (countMap.size() > MAP_SIZE_THRESHOLD) {
@@ -96,7 +104,7 @@ public class ${className} implements Accumulator {
// Step of ModeAccumulator is STATIC,
// countMap only need to record one entry which key is finalResult
- countMap.put(finalResult.get${type.dataType?cap_first}(0), 0L);
+ countMap.put(finalResult.get${type.dataType?cap_first}(0), new Pair<>(0L, Long.MIN_VALUE));
}
@Override
@@ -110,7 +118,14 @@ public class ${className} implements Accumulator {
tsBlockBuilder.appendNull();
} else {
tsBlockBuilder.write${type.dataType?cap_first}(
- Collections.max(countMap.entrySet(), Map.Entry.comparingByValue()).getKey());
+ Collections.max(
+ countMap.entrySet(),
+ Map.Entry.comparingByValue(
+ new ComparatorChain<>(
+ ImmutableList.of(
+ (v1, v2) -> v1.left.compareTo(v2.left),
+ (v1, v2) -> v2.right.compareTo(v1.right)))))
+ .getKey());
}
}
@@ -138,9 +153,10 @@ public class ${className} implements Accumulator {
ByteArrayOutputStream stream = new ByteArrayOutputStream();
try {
ReadWriteIOUtils.write(countMap.size(), stream);
- for (Map.Entry<${type.javaBoxName}, Long> entry : countMap.entrySet()) {
+ for (Map.Entry<${type.javaBoxName}, Pair<Long, Long>> entry : countMap.entrySet()) {
ReadWriteIOUtils.write(entry.getKey(), stream);
- ReadWriteIOUtils.write(entry.getValue(), stream);
+ ReadWriteIOUtils.write(entry.getValue().left, stream);
+ ReadWriteIOUtils.write(entry.getValue().right, stream);
}
} catch (IOException e) {
// Totally memory operation. This case won't happen.
@@ -153,13 +169,20 @@ public class ${className} implements Accumulator {
try {
int size = ReadWriteIOUtils.readInt(stream);
for (int i = 0; i < size; i++) {
- countMap.compute(ReadWriteIOUtils.read${type.dataType?cap_first}(stream), (k, v) -> {
- try {
- return v == null ? ReadWriteIOUtils.readLong(stream) : v + ReadWriteIOUtils.readLong(stream);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- });
+ countMap.compute(
+ ReadWriteIOUtils.read${type.dataType?cap_first}(stream),
+ (k, v) -> {
+ try {
+ return v == null
+ ? new Pair<>(
+ ReadWriteIOUtils.readLong(stream), ReadWriteIOUtils.readLong(stream))
+ : new Pair<>(
+ v.left + ReadWriteIOUtils.readLong(stream),
+ Math.min(v.right, ReadWriteIOUtils.readLong(stream)));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
<#if type.dataType != "boolean">
if (countMap.size() > MAP_SIZE_THRESHOLD) {