You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yo...@apache.org on 2023/06/29 07:43:01 UTC
[iotdb] branch TOP-K-DTW updated: Finish
This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a commit to branch TOP-K-DTW
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/TOP-K-DTW by this push:
new 1c94ff7b251 Finish
1c94ff7b251 is described below
commit 1c94ff7b251ee7c7f0cc4fbbf10101a9238733c6
Author: YongzaoDan <53...@qq.com>
AuthorDate: Thu Jun 29 15:42:52 2023 +0800
Finish
---
.../UserGuide/Operators-Functions/Data-Matching.md | 58 +++++++
.../UserGuide/Operators-Functions/Data-Matching.md | 56 +++++++
.../main/java/org/apache/iotdb/DTWSessionPool.java | 168 ---------------------
.../apache/iotdb/db/it/udf/IoTDBUDTFTopKDTWIT.java | 2 -
4 files changed, 114 insertions(+), 170 deletions(-)
diff --git a/docs/UserGuide/Operators-Functions/Data-Matching.md b/docs/UserGuide/Operators-Functions/Data-Matching.md
index 2d7d1228a08..8d39f167729 100644
--- a/docs/UserGuide/Operators-Functions/Data-Matching.md
+++ b/docs/UserGuide/Operators-Functions/Data-Matching.md
@@ -332,4 +332,62 @@ Output series:
|1970-01-01T08:00:00.008+08:00| 11.8|
|1970-01-01T08:00:00.009+08:00| 6.0|
+-----------------------------+---------------------------------------+
+```
+
+### Top-K DTW
+
+#### Usage
+
+This function calculates the first K substring with the closest DTW distance between
+the target timeseries and the pattern timeseries by using the sliding window matching algorithm.
+
+**Name:** TOP_K_DTW_SLIDING_WINDOW
+
+**Input Parameters:**
++ First timeseries: target timeseries
++ Second timeseries: pattern timeseries
++ Parameter k: the number of substrings with the closest DTW distance to be calculated
++ (Optional) Parameter batchSize: The number of rows entered by the algorithm per batch. The default value is 65535
+
+**Output Series:** Output a single timeseries containing a total of K data points, each containing the following information:
++ startTime:The start timestamp of the substring in the target timeseries
++ endTime:The end timestamp of the substring in the target timeseries
++ distance:The DTW distance between the substring and the pattern timeseries
+
+#### Examples
+
+First ensure that the IoTDB has written the target timeseries
+and the pattern timeseries, and then use the following SQL statement to query:
+
+```sql
+select top_k_dtw_sliding_window(s, p, 'k'='20') from root.database.device;
+```
+
+Output series:
+
+```
++-----------------------------+----------------------------------------------------------------------------------+
+| Time|top_k_dtw_sliding_window(root.database.device.s, root.database.device.p, "k"="20")|
++-----------------------------+----------------------------------------------------------------------------------+
+|2000-03-31T14:46:40.000+08:00| DTWPath{startTime=954485200000, endTime=954485224000, distance=0.0}|
+|2000-03-31T17:33:45.000+08:00| DTWPath{startTime=954495225000, endTime=954495250000, distance=0.0}|
+|2000-03-31T20:20:51.000+08:00| DTWPath{startTime=954505251000, endTime=954505278000, distance=0.0}|
+|2000-03-31T23:07:59.000+08:00| DTWPath{startTime=954515279000, endTime=954515301000, distance=0.0}|
+|2000-04-01T01:55:02.000+08:00| DTWPath{startTime=954525302000, endTime=954525325000, distance=0.0}|
+|2000-04-01T04:42:06.000+08:00| DTWPath{startTime=954535326000, endTime=954535350000, distance=0.0}|
+|2000-04-01T07:29:11.000+08:00| DTWPath{startTime=954545351000, endTime=954545376000, distance=0.0}|
+|2000-04-01T10:16:17.000+08:00| DTWPath{startTime=954555377000, endTime=954555403000, distance=0.0}|
+|2000-04-01T13:03:24.000+08:00| DTWPath{startTime=954565404000, endTime=954565432000, distance=0.0}|
+|2000-04-01T15:50:33.000+08:00| DTWPath{startTime=954575433000, endTime=954575461000, distance=0.0}|
+|2000-04-01T18:37:42.000+08:00| DTWPath{startTime=954585462000, endTime=954585488000, distance=0.0}|
+|2000-04-01T21:24:49.000+08:00| DTWPath{startTime=954595489000, endTime=954595516000, distance=0.0}|
+|2000-04-02T00:11:57.000+08:00| DTWPath{startTime=954605517000, endTime=954605541000, distance=0.0}|
+|2000-04-02T02:59:02.000+08:00| DTWPath{startTime=954615542000, endTime=954615570000, distance=0.0}|
+|2000-04-02T05:46:11.000+08:00| DTWPath{startTime=954625571000, endTime=954625597000, distance=0.0}|
+|2000-04-02T08:33:18.000+08:00| DTWPath{startTime=954635598000, endTime=954635623000, distance=0.0}|
+|2000-04-02T11:20:24.000+08:00| DTWPath{startTime=954645624000, endTime=954645657000, distance=0.0}|
+|2000-04-02T14:07:38.000+08:00| DTWPath{startTime=954655658000, endTime=954655682000, distance=0.0}|
+|2000-04-02T16:54:43.000+08:00| DTWPath{startTime=954665683000, endTime=954665708000, distance=0.0}|
+|2000-04-02T19:41:49.000+08:00| DTWPath{startTime=954675709000, endTime=954675736000, distance=0.0}|
++-----------------------------+----------------------------------------------------------------------------------+
```
\ No newline at end of file
diff --git a/docs/zh/UserGuide/Operators-Functions/Data-Matching.md b/docs/zh/UserGuide/Operators-Functions/Data-Matching.md
index 99a8e196ce0..bc5951d6c9e 100644
--- a/docs/zh/UserGuide/Operators-Functions/Data-Matching.md
+++ b/docs/zh/UserGuide/Operators-Functions/Data-Matching.md
@@ -330,4 +330,60 @@ select xcorr(s1, s2) from root.test.d1 where time <= 2020-01-01 00:00:05
|1970-01-01T08:00:00.008+08:00| 11.8|
|1970-01-01T08:00:00.009+08:00| 6.0|
+-----------------------------+---------------------------------------+
+```
+
+### Top-K DTW
+
+#### 函数简介
+
+本函数通过滑动窗口匹配算法,计算在目标时间序列中与模式时间序列的 DTW 距离最接近的 前 K 个子串。
+
+**函数名:** TOP_K_DTW_SLIDING_WINDOW
+
+**输入参数:**
++ 第一个时间序列:目标时间序列
++ 第二个时间序列:模式时间序列
++ 参数 k:返回的子串个数
++ (可选)参数 batchSize:算法每批次输入的数据行数,默认为 65535
+
+**输出序列:** 输出单个序列,序列中共包含K个数据点,每个数据点包含如下信息:
++ startTime:子串在目标时间序列中的起始时间戳
++ endTime:子串在目标时间序列中的结束时间戳
++ distance:子串与模式时间序列的 DTW 距离
+
+#### 使用示例
+
+首先保证 IoTDB 中已写入目标时间序列和模式时间序列,随后可用下面的 SQL 语句查询:
+
+```sql
+select top_k_dtw_sliding_window(s, p, 'k'='20') from root.database.device;
+```
+
+输出序列:
+
+```
++-----------------------------+----------------------------------------------------------------------------------+
+| Time|top_k_dtw_sliding_window(root.database.device.s, root.database.device.p, "k"="20")|
++-----------------------------+----------------------------------------------------------------------------------+
+|2000-03-31T14:46:40.000+08:00| DTWPath{startTime=954485200000, endTime=954485224000, distance=0.0}|
+|2000-03-31T17:33:45.000+08:00| DTWPath{startTime=954495225000, endTime=954495250000, distance=0.0}|
+|2000-03-31T20:20:51.000+08:00| DTWPath{startTime=954505251000, endTime=954505278000, distance=0.0}|
+|2000-03-31T23:07:59.000+08:00| DTWPath{startTime=954515279000, endTime=954515301000, distance=0.0}|
+|2000-04-01T01:55:02.000+08:00| DTWPath{startTime=954525302000, endTime=954525325000, distance=0.0}|
+|2000-04-01T04:42:06.000+08:00| DTWPath{startTime=954535326000, endTime=954535350000, distance=0.0}|
+|2000-04-01T07:29:11.000+08:00| DTWPath{startTime=954545351000, endTime=954545376000, distance=0.0}|
+|2000-04-01T10:16:17.000+08:00| DTWPath{startTime=954555377000, endTime=954555403000, distance=0.0}|
+|2000-04-01T13:03:24.000+08:00| DTWPath{startTime=954565404000, endTime=954565432000, distance=0.0}|
+|2000-04-01T15:50:33.000+08:00| DTWPath{startTime=954575433000, endTime=954575461000, distance=0.0}|
+|2000-04-01T18:37:42.000+08:00| DTWPath{startTime=954585462000, endTime=954585488000, distance=0.0}|
+|2000-04-01T21:24:49.000+08:00| DTWPath{startTime=954595489000, endTime=954595516000, distance=0.0}|
+|2000-04-02T00:11:57.000+08:00| DTWPath{startTime=954605517000, endTime=954605541000, distance=0.0}|
+|2000-04-02T02:59:02.000+08:00| DTWPath{startTime=954615542000, endTime=954615570000, distance=0.0}|
+|2000-04-02T05:46:11.000+08:00| DTWPath{startTime=954625571000, endTime=954625597000, distance=0.0}|
+|2000-04-02T08:33:18.000+08:00| DTWPath{startTime=954635598000, endTime=954635623000, distance=0.0}|
+|2000-04-02T11:20:24.000+08:00| DTWPath{startTime=954645624000, endTime=954645657000, distance=0.0}|
+|2000-04-02T14:07:38.000+08:00| DTWPath{startTime=954655658000, endTime=954655682000, distance=0.0}|
+|2000-04-02T16:54:43.000+08:00| DTWPath{startTime=954665683000, endTime=954665708000, distance=0.0}|
+|2000-04-02T19:41:49.000+08:00| DTWPath{startTime=954675709000, endTime=954675736000, distance=0.0}|
++-----------------------------+----------------------------------------------------------------------------------+
```
\ No newline at end of file
diff --git a/example/session/src/main/java/org/apache/iotdb/DTWSessionPool.java b/example/session/src/main/java/org/apache/iotdb/DTWSessionPool.java
deleted file mode 100644
index 0ece8e54159..00000000000
--- a/example/session/src/main/java/org/apache/iotdb/DTWSessionPool.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb;
-
-import org.apache.iotdb.isession.pool.SessionDataSetWrapper;
-import org.apache.iotdb.rpc.IoTDBConnectionException;
-import org.apache.iotdb.rpc.StatementExecutionException;
-import org.apache.iotdb.session.pool.SessionPool;
-import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.tsfile.read.common.RowRecord;
-import org.apache.iotdb.tsfile.write.record.Tablet;
-import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-
-public class DTWSessionPool {
-
- private static final String DATABASE = "root.database";
- private static final String S = "root.database.device.s";
- private static final String P = "root.database.device.p";
-
- private static final int BATCH_SIZE = 32768;
- private static final long TIMESTAMP_STRIDE = 1000L;
- private static final long DATA_START_TIME = 954475200000L;
-
- private static final int PATTERN_COUNT = 100;
- private static final int PATTERN_LENGTH = 20;
- private static final double PATTERN_RANGE = 100.0;
-
- private static final int SERIES_LENGTH = 10000;
- private static final double SERIES_RANGE = 1000.0;
-
- private static final String SQL =
- String.format(
- "select top_k_dtw_sliding_window(s, p, 'k'='%d') from root.database.device;",
- PATTERN_COUNT);
-
- private static SessionPool sessionPool;
-
- private static void constructSessionPool() {
- sessionPool =
- new SessionPool.Builder()
- .host("127.0.0.1")
- .port(6667)
- .user("root")
- .password("root")
- .maxSize(3)
- .build();
- }
-
- private static void registerSchema() {
- try {
- sessionPool.deleteDatabase(DATABASE);
- System.out.println("Delete database successfully");
- } catch (Exception e) {
- // ignore
- }
- try {
- sessionPool.createDatabase(DATABASE);
- sessionPool.createTimeseries(
- S, TSDataType.DOUBLE, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED);
- sessionPool.createTimeseries(
- P, TSDataType.DOUBLE, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED);
- System.out.println("Create timeseries successfully");
- } catch (Exception e) {
- // ignore
- }
- }
-
- private static void insertRecordByTablet()
- throws IoTDBConnectionException, StatementExecutionException {
- Random random = new Random();
- List<MeasurementSchema> sList =
- Collections.singletonList(new MeasurementSchema("s", TSDataType.DOUBLE));
- List<MeasurementSchema> pList =
- Collections.singletonList(new MeasurementSchema("p", TSDataType.DOUBLE));
-
- double[] pattern = new double[PATTERN_LENGTH];
- Tablet tablet = new Tablet("root.database.device", pList, BATCH_SIZE);
- for (int i = 0; i < PATTERN_LENGTH; i++) {
- // each point is a random double in [-100, 100]
- pattern[i] = PATTERN_RANGE * (2.0 * random.nextDouble() - 1.0);
- tablet.addTimestamp(i, i * TIMESTAMP_STRIDE);
- tablet.addValue("p", i, pattern[i]);
- }
- tablet.rowSize = PATTERN_LENGTH;
- sessionPool.insertTablet(tablet);
- System.out.println("Insert pattern successfully");
-
- long currentTime = DATA_START_TIME;
- for (int i = 0; i < PATTERN_COUNT; i++) {
- int rowCount = 0;
- tablet = new Tablet("root.database.device", sList, BATCH_SIZE);
- // each point is a random double in [-1000, 1000]
- for (int j = 0; j < SERIES_LENGTH; j++) {
- tablet.addTimestamp(rowCount, currentTime);
- tablet.addValue("s", rowCount, SERIES_RANGE * (2.0 * random.nextDouble() - 1.0));
- currentTime += TIMESTAMP_STRIDE;
- rowCount++;
- }
-
- for (int j = 0; j < PATTERN_LENGTH; j++) {
- // Construct a similar pattern
- int repeatTime = (int) Math.abs(random.nextGaussian()) + 1;
- if (j == 0 || j == PATTERN_LENGTH - 1) {
- // Make sure the first and last points are the same
- repeatTime = 1;
- }
- for (int k = 0; k < repeatTime; k++) {
- tablet.addTimestamp(rowCount, currentTime);
- tablet.addValue("s", rowCount, pattern[j]);
- currentTime += TIMESTAMP_STRIDE;
- rowCount++;
- }
- }
-
- tablet.rowSize = rowCount;
- sessionPool.insertTablet(tablet);
- }
- System.out.println("Insert series successfully");
- }
-
- private static void executeQuery() throws IoTDBConnectionException, StatementExecutionException {
- long startTime = System.currentTimeMillis();
- try (SessionDataSetWrapper wrapper = sessionPool.executeQueryStatement(SQL, 10 * 60 * 1000)) {
- while (wrapper.hasNext()) {
- RowRecord record = wrapper.next();
- System.out.println(record);
- }
- }
- long endTime = System.currentTimeMillis();
-
- System.out.printf(
- "Query series length: %d pattern length: %d cost: %fs%n",
- SERIES_LENGTH * PATTERN_COUNT, PATTERN_LENGTH, (double) (endTime - startTime) / 1000.0);
- }
-
- public static void main(String[] args)
- throws StatementExecutionException, IoTDBConnectionException {
-
- constructSessionPool();
- registerSchema();
- insertRecordByTablet();
- executeQuery();
- sessionPool.close();
- }
-}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFTopKDTWIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFTopKDTWIT.java
index 902f0b01c35..2f7e82ea83b 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFTopKDTWIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFTopKDTWIT.java
@@ -163,8 +163,6 @@ public class IoTDBUDTFTopKDTWIT {
}
}
- // TODO: QGRAM Test
-
private void checkResultSet(ResultSet resultSet) throws SQLException {
List<Long> startTimes = new ArrayList<>();
List<Long> endTimes = new ArrayList<>();