You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ka...@apache.org on 2021/05/13 04:14:34 UTC
[iotdb] branch f_index_dev updated (13b6b88 -> 68a2944)
This is an automated email from the ASF dual-hosted git repository.
kangrong pushed a change to branch f_index_dev
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
from 13b6b88 add time out, show INFO log
new 629ae13 fix format
new 68a2944 fix elb bug
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../iotdb/db/index/algorithm/elb/ELBIndex.java | 18 ++-
.../algorithm/elb/feature/SequenceELBFeature.java | 2 +-
.../iotdb/db/index/algorithm/mmhh/MMHHIndex.java | 4 +-
.../iotdb/db/index/algorithm/rtree/RTree.java | 2 +-
.../iotdb/db/index/stats/IndexStatManager.java | 2 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 6 +-
.../org/apache/iotdb/db/index/IndexTestUtils.java | 9 ++
.../index/it/{ELBNoIndexIT.java => ELBWindIT.java} | 165 ++++++++++-----------
.../apache/iotdb/db/index/it/MMHHIndexReadIT.java | 1 -
.../apache/zeppelin/iotdb/IoTDBInterpreter.java | 8 +-
.../zeppelin/iotdb/IoTDBInterpreterTest.java | 6 +-
11 files changed, 111 insertions(+), 112 deletions(-)
copy server/src/test/java/org/apache/iotdb/db/index/it/{ELBNoIndexIT.java => ELBWindIT.java} (59%)
[iotdb] 01/02: fix format
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kangrong pushed a commit to branch f_index_dev
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 629ae1342e9d63a0ddf83aa2968fbc0649eaa12d
Author: kr11 <3095717866.com>
AuthorDate: Thu May 13 11:32:21 2021 +0800
fix format
---
.../iotdb/db/index/algorithm/elb/ELBIndex.java | 4 +-
.../iotdb/db/index/algorithm/mmhh/MMHHIndex.java | 4 +-
.../iotdb/db/index/algorithm/rtree/RTree.java | 2 +-
.../iotdb/db/index/stats/IndexStatManager.java | 2 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 6 +-
.../org/apache/iotdb/db/index/IndexTestUtils.java | 9 +
.../org/apache/iotdb/db/index/it/ELBWindIT.java | 303 +++++++++++++++++++++
.../apache/iotdb/db/index/it/MMHHIndexReadIT.java | 1 -
.../apache/zeppelin/iotdb/IoTDBInterpreter.java | 8 +-
.../zeppelin/iotdb/IoTDBInterpreterTest.java | 6 +-
10 files changed, 326 insertions(+), 19 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/index/algorithm/elb/ELBIndex.java b/server/src/main/java/org/apache/iotdb/db/index/algorithm/elb/ELBIndex.java
index 44c8b9c..552fcd8 100644
--- a/server/src/main/java/org/apache/iotdb/db/index/algorithm/elb/ELBIndex.java
+++ b/server/src/main/java/org/apache/iotdb/db/index/algorithm/elb/ELBIndex.java
@@ -314,7 +314,7 @@ public class ELBIndex extends IoTDBIndex {
TVList.append(tvList, p.tvList, p.offset, p.length);
PartialPath showPath = indexSeries.concatNode(String.valueOf(tvList.getMinTime()));
res.add(new DistSeries(0, tvList, showPath));
- if(IndexStatManager.alreadyTimeout()){
+ if (IndexStatManager.alreadyTimeout()) {
logger.warn("ELB query on {}: already timeout", indexSeries);
alreadyTimeout = true;
break;
@@ -330,7 +330,7 @@ public class ELBIndex extends IoTDBIndex {
}
featureExtractor.clearProcessedSrcData();
featureTotal += System.nanoTime() - featureStart;
- if(alreadyTimeout){
+ if (alreadyTimeout) {
break;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/index/algorithm/mmhh/MMHHIndex.java b/server/src/main/java/org/apache/iotdb/db/index/algorithm/mmhh/MMHHIndex.java
index eaeab75..101cf2a 100644
--- a/server/src/main/java/org/apache/iotdb/db/index/algorithm/mmhh/MMHHIndex.java
+++ b/server/src/main/java/org/apache/iotdb/db/index/algorithm/mmhh/MMHHIndex.java
@@ -367,7 +367,7 @@ public class MMHHIndex extends IoTDBIndex {
if (full) {
break;
}
- if(IndexStatManager.alreadyTimeout()){
+ if (IndexStatManager.alreadyTimeout()) {
logger.warn("MMHH query on {}: already timeout", indexSeries);
break;
}
@@ -404,7 +404,7 @@ public class MMHHIndex extends IoTDBIndex {
if (full) {
return true;
}
- if(IndexStatManager.alreadyTimeout()){
+ if (IndexStatManager.alreadyTimeout()) {
logger.warn("MMHH query on {}: already timeout, MMHHIndex:406", indexSeries);
return true;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/index/algorithm/rtree/RTree.java b/server/src/main/java/org/apache/iotdb/db/index/algorithm/rtree/RTree.java
index 1fc07e5..ecf4fc5 100644
--- a/server/src/main/java/org/apache/iotdb/db/index/algorithm/rtree/RTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/index/algorithm/rtree/RTree.java
@@ -741,7 +741,7 @@ public class RTree<T> {
patternFeatures);
bsfAnswer.dist =
minTopKDistOfNode(pairs.right, pairs.left, queryTs, topKPQ, calcRealDistFunc, topK);
- if(IndexStatManager.alreadyTimeout()){
+ if (IndexStatManager.alreadyTimeout()) {
logger.warn("RTree query already timeout, no more visiting leaf node, RTree:745");
break;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/index/stats/IndexStatManager.java b/server/src/main/java/org/apache/iotdb/db/index/stats/IndexStatManager.java
index 84e7f44..d26d21c 100644
--- a/server/src/main/java/org/apache/iotdb/db/index/stats/IndexStatManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/index/stats/IndexStatManager.java
@@ -31,7 +31,7 @@ public class IndexStatManager {
public static long totalQueryCost = 0;
public static long loadRawDataCost = 0;
public static long latestTimestamp = 0;
-
+
public static String provideReport() {
// total:%.3fms-feature:%.3fms-load:%.3fms
String ret =
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index b6f9b6c..3ace443 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -636,15 +636,13 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
// it's a hyper-parameter, we can add it into .properties.
// although we cannot trust that index always return result before time out, but index
// will early-stop the query
-// long indexQueryExtraTimeInMs = 3_000;
+ // long indexQueryExtraTimeInMs = 3_000;
long indexQueryExtraTimeInMs = 3_000;
long indexFinalTimeout = actualTimeout + indexQueryExtraTimeInMs;
queryTimeManager.registerQuery(queryId, startTime, statement, indexFinalTimeout);
- }
- else{
+ } else {
queryTimeManager.registerQuery(queryId, startTime, statement, timeout);
}
-
}
if (plan instanceof QueryPlan && config.isEnablePerformanceTracing()) {
TracingManager tracingManager = TracingManager.getInstance();
diff --git a/server/src/test/java/org/apache/iotdb/db/index/IndexTestUtils.java b/server/src/test/java/org/apache/iotdb/db/index/IndexTestUtils.java
index e9b235f..0f37b63 100644
--- a/server/src/test/java/org/apache/iotdb/db/index/IndexTestUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/index/IndexTestUtils.java
@@ -67,6 +67,15 @@ public class IndexTestUtils {
return array.toString();
}
+ public static String getStringFromList(List<Float> list, int start, int end) {
+ StringBuilder array = new StringBuilder();
+ for (int i = start; i < end; i++) {
+ array.append(list.get(i)).append(',');
+ }
+ array.deleteCharAt(array.length() - 1);
+ return array.toString();
+ }
+
public static String tvListToString(TVList tvList) {
StringBuilder sb = new StringBuilder();
sb.append("{");
diff --git a/server/src/test/java/org/apache/iotdb/db/index/it/ELBWindIT.java b/server/src/test/java/org/apache/iotdb/db/index/it/ELBWindIT.java
new file mode 100644
index 0000000..1387c42
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/index/it/ELBWindIT.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.index.it;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.index.IndexManager;
+import org.apache.iotdb.db.index.common.IndexUtils;
+import org.apache.iotdb.db.rescon.TVListAllocator;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.db.utils.datastructure.TVList;
+import org.apache.iotdb.jdbc.Config;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.Statement;
+
+import static org.apache.iotdb.db.index.IndexTestUtils.getArrayRange;
+import static org.apache.iotdb.db.index.common.IndexType.ELB_INDEX;
+import static org.junit.Assert.fail;
+
+public class ELBIndexReadIT {
+
+ private static final String insertPattern = "INSERT INTO %s(timestamp, %s) VALUES (%d, %.3f)";
+
+ private static final String storageGroupSub = "root.wind1";
+ private static final String storageGroupWhole = "root.wind2";
+
+ private static final String speed1 = "root.wind1.azq01.speed";
+ private static final String speed1Device = "root.wind1.azq01";
+ private static final String speed1Sensor = "speed";
+
+ private static final String directionDevicePattern = "root.wind2.%d";
+ private static final String directionPattern = "root.wind2.%d.direction";
+ private static final String directionSensor = "direction";
+
+ private static final String indexSub = speed1;
+ private static final String indexWhole = "root.wind2.*.direction";
+ private static final int wholeSize = 5;
+ private static final int wholeDim = 100;
+ private static final int subLength = 50;
+
+ @Before
+ public void setUp() throws Exception {
+ IoTDBDescriptor.getInstance().getConfig().setEnableIndex(true);
+ EnvironmentUtils.closeStatMonitor();
+ EnvironmentUtils.envSetUp();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ EnvironmentUtils.cleanEnv();
+ IoTDBDescriptor.getInstance().getConfig().setEnableIndex(false);
+ }
+
+ private static void insertSQL(boolean createTS) throws ClassNotFoundException {
+ for (int i = 0; i < 10000; i++) {
+ System.out.println(
+ String.format(insertPattern, speed1Device, speed1Sensor, i, (float) (i * 10 + 9)));
+ }
+ IndexUtils.breakDown("asd");
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement(); ) {
+ statement.execute(String.format("SET STORAGE GROUP TO %s", storageGroupSub));
+ statement.execute(String.format("SET STORAGE GROUP TO %s", storageGroupWhole));
+ System.out.println(String.format("SET STORAGE GROUP TO %s", storageGroupSub));
+ System.out.println(String.format("SET STORAGE GROUP TO %s", storageGroupWhole));
+ if (createTS) {
+ statement.execute(
+ String.format("CREATE TIMESERIES %s WITH DATATYPE=FLOAT,ENCODING=PLAIN", speed1));
+ System.out.println(
+ String.format("CREATE TIMESERIES %s WITH DATATYPE=FLOAT,ENCODING=PLAIN", speed1));
+ for (int i = 0; i < wholeSize; i++) {
+ String wholePath = String.format(directionPattern, i);
+ statement.execute(
+ String.format("CREATE TIMESERIES %s WITH DATATYPE=FLOAT,ENCODING=PLAIN", wholePath));
+ System.out.println(
+ String.format("CREATE TIMESERIES %s WITH DATATYPE=FLOAT,ENCODING=PLAIN", wholePath));
+ }
+ }
+ statement.execute(
+ String.format("CREATE INDEX ON %s WITH INDEX=%s, BLOCK_SIZE=%d", indexSub, ELB_INDEX, 5));
+ System.out.println(
+ String.format("CREATE INDEX ON %s WITH INDEX=%s, BLOCK_SIZE=%d", indexSub, ELB_INDEX, 5));
+
+ TVList subInput = TVListAllocator.getInstance().allocate(TSDataType.DOUBLE);
+ for (int i = 0; i < subLength; i++) {
+ subInput.putDouble(i, i * 10);
+ }
+ for (int i = 0; i < 20; i++) {
+ statement.execute(
+ String.format(
+ insertPattern,
+ speed1Device,
+ speed1Sensor,
+ subInput.getTime(i),
+ subInput.getDouble(i)));
+ System.out.println(
+ String.format(
+ insertPattern,
+ speed1Device,
+ speed1Sensor,
+ subInput.getTime(i),
+ subInput.getDouble(i)));
+ }
+ statement.execute("flush");
+ // System.out.println("==========================");
+ // System.out.println(IndexManager.getInstance().getRouter());
+
+ for (int i = 25; i < 30; i++) {
+ statement.execute(
+ String.format(
+ insertPattern,
+ speed1Device,
+ speed1Sensor,
+ subInput.getTime(i),
+ subInput.getDouble(i)));
+ System.out.println(
+ String.format(
+ insertPattern,
+ speed1Device,
+ speed1Sensor,
+ subInput.getTime(i),
+ subInput.getDouble(i)));
+ }
+ statement.execute("flush");
+
+ IndexManager.getInstance().stop();
+ IndexManager.getInstance().start();
+
+ for (int i = 40; i < subLength; i++) {
+ statement.execute(
+ String.format(
+ insertPattern,
+ speed1Device,
+ speed1Sensor,
+ subInput.getTime(i),
+ subInput.getDouble(i)));
+ System.out.println(
+ String.format(
+ insertPattern,
+ speed1Device,
+ speed1Sensor,
+ subInput.getTime(i),
+ subInput.getDouble(i)));
+ }
+ statement.execute("flush");
+ // System.out.println("==========================");
+ // System.out.println(IndexManager.getInstance().getRouter());
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void checkMultiSeriesTimeAlignedWithCreateTS() throws ClassNotFoundException {
+ checkMultiSeriesTimeAligned(true);
+ }
+
+ @Test
+ public void checkMultiSeriesTimeAlignedWithoutCreateTS() throws ClassNotFoundException {
+ checkMultiSeriesTimeAligned(false);
+ }
+
+ private void checkMultiSeriesTimeAligned(boolean createTS) throws ClassNotFoundException {
+ insertSQL(createTS);
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ String querySQL =
+ "SELECT speed.* FROM root.wind1.azq01 WHERE Speed "
+ + String.format("CONTAIN (%s) WITH TOLERANCE 100000 ", getArrayRange(170, 200, 10))
+ + String.format("CONCAT (%s) WITH TOLERANCE 200 ", getArrayRange(250, 300, 10))
+ + String.format("CONCAT (%s) WITH TOLERANCE 100000 ", getArrayRange(400, 430, 10));
+ System.out.println(querySQL);
+ boolean hasIndex = statement.execute(querySQL);
+ String gt =
+ "Time,root.wind1.azq01.speed.17,\n"
+ + "17,170.0,\n"
+ + "18,180.0,\n"
+ + "19,190.0,\n"
+ + "25,250.0,\n"
+ + "26,260.0,\n"
+ + "27,270.0,\n"
+ + "28,280.0,\n"
+ + "29,290.0,\n"
+ + "40,400.0,\n"
+ + "41,410.0,\n"
+ + "42,420.0,";
+ Assert.assertTrue(hasIndex);
+ try (ResultSet resultSet = statement.getResultSet()) {
+ ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ StringBuilder sb = new StringBuilder();
+ for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
+ sb.append(resultSetMetaData.getColumnName(i)).append(",");
+ }
+ sb.append("\n");
+ while (resultSet.next()) {
+ for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
+ sb.append(resultSet.getString(i)).append(",");
+ }
+ sb.append("\n");
+ }
+ System.out.println(sb);
+ // Assert.assertEquals(gt, sb.toString());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void checkReadWithCreateTS() throws ClassNotFoundException {
+ checkRead(true);
+ }
+
+ @Test
+ public void checkReadWithoutCreateTS() throws ClassNotFoundException {
+ checkRead(false);
+ }
+
+ private void checkRead(boolean createTS) throws ClassNotFoundException {
+ insertSQL(createTS);
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ String querySQL =
+ "SELECT speed.* FROM root.wind1.azq01 WHERE Speed "
+ + String.format("CONTAIN (%s) WITH TOLERANCE 0 ", getArrayRange(170, 200, 10))
+ + String.format("CONCAT (%s) WITH TOLERANCE 0 ", getArrayRange(250, 300, 10))
+ + String.format("CONCAT (%s) WITH TOLERANCE 0 ", getArrayRange(400, 430, 10));
+ System.out.println(querySQL);
+ statement.setQueryTimeout(20);
+ boolean hasIndex = statement.execute(querySQL);
+ String gt =
+ "Time,root.wind1.azq01.speed.17,\n"
+ + "17,170.0,\n"
+ + "18,180.0,\n"
+ + "19,190.0,\n"
+ + "25,250.0,\n"
+ + "26,260.0,\n"
+ + "27,270.0,\n"
+ + "28,280.0,\n"
+ + "29,290.0,\n"
+ + "40,400.0,\n"
+ + "41,410.0,\n"
+ + "42,420.0,\n";
+ Assert.assertTrue(hasIndex);
+ try (ResultSet resultSet = statement.getResultSet()) {
+ ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
+ StringBuilder sb = new StringBuilder();
+ for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
+ sb.append(resultSetMetaData.getColumnName(i)).append(",");
+ }
+ sb.append("\n");
+ while (resultSet.next()) {
+ for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
+ sb.append(resultSet.getString(i)).append(",");
+ }
+ sb.append("\n");
+ }
+ System.out.println(sb);
+ Assert.assertEquals(gt, sb.toString());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}
diff --git a/server/src/test/java/org/apache/iotdb/db/index/it/MMHHIndexReadIT.java b/server/src/test/java/org/apache/iotdb/db/index/it/MMHHIndexReadIT.java
index 38974f3..950db1f 100644
--- a/server/src/test/java/org/apache/iotdb/db/index/it/MMHHIndexReadIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/index/it/MMHHIndexReadIT.java
@@ -122,7 +122,6 @@ public class MMHHIndexReadIT {
statement.execute("flush");
System.out.println(IndexManager.getInstance().getRouter());
-
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/iotdb/IoTDBInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/iotdb/IoTDBInterpreter.java
index f4d0102..0665db0 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/iotdb/IoTDBInterpreter.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/iotdb/IoTDBInterpreter.java
@@ -222,8 +222,7 @@ public class IoTDBInterpreter extends AbstractInterpreter {
String timeDisplayType = values[1].trim();
this.timeFormat = setTimeFormat(values[1]);
return new InterpreterResult(Code.SUCCESS, "Time display type has set to " + timeDisplayType);
- }
- else if (specialCmd.startsWith(SET_QUERY_TIMEOUT)) {
+ } else if (specialCmd.startsWith(SET_QUERY_TIMEOUT)) {
String[] values = cmd.split(EQUAL_SIGN);
if (values.length != 2) {
throw new StatementExecutionException(
@@ -231,7 +230,8 @@ public class IoTDBInterpreter extends AbstractInterpreter {
"Query timeout format error, please input like %s=10", SET_QUERY_TIMEOUT));
}
this.queryTimeout = Integer.parseInt(values[1].trim());
- return new InterpreterResult(Code.SUCCESS, "Query timeout has set to " + queryTimeout + " seconds");
+ return new InterpreterResult(
+ Code.SUCCESS, "Query timeout has set to " + queryTimeout + " seconds");
}
return executeQuery(connection, cmd);
}
@@ -240,7 +240,7 @@ public class IoTDBInterpreter extends AbstractInterpreter {
StringBuilder stringBuilder = new StringBuilder();
try (Statement statement = connection.createStatement()) {
statement.setFetchSize(fetchSize);
- if(this.queryTimeout > 0){
+ if (this.queryTimeout > 0) {
statement.setQueryTimeout(this.queryTimeout);
}
boolean hasResultSet = statement.execute(cmd.trim());
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/iotdb/IoTDBInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/iotdb/IoTDBInterpreterTest.java
index d5dbbd8..0d775aa 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/iotdb/IoTDBInterpreterTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/iotdb/IoTDBInterpreterTest.java
@@ -138,12 +138,10 @@ public class IoTDBInterpreterTest {
public void testSetQueryTimeout() {
// default
Assert.assertEquals(-1, interpreter.getQueryTimeout());
- InterpreterResult actual =
- interpreter.internalInterpret(SET_QUERY_TIMEOUT + "=" + 2, null);
+ InterpreterResult actual = interpreter.internalInterpret(SET_QUERY_TIMEOUT + "=" + 2, null);
Assert.assertNotNull(actual);
Assert.assertEquals(Code.SUCCESS, actual.code());
- Assert.assertEquals(
- "Query timeout has set to 2 seconds", actual.message().get(0).getData());
+ Assert.assertEquals("Query timeout has set to 2 seconds", actual.message().get(0).getData());
Assert.assertEquals(2, interpreter.getQueryTimeout());
}
[iotdb] 02/02: fix elb bug
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kangrong pushed a commit to branch f_index_dev
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 68a2944ab4b5efba8a701f8a460b83c5873bb5f1
Author: kr11 <3095717866.com>
AuthorDate: Thu May 13 12:14:04 2021 +0800
fix elb bug
---
.../iotdb/db/index/algorithm/elb/ELBIndex.java | 14 +-
.../algorithm/elb/feature/SequenceELBFeature.java | 2 +-
.../org/apache/iotdb/db/index/it/ELBWindIT.java | 196 ++++++---------------
3 files changed, 64 insertions(+), 148 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/index/algorithm/elb/ELBIndex.java b/server/src/main/java/org/apache/iotdb/db/index/algorithm/elb/ELBIndex.java
index 552fcd8..99f8cee 100644
--- a/server/src/main/java/org/apache/iotdb/db/index/algorithm/elb/ELBIndex.java
+++ b/server/src/main/java/org/apache/iotdb/db/index/algorithm/elb/ELBIndex.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.db.index.common.DistSeries;
import org.apache.iotdb.db.index.common.IndexInfo;
import org.apache.iotdb.db.index.common.IndexUtils;
import org.apache.iotdb.db.index.common.distance.Distance;
+import org.apache.iotdb.db.index.common.distance.LInfinityNormdouble;
import org.apache.iotdb.db.index.read.TVListPointer;
import org.apache.iotdb.db.index.read.optimize.IIndexCandidateOrderOptimize;
import org.apache.iotdb.db.index.stats.IndexStatManager;
@@ -383,7 +384,16 @@ public class ELBIndex extends IoTDBIndex {
for (int i = 0; i < patternList.size(); i++) {
double[] pattern = patternList.get(i);
patternLength += pattern.length;
- struct.thresholds[i] = thresholdList.get(i);
+ // note that, in following paper implementation, thres_avg means "avg threshold".
+ // The actual threshold (the meaning in SQL) of "multi-threshold pattern" is:
+ // thres_sql ** p = (thres_avg ** p) * len(subpattern)
+ // thres_avg = [(thres_sql ** p) / len(subpattern)] ** 1/p
+
+ if (struct.distance instanceof LInfinityNormdouble)
+ throw new IllegalIndexParamException("L-inf not supported yet");
+ double p = struct.distance.getP();
+ struct.thresholds[i] = Math.pow(Math.pow(thresholdList.get(i), p) / pattern.length, 1 / p);
+ // struct.thresholds[i] = thresholdList.get(i);
struct.borders[i + 1] = patternLength;
}
struct.pattern = new double[patternLength];
@@ -439,7 +449,7 @@ public class ELBIndex extends IoTDBIndex {
}
System.out.println(
String.format(
- "========== elb prune: %d/%d=%.3f",
+ "========== elb prune(1-cannot_pruned/total): 1-%d/%d=%.3f",
cannotPrunedCount,
windowBlockFeatures.size() - struct.blockNum + 1,
(double) cannotPrunedCount / (windowBlockFeatures.size() - struct.blockNum)));
diff --git a/server/src/main/java/org/apache/iotdb/db/index/algorithm/elb/feature/SequenceELBFeature.java b/server/src/main/java/org/apache/iotdb/db/index/algorithm/elb/feature/SequenceELBFeature.java
index a3f8f15..cf7dd48 100644
--- a/server/src/main/java/org/apache/iotdb/db/index/algorithm/elb/feature/SequenceELBFeature.java
+++ b/server/src/main/java/org/apache/iotdb/db/index/algorithm/elb/feature/SequenceELBFeature.java
@@ -122,7 +122,7 @@ public class SequenceELBFeature extends ELBFeature {
(pattern.maxLeftBorders[m - 1] - pattern.minLeftBorders[m - 1])
* distance.getThresholdNoRoot(pattern.thresholdsArray[m - 1]);
// Line 5~7，middle
- for (int i = pattern.maxLeftBorders[m - 1]; i < pattern.minLeftBorders[n + 1] - 1; i++) {
+ for (int i = pattern.maxLeftBorders[m - 1]; i < pattern.minLeftBorders[n + 1]; i++) {
sum += distance.getThresholdNoRoot(getAldMax(pattern, i));
}
// Line 8，right
diff --git a/server/src/test/java/org/apache/iotdb/db/index/it/ELBWindIT.java b/server/src/test/java/org/apache/iotdb/db/index/it/ELBWindIT.java
index 1387c42..35daf04 100644
--- a/server/src/test/java/org/apache/iotdb/db/index/it/ELBWindIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/index/it/ELBWindIT.java
@@ -19,8 +19,6 @@
package org.apache.iotdb.db.index.it;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.index.IndexManager;
-import org.apache.iotdb.db.index.common.IndexUtils;
import org.apache.iotdb.db.rescon.TVListAllocator;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.db.utils.datastructure.TVList;
@@ -32,17 +30,21 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.BufferedReader;
+import java.io.FileReader;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
-import static org.apache.iotdb.db.index.IndexTestUtils.getArrayRange;
+import static org.apache.iotdb.db.index.IndexTestUtils.getStringFromList;
import static org.apache.iotdb.db.index.common.IndexType.ELB_INDEX;
import static org.junit.Assert.fail;
-public class ELBIndexReadIT {
+public class ELBWindIT {
private static final String insertPattern = "INSERT INTO %s(timestamp, %s) VALUES (%d, %.3f)";
@@ -77,11 +79,6 @@ public class ELBIndexReadIT {
}
private static void insertSQL(boolean createTS) throws ClassNotFoundException {
- for (int i = 0; i < 10000; i++) {
- System.out.println(
- String.format(insertPattern, speed1Device, speed1Sensor, i, (float) (i * 10 + 9)));
- }
- IndexUtils.breakDown("asd");
Class.forName(Config.JDBC_DRIVER_NAME);
try (Connection connection =
DriverManager.getConnection(
@@ -109,66 +106,43 @@ public class ELBIndexReadIT {
System.out.println(
String.format("CREATE INDEX ON %s WITH INDEX=%s, BLOCK_SIZE=%d", indexSub, ELB_INDEX, 5));
- TVList subInput = TVListAllocator.getInstance().allocate(TSDataType.DOUBLE);
- for (int i = 0; i < subLength; i++) {
- subInput.putDouble(i, i * 10);
- }
- for (int i = 0; i < 20; i++) {
- statement.execute(
- String.format(
- insertPattern,
- speed1Device,
- speed1Sensor,
- subInput.getTime(i),
- subInput.getDouble(i)));
- System.out.println(
- String.format(
- insertPattern,
- speed1Device,
- speed1Sensor,
- subInput.getTime(i),
- subInput.getDouble(i)));
- }
- statement.execute("flush");
- // System.out.println("==========================");
- // System.out.println(IndexManager.getInstance().getRouter());
-
- for (int i = 25; i < 30; i++) {
- statement.execute(
- String.format(
- insertPattern,
- speed1Device,
- speed1Sensor,
- subInput.getTime(i),
- subInput.getDouble(i)));
- System.out.println(
- String.format(
- insertPattern,
- speed1Device,
- speed1Sensor,
- subInput.getTime(i),
- subInput.getDouble(i)));
+ TVList subInput = TVListAllocator.getInstance().allocate(TSDataType.FLOAT);
+ // read base
+ String basePath = "/Users/kangrong/tsResearch/tols/JINFENG/d2/out_sub_base.csv";
+ int idx = 0;
+ try (BufferedReader csvReader = new BufferedReader(new FileReader(basePath))) {
+ String row;
+ while ((row = csvReader.readLine()) != null) {
+ if (idx >= 1) {
+ String[] data = row.split(",");
+ long t = Long.parseLong(data[0]);
+ float v = Float.parseFloat(data[1]);
+
+ // subInput.putDouble(t, v);
+ subInput.putFloat(idx, v);
+ }
+ idx++;
+ }
}
- statement.execute("flush");
-
- IndexManager.getInstance().stop();
- IndexManager.getInstance().start();
- for (int i = 40; i < subLength; i++) {
+ // for (int i = 0; i < subLength; i++) {
+ // subInput.putDouble(i, i * 10);
+ // }
+ for (int i = 0; i < subInput.size(); i++) {
statement.execute(
String.format(
insertPattern,
speed1Device,
speed1Sensor,
subInput.getTime(i),
- subInput.getDouble(i)));
- System.out.println(
- String.format(
- insertPattern,
- speed1Device,
- speed1Sensor,
- subInput.getTime(i),
- subInput.getDouble(i)));
+ subInput.getFloat(i)));
+ // System.out.println(
+ // String.format(
+ // insertPattern,
+ // speed1Device,
+ // speed1Sensor,
+ // subInput.getTime(i),
+ // subInput.getFloat(i)));
}
statement.execute("flush");
// System.out.println("==========================");
@@ -181,71 +155,6 @@ public class ELBIndexReadIT {
}
@Test
- public void checkMultiSeriesTimeAlignedWithCreateTS() throws ClassNotFoundException {
- checkMultiSeriesTimeAligned(true);
- }
-
- @Test
- public void checkMultiSeriesTimeAlignedWithoutCreateTS() throws ClassNotFoundException {
- checkMultiSeriesTimeAligned(false);
- }
-
- private void checkMultiSeriesTimeAligned(boolean createTS) throws ClassNotFoundException {
- insertSQL(createTS);
- Class.forName(Config.JDBC_DRIVER_NAME);
- try (Connection connection =
- DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
- Statement statement = connection.createStatement()) {
-
- String querySQL =
- "SELECT speed.* FROM root.wind1.azq01 WHERE Speed "
- + String.format("CONTAIN (%s) WITH TOLERANCE 100000 ", getArrayRange(170, 200, 10))
- + String.format("CONCAT (%s) WITH TOLERANCE 200 ", getArrayRange(250, 300, 10))
- + String.format("CONCAT (%s) WITH TOLERANCE 100000 ", getArrayRange(400, 430, 10));
- System.out.println(querySQL);
- boolean hasIndex = statement.execute(querySQL);
- String gt =
- "Time,root.wind1.azq01.speed.17,\n"
- + "17,170.0,\n"
- + "18,180.0,\n"
- + "19,190.0,\n"
- + "25,250.0,\n"
- + "26,260.0,\n"
- + "27,270.0,\n"
- + "28,280.0,\n"
- + "29,290.0,\n"
- + "40,400.0,\n"
- + "41,410.0,\n"
- + "42,420.0,";
- Assert.assertTrue(hasIndex);
- try (ResultSet resultSet = statement.getResultSet()) {
- ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
- StringBuilder sb = new StringBuilder();
- for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
- sb.append(resultSetMetaData.getColumnName(i)).append(",");
- }
- sb.append("\n");
- while (resultSet.next()) {
- for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
- sb.append(resultSet.getString(i)).append(",");
- }
- sb.append("\n");
- }
- System.out.println(sb);
- // Assert.assertEquals(gt, sb.toString());
- }
- } catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
- }
-
- @Test
- public void checkReadWithCreateTS() throws ClassNotFoundException {
- checkRead(true);
- }
-
- @Test
public void checkReadWithoutCreateTS() throws ClassNotFoundException {
checkRead(false);
}
@@ -256,28 +165,25 @@ public class ELBIndexReadIT {
try (Connection connection =
DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
Statement statement = connection.createStatement()) {
-
+ String queryPath = "/Users/kangrong/tsResearch/tols/JINFENG/d2/out_sub_pattern.csv";
+ List<Float> pattern = new ArrayList<>();
+ try (BufferedReader csvReader = new BufferedReader(new FileReader(queryPath))) {
+ String row;
+ while ((row = csvReader.readLine()) != null) {
+ float v = Float.parseFloat(row);
+ pattern.add(v);
+ }
+ }
String querySQL =
"SELECT speed.* FROM root.wind1.azq01 WHERE Speed "
- + String.format("CONTAIN (%s) WITH TOLERANCE 0 ", getArrayRange(170, 200, 10))
- + String.format("CONCAT (%s) WITH TOLERANCE 0 ", getArrayRange(250, 300, 10))
- + String.format("CONCAT (%s) WITH TOLERANCE 0 ", getArrayRange(400, 430, 10));
+ + String.format("CONTAIN (%s) WITH TOLERANCE 10 ", getStringFromList(pattern, 0, 30))
+ + String.format("CONCAT (%s) WITH TOLERANCE 25 ", getStringFromList(pattern, 30, 70))
+ + String.format(
+ "CONCAT (%s) WITH TOLERANCE 10 ", getStringFromList(pattern, 70, 100));
System.out.println(querySQL);
- statement.setQueryTimeout(20);
+ statement.setQueryTimeout(200);
boolean hasIndex = statement.execute(querySQL);
- String gt =
- "Time,root.wind1.azq01.speed.17,\n"
- + "17,170.0,\n"
- + "18,180.0,\n"
- + "19,190.0,\n"
- + "25,250.0,\n"
- + "26,260.0,\n"
- + "27,270.0,\n"
- + "28,280.0,\n"
- + "29,290.0,\n"
- + "40,400.0,\n"
- + "41,410.0,\n"
- + "42,420.0,\n";
+ // String gt = "Time,root.wind1.azq01.speed.17,\n";
Assert.assertTrue(hasIndex);
try (ResultSet resultSet = statement.getResultSet()) {
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
@@ -293,7 +199,7 @@ public class ELBIndexReadIT {
sb.append("\n");
}
System.out.println(sb);
- Assert.assertEquals(gt, sb.toString());
+ // Assert.assertEquals(gt, sb.toString());
}
} catch (Exception e) {
e.printStackTrace();