You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2022/11/23 01:14:58 UTC
[iotdb] branch master updated: Support Create timeseries with deadband field (#8093)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 32e2f982c2 Support Create timeseries with deadband field (#8093)
32e2f982c2 is described below
commit 32e2f982c20f73b84c3b606f1448428f8686bc9d
Author: Marcos_Zyk <38...@users.noreply.github.com>
AuthorDate: Wed Nov 23 09:14:53 2022 +0800
Support Create timeseries with deadband field (#8093)
---
docs/UserGuide/Data-Concept/Deadband-Process.md | 2 +-
docs/UserGuide/Reference/SQL-Reference.md | 12 +++----
docs/zh/UserGuide/Data-Concept/Deadband-Process.md | 2 +-
docs/zh/UserGuide/Reference/SQL-Reference.md | 12 +++----
.../apache/iotdb/commons/conf/IoTDBConstant.java | 16 +++++++++
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 42 ++++++++++++++++++++--
6 files changed, 70 insertions(+), 16 deletions(-)
diff --git a/docs/UserGuide/Data-Concept/Deadband-Process.md b/docs/UserGuide/Data-Concept/Deadband-Process.md
index 32fe030ac6..115afbe2b8 100644
--- a/docs/UserGuide/Data-Concept/Deadband-Process.md
+++ b/docs/UserGuide/Data-Concept/Deadband-Process.md
@@ -61,7 +61,7 @@ Supported datatypes:
The following is an example of using SDT compression.
```
-IoTDB> CREATE TIMESERIES root.sg1.d0.s0 WITH DATATYPE=INT32,ENCODING=PLAIN,LOSS=SDT,COMPDEV=2
+IoTDB> CREATE TIMESERIES root.sg1.d0.s0 WITH DATATYPE=INT32,ENCODING=PLAIN,DEADBAND=SDT,COMPDEV=2
```
Prior to flushing and SDT compression, the results are shown below:
diff --git a/docs/UserGuide/Reference/SQL-Reference.md b/docs/UserGuide/Reference/SQL-Reference.md
index 28117e9abe..db5dbb76b0 100644
--- a/docs/UserGuide/Reference/SQL-Reference.md
+++ b/docs/UserGuide/Reference/SQL-Reference.md
@@ -98,9 +98,9 @@ Eg: CREATE TIMESERIES root.ln.wf01.wt01.status WITH DATATYPE=BOOLEAN, ENCODING=P
Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE
Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE, COMPRESSOR=SNAPPY, MAX_POINT_NUMBER=3
Eg: CREATE TIMESERIES root.turbine.d0.s0(temperature) WITH DATATYPE=FLOAT, ENCODING=RLE, COMPRESSOR=SNAPPY tags(unit=f, description='turbine this is a test1') attributes(H_Alarm=100, M_Alarm=50)
-Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE, LOSS=SDT, COMPDEV=0.01
-Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE, LOSS=SDT, COMPDEV=0.01, COMPMINTIME=3
-Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE, LOSS=SDT, COMPDEV=0.01, COMPMINTIME=2, COMPMAXTIME=15
+Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE, DEADBAND=SDT, COMPDEV=0.01
+Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE, DEADBAND=SDT, COMPDEV=0.01, COMPMINTIME=3
+Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE, DEADBAND=SDT, COMPDEV=0.01, COMPMINTIME=2, COMPMAXTIME=15
Note: Datatype and encoding type must be corresponding. Please check Chapter 3 Encoding Section for details.
Note: When propertyValue is SDT, it is required to set compression deviation COMPDEV, which is the maximum absolute difference between values.
Note: For SDT, values withtin COMPDEV will be discarded.
@@ -123,9 +123,9 @@ Eg: CREATE TIMESERIES root.ln.wf01.wt01.status BOOLEAN ENCODING=PLAIN
Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature FLOAT ENCODING=RLE
Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature FLOAT ENCODING=RLE COMPRESSOR=SNAPPY MAX_POINT_NUMBER=3
Eg: CREATE TIMESERIES root.turbine.d0.s0(temperature) FLOAT ENCODING=RLE COMPRESSOR=SNAPPY tags(unit=f, description='turbine this is a test1') attributes(H_Alarm=100, M_Alarm=50)
-Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature FLOAT ENCODING=RLE LOSS=SDT COMPDEV=0.01
-Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature FLOAT ENCODING=RLE LOSS=SDT COMPDEV=0.01 COMPMINTIME=3
-Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature FLOAT ENCODING=RLE LOSS=SDT COMPDEV=0.01 COMPMINTIME=2 COMPMAXTIME=15
+Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature FLOAT ENCODING=RLE DEADBAND=SDT COMPDEV=0.01
+Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature FLOAT ENCODING=RLE DEADBAND=SDT COMPDEV=0.01 COMPMINTIME=3
+Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature FLOAT ENCODING=RLE DEADBAND=SDT COMPDEV=0.01 COMPMINTIME=2 COMPMAXTIME=15
```
* Create Aligned Timeseries Statement
diff --git a/docs/zh/UserGuide/Data-Concept/Deadband-Process.md b/docs/zh/UserGuide/Data-Concept/Deadband-Process.md
index 8f8427e426..cac24d28a5 100644
--- a/docs/zh/UserGuide/Data-Concept/Deadband-Process.md
+++ b/docs/zh/UserGuide/Data-Concept/Deadband-Process.md
@@ -58,7 +58,7 @@ SDT 的指定语法详见本文 [SQL 参考文档](../Reference/SQL-Reference.md
以下是使用 SDT 压缩的示例。
```
-IoTDB> CREATE TIMESERIES root.sg1.d0.s0 WITH DATATYPE=INT32,ENCODING=PLAIN,LOSS=SDT,COMPDEV=2
+IoTDB> CREATE TIMESERIES root.sg1.d0.s0 WITH DATATYPE=INT32,ENCODING=PLAIN,DEADBAND=SDT,COMPDEV=2
```
刷入磁盘和 SDT 压缩之前，结果如下所示：
diff --git a/docs/zh/UserGuide/Reference/SQL-Reference.md b/docs/zh/UserGuide/Reference/SQL-Reference.md
index 8ab352400b..6d0f4e02db 100644
--- a/docs/zh/UserGuide/Reference/SQL-Reference.md
+++ b/docs/zh/UserGuide/Reference/SQL-Reference.md
@@ -87,9 +87,9 @@ Eg: CREATE TIMESERIES root.ln.wf01.wt01.status WITH DATATYPE=BOOLEAN, ENCODING=P
Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE
Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE, COMPRESSOR=SNAPPY, MAX_POINT_NUMBER=3
Eg: CREATE TIMESERIES root.turbine.d0.s0(temperature) WITH DATATYPE=FLOAT, ENCODING=RLE, COMPRESSOR=SNAPPY tags(unit=f, description='turbine this is a test1') attributes(H_Alarm=100, M_Alarm=50)
-Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE, LOSS=SDT, COMPDEV=0.01
-Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE, LOSS=SDT, COMPDEV=0.01, COMPMINTIME=3
-Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE, LOSS=SDT, COMPDEV=0.01, COMPMINTIME=2, COMPMAXTIME=15
+Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE, DEADBAND=SDT, COMPDEV=0.01
+Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE, DEADBAND=SDT, COMPDEV=0.01, COMPMINTIME=3
+Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=RLE, DEADBAND=SDT, COMPDEV=0.01, COMPMINTIME=2, COMPMAXTIME=15
Note: Datatype and encoding type must be corresponding. Please check Chapter 3 Encoding Section for details.
Note: When propertyValue is SDT, it is required to set compression deviation COMPDEV, which is the maximum absolute difference between values.
Note: For SDT, values withtin COMPDEV will be discarded.
@@ -112,9 +112,9 @@ Eg: CREATE TIMESERIES root.ln.wf01.wt01.status BOOLEAN ENCODING=PLAIN
Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature FLOAT ENCODING=RLE
Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature FLOAT ENCODING=RLE COMPRESSOR=SNAPPY MAX_POINT_NUMBER=3
Eg: CREATE TIMESERIES root.turbine.d0.s0(temperature) FLOAT ENCODING=RLE COMPRESSOR=SNAPPY tags(unit=f, description='turbine this is a test1') attributes(H_Alarm=100, M_Alarm=50)
-Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature FLOAT ENCODING=RLE LOSS=SDT COMPDEV=0.01
-Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature FLOAT ENCODING=RLE LOSS=SDT COMPDEV=0.01 COMPMINTIME=3
-Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature FLOAT ENCODING=RLE LOSS=SDT COMPDEV=0.01 COMPMINTIME=2 COMPMAXTIME=15
+Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature FLOAT ENCODING=RLE DEADBAND=SDT COMPDEV=0.01
+Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature FLOAT ENCODING=RLE DEADBAND=SDT COMPDEV=0.01 COMPMINTIME=3
+Eg: CREATE TIMESERIES root.ln.wf01.wt01.temperature FLOAT ENCODING=RLE DEADBAND=SDT COMPDEV=0.01 COMPMINTIME=2 COMPMAXTIME=15
```
* 创建对齐时间序列语句
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
index bb0df10e17..00d8a48eaa 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.commons.conf;
import java.io.File;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.Properties;
import java.util.Set;
@@ -178,6 +179,21 @@ public class IoTDBConstant {
public static final String[] SDT_PARAMETERS =
new String[] {SDT_COMP_DEV, SDT_COMP_MIN_TIME, SDT_COMP_MAX_TIME};
+ public static final String DEADBAND = "deadband";
+ public static final String MAX_POINT_NUMBER = "max_point_number";
+ public static final String MAX_STRING_LENGTH = "max_string_length";
+ public static final Set<String> ALLOWED_SCHEMA_PROPS =
+ new HashSet<>(
+ Arrays.asList(
+ DEADBAND,
+ LOSS,
+ SDT,
+ SDT_COMP_DEV,
+ SDT_COMP_MIN_TIME,
+ SDT_COMP_MAX_TIME,
+ MAX_POINT_NUMBER,
+ MAX_STRING_LENGTH));
+
// default base dir, stores all IoTDB runtime files
public static final String DEFAULT_BASE_DIR = "data" + File.separator + "datanode";
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
index 11e97a4561..0c49d50a4e 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
@@ -137,6 +137,7 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@@ -144,6 +145,9 @@ import java.util.TimeZone;
import java.util.stream.Collectors;
import static com.google.common.base.Preconditions.checkState;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.ALLOWED_SCHEMA_PROPS;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.DEADBAND;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.LOSS;
import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD;
import static org.apache.iotdb.db.metadata.MetadataConstant.ALL_RESULT_NODES;
import static org.apache.iotdb.db.mpp.common.header.ColumnHeaderConstant.DEVICE;
@@ -1266,11 +1270,12 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
@Override
public Analysis visitCreateTimeseries(
CreateTimeSeriesStatement createTimeSeriesStatement, MPPQueryContext context) {
+ context.setQueryType(QueryType.WRITE);
if (createTimeSeriesStatement.getPath().getNodeLength() < 3) {
throw new RuntimeException(
new IllegalPathException(createTimeSeriesStatement.getPath().getFullPath()));
}
- context.setQueryType(QueryType.WRITE);
+ analyzeSchemaProps(createTimeSeriesStatement.getProps());
if (createTimeSeriesStatement.getTags() != null
&& !createTimeSeriesStatement.getTags().isEmpty()
&& createTimeSeriesStatement.getAttributes() != null
@@ -1282,6 +1287,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
}
}
}
+
Analysis analysis = new Analysis();
analysis.setStatement(createTimeSeriesStatement);
@@ -1345,14 +1351,44 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
}
}
+ private void analyzeSchemaProps(Map<String, String> props) {
+ if (props == null || props.isEmpty()) {
+ return;
+ }
+ Map<String, String> caseChangeMap = new HashMap<>();
+ for (String key : props.keySet()) {
+ caseChangeMap.put(key.toLowerCase(Locale.ROOT), key);
+ }
+ for (String lowerCaseKey : caseChangeMap.keySet()) {
+ if (!ALLOWED_SCHEMA_PROPS.contains(lowerCaseKey)) {
+ throw new SemanticException(
+ new MetadataException(
+ String.format("%s is not a legal prop.", caseChangeMap.get(lowerCaseKey))));
+ }
+ props.put(lowerCaseKey, props.remove(caseChangeMap.get(lowerCaseKey)));
+ }
+ if (props.containsKey(DEADBAND)) {
+ props.put(LOSS, props.remove(DEADBAND));
+ }
+ }
+
+ private void analyzeSchemaProps(List<Map<String, String>> propsList) {
+ if (propsList == null) {
+ return;
+ }
+ for (Map<String, String> props : propsList) {
+ analyzeSchemaProps(props);
+ }
+ }
+
@Override
public Analysis visitCreateAlignedTimeseries(
CreateAlignedTimeSeriesStatement createAlignedTimeSeriesStatement, MPPQueryContext context) {
+ context.setQueryType(QueryType.WRITE);
if (createAlignedTimeSeriesStatement.getDevicePath().getNodeLength() < 2) {
throw new RuntimeException(
new IllegalPathException(createAlignedTimeSeriesStatement.getDevicePath().getFullPath()));
}
- context.setQueryType(QueryType.WRITE);
List<String> measurements = createAlignedTimeSeriesStatement.getMeasurements();
Set<String> measurementsSet = new HashSet<>(measurements);
if (measurementsSet.size() < measurements.size()) {
@@ -1412,6 +1448,8 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
Analysis analysis = new Analysis();
analysis.setStatement(createMultiTimeSeriesStatement);
+ analyzeSchemaProps(createMultiTimeSeriesStatement.getPropsList());
+
List<PartialPath> timeseriesPathList = createMultiTimeSeriesStatement.getPaths();
List<String> aliasList = createMultiTimeSeriesStatement.getAliasList();
for (int i = 0; i < timeseriesPathList.size(); i++) {