You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2022/01/13 16:04:53 UTC
[iotdb] branch master updated: [IOTDB-2342] Add compatible interface with 0.12.x for creating template of flat measurements (#4720)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 84a1df3 [IOTDB-2342] Add compatible interface with 0.12.x for creating template of flat measurements (#4720)
84a1df3 is described below
commit 84a1df3576de0e68c90e5499adfecb332c3f53d7
Author: ZhaoXin <x_...@163.com>
AuthorDate: Fri Jan 14 00:04:19 2022 +0800
[IOTDB-2342] Add compatible interface with 0.12.x for creating template of flat measurements (#4720)
---
.../iotdb/db/metadata/template/Template.java | 10 +-
.../java/org/apache/iotdb/session/Session.java | 79 +++++++++++++
.../org/apache/iotdb/session/pool/SessionPool.java | 82 ++++++++++++++
.../java/org/apache/iotdb/session/SessionTest.java | 122 ++++++++++++++++++++-
.../apache/iotdb/session/template/TemplateUT.java | 27 ++++-
5 files changed, 313 insertions(+), 7 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/template/Template.java b/server/src/main/java/org/apache/iotdb/db/metadata/template/Template.java
index 4af2cfb..cb61a67 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/template/Template.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/template/Template.java
@@ -170,7 +170,7 @@ public class Template {
for (String path : alignedPaths) {
// check aligned whether legal, and records measurements name
if (getPathNodeInTemplate(path) != null) {
- throw new IllegalPathException("Path duplicated: " + prefix);
+ throw new IllegalPathException("Path duplicated: " + path);
}
pathNodes = MetaUtils.splitPathToDetachedPath(path);
@@ -479,7 +479,8 @@ public class Template {
pathNode = MetaUtils.splitPathToDetachedPath(measurements[0]);
prefix = joinBySeparator(Arrays.copyOf(pathNode, pathNode.length - 1));
IMNode targetNode = getPathNodeInTemplate(prefix);
- if (targetNode != null && !targetNode.getAsEntityMNode().isAligned()) {
+ if ((targetNode != null && !targetNode.getAsEntityMNode().isAligned())
+ || (prefix.equals("") && !this.isDirectAligned())) {
throw new IllegalPathException(prefix, "path already exists but not aligned");
}
@@ -512,8 +513,9 @@ public class Template {
// If prefix exists and aligned, it will throw exception
prefix = joinBySeparator(Arrays.copyOf(pathNode, pathNode.length - 1));
IMNode parNode = getPathNodeInTemplate(prefix);
- if (parNode != null && parNode.getAsEntityMNode().isAligned()) {
- throw new IllegalPathException(prefix, "path already exists and aligned");
+ if ((parNode != null && parNode.getAsEntityMNode().isAligned())
+ || (prefix.equals("") && this.isDirectAligned())) {
+ throw new IllegalPathException(measurements[i], "path already exists and aligned");
}
IMeasurementSchema schema =
diff --git a/session/src/main/java/org/apache/iotdb/session/Session.java b/session/src/main/java/org/apache/iotdb/session/Session.java
index 5c4f1f2..2b339f4 100644
--- a/session/src/main/java/org/apache/iotdb/session/Session.java
+++ b/session/src/main/java/org/apache/iotdb/session/Session.java
@@ -42,6 +42,7 @@ import org.apache.iotdb.service.rpc.thrift.TSQueryTemplateReq;
import org.apache.iotdb.service.rpc.thrift.TSQueryTemplateResp;
import org.apache.iotdb.service.rpc.thrift.TSSetSchemaTemplateReq;
import org.apache.iotdb.service.rpc.thrift.TSUnsetSchemaTemplateReq;
+import org.apache.iotdb.session.template.MeasurementNode;
import org.apache.iotdb.session.template.Template;
import org.apache.iotdb.session.template.TemplateQueryType;
import org.apache.iotdb.session.util.SessionUtils;
@@ -1900,6 +1901,84 @@ public class Session {
}
/**
+ * Create a template with flat measurements, not tree structured. Need to specify datatype,
+ * encoding and compressor of each measurement, and alignment of these measurements at once.
+ *
+ * @oaram templateName name of template to create
+ * @param measurements flat measurements of the template, cannot contain character dot
+ * @param dataTypes datatype of each measurement in the template
+ * @param encodings encodings of each measurement in the template
+ * @param compressors compression type of each measurement in the template
+ * @param isAligned specify whether these flat measurements are aligned
+ */
+ public void createSchemaTemplate(
+ String templateName,
+ List<String> measurements,
+ List<TSDataType> dataTypes,
+ List<TSEncoding> encodings,
+ List<CompressionType> compressors,
+ boolean isAligned)
+ throws IOException, IoTDBConnectionException, StatementExecutionException {
+ Template temp = new Template(templateName, isAligned);
+ int len = measurements.size();
+ if (len != dataTypes.size() || len != encodings.size() || len != compressors.size()) {
+ throw new StatementExecutionException(
+ "Different length of measurements, datatypes, encodings "
+ + "or compressors when create schema tempalte.");
+ }
+ for (int idx = 0; idx < measurements.size(); idx++) {
+ MeasurementNode mNode =
+ new MeasurementNode(
+ measurements.get(idx), dataTypes.get(idx), encodings.get(idx), compressors.get(idx));
+ temp.addToTemplate(mNode);
+ }
+ createSchemaTemplate(temp);
+ }
+
+ /**
+ * Compatible for rel/0.12, this method will create an unaligned flat template as a result. Notice
+ * that there is no aligned concept in 0.12, so only the first measurement in each nested list
+ * matters.
+ *
+ * @param name name of the template
+ * @param schemaNames it works as a virtual layer inside template in 0.12, and makes no difference
+ * after 0.13
+ * @param measurements the first measurement in each nested list will constitute the final flat
+ * template
+ * @param dataTypes the data type of each measurement, only the first one in each nested list
+ * matters as above
+ * @param encodings the encoding of each measurement, only the first one in each nested list
+ * matters as above
+ * @param compressors the compressor of each measurement
+ * @throws IoTDBConnectionException
+ * @throws StatementExecutionException
+ */
+ @Deprecated
+ public void createSchemaTemplate(
+ String name,
+ List<String> schemaNames,
+ List<List<String>> measurements,
+ List<List<TSDataType>> dataTypes,
+ List<List<TSEncoding>> encodings,
+ List<CompressionType> compressors)
+ throws IoTDBConnectionException, StatementExecutionException {
+ List<String> flatMeasurements = new ArrayList<>();
+ List<TSDataType> flatDataTypes = new ArrayList<>();
+ List<TSEncoding> flatEncodings = new ArrayList<>();
+ for (int idx = 0; idx < measurements.size(); idx++) {
+ flatMeasurements.add(measurements.get(idx).get(0));
+ flatDataTypes.add(dataTypes.get(idx).get(0));
+ flatEncodings.add(encodings.get(idx).get(0));
+ }
+ try {
+ createSchemaTemplate(
+ name, flatMeasurements, flatDataTypes, flatEncodings, compressors, false);
+ } catch (IOException e) {
+ throw new StatementExecutionException(e.getMessage());
+ }
+ }
+
+ /**
* @param templateName Template to add aligned measurements.
* @param measurementsPath If measurements get different prefix, or the prefix already exists in
* template but not aligned, throw exception.
diff --git a/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java b/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java
index a8da9ec..d61d86e 100644
--- a/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java
+++ b/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java
@@ -1380,6 +1380,88 @@ public class SessionPool {
}
}
+ /**
+ * Create a template with flat measurements, not tree structured. Need to specify datatype,
+ * encoding and compressor of each measurement, and alignment of these measurements at once.
+ *
+ * @oaram templateName name of template to create
+ * @param measurements flat measurements of the template, cannot contain character dot
+ * @param dataTypes datatype of each measurement in the template
+ * @param encodings encodings of each measurement in the template
+ * @param compressors compression type of each measurement in the template
+ * @param isAligned specify whether these flat measurements are aligned
+ */
+ public void createSchemaTemplate(
+ String templateName,
+ List<String> measurements,
+ List<TSDataType> dataTypes,
+ List<TSEncoding> encodings,
+ List<CompressionType> compressors,
+ boolean isAligned)
+ throws IOException, IoTDBConnectionException, StatementExecutionException {
+ for (int i = 0; i < RETRY; i++) {
+ Session session = getSession();
+ try {
+ session.createSchemaTemplate(
+ templateName, measurements, dataTypes, encodings, compressors, isAligned);
+ putBack(session);
+ return;
+ } catch (IoTDBConnectionException e) {
+ // TException means the connection is broken, remove it and get a new one.
+ logger.warn("createSchemaTemplate failed", e);
+ cleanSessionAndMayThrowConnectionException(session, i, e);
+ } catch (StatementExecutionException | RuntimeException e) {
+ putBack(session);
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * Compatible for rel/0.12, this method will create an unaligned flat template as a result. Notice
+ * that there is no aligned concept in 0.12, so only the first measurement in each nested list
+ * matters.
+ *
+ * @param name name of the template
+ * @param schemaNames it works as a virtual layer inside template in 0.12, and makes no difference
+ * after 0.13
+ * @param measurements the first measurement in each nested list will constitute the final flat
+ * template
+ * @param dataTypes the data type of each measurement, only the first one in each nested list
+ * matters as above
+ * @param encodings the encoding of each measurement, only the first one in each nested list
+ * matters as above
+ * @param compressors the compressor of each measurement
+ * @throws IoTDBConnectionException
+ * @throws StatementExecutionException
+ */
+ @Deprecated
+ public void createSchemaTemplate(
+ String name,
+ List<String> schemaNames,
+ List<List<String>> measurements,
+ List<List<TSDataType>> dataTypes,
+ List<List<TSEncoding>> encodings,
+ List<CompressionType> compressors)
+ throws IoTDBConnectionException, StatementExecutionException {
+ for (int i = 0; i < RETRY; i++) {
+ Session session = getSession();
+ try {
+ session.createSchemaTemplate(
+ name, schemaNames, measurements, dataTypes, encodings, compressors);
+ putBack(session);
+ return;
+ } catch (IoTDBConnectionException e) {
+ // TException means the connection is broken, remove it and get a new one.
+ logger.warn("createSchemaTemplate failed", e);
+ cleanSessionAndMayThrowConnectionException(session, i, e);
+ } catch (StatementExecutionException | RuntimeException e) {
+ putBack(session);
+ throw e;
+ }
+ }
+ }
+
public void addAlignedMeasurementsInTemplate(
String templateName,
List<String> measurementsPath,
diff --git a/session/src/test/java/org/apache/iotdb/session/SessionTest.java b/session/src/test/java/org/apache/iotdb/session/SessionTest.java
index 0112479..6b8c595 100644
--- a/session/src/test/java/org/apache/iotdb/session/SessionTest.java
+++ b/session/src/test/java/org/apache/iotdb/session/SessionTest.java
@@ -44,6 +44,7 @@ import java.time.ZoneId;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import static org.junit.Assert.assertArrayEquals;
@@ -266,6 +267,115 @@ public class SessionTest {
}
@Test
+ public void createSchemaTemplateWithFlatMeasurement()
+ throws IoTDBConnectionException, StatementExecutionException, IOException {
+ session = new Session("127.0.0.1", 6667, "root", "root", ZoneId.of("+05:00"));
+ session.open();
+ String tempName = "flatTemplate";
+ List<String> measurements = Arrays.asList("x", "y", "speed");
+ List<TSDataType> dataTypes =
+ Arrays.asList(TSDataType.FLOAT, TSDataType.FLOAT, TSDataType.DOUBLE);
+ List<TSEncoding> encodings = Arrays.asList(TSEncoding.RLE, TSEncoding.RLE, TSEncoding.GORILLA);
+ List<CompressionType> compressors =
+ Arrays.asList(CompressionType.SNAPPY, CompressionType.SNAPPY, CompressionType.LZ4);
+
+ session.createSchemaTemplate(tempName, measurements, dataTypes, encodings, compressors, true);
+ assertEquals("[x, y, speed]", session.showMeasurementsInTemplate("flatTemplate").toString());
+
+ try {
+ session.addAlignedMeasurementsInTemplate(
+ "flatTemplate",
+ Arrays.asList("temp", "x", "humidity"),
+ dataTypes,
+ encodings,
+ compressors);
+ fail();
+ } catch (Exception e) {
+ assertEquals("315: Path duplicated: x is not a legal path", e.getMessage());
+ }
+ try {
+ session.addUnalignedMeasurementsInTemplate(
+ "flatTemplate",
+ Arrays.asList("temp", "velocity", "heartbeat"),
+ dataTypes,
+ encodings,
+ compressors);
+ fail();
+ } catch (Exception e) {
+ assertEquals(
+ "315: temp is not a legal path, because path already exists and aligned", e.getMessage());
+ }
+
+ session.addAlignedMeasurementsInTemplate(
+ "flatTemplate",
+ Arrays.asList("turbine.temp", "turbine.rounds", "turbine.velocity"),
+ dataTypes,
+ encodings,
+ compressors);
+ assertEquals(6, session.countMeasurementsInTemplate("flatTemplate"));
+ assertEquals(false, session.isMeasurementInTemplate("flatTemplate", "turbine"));
+ assertEquals(true, session.isMeasurementInTemplate("flatTemplate", "speed"));
+ assertEquals(true, session.isPathExistInTemplate("flatTemplate", "turbine"));
+
+ session.deleteNodeInTemplate("flatTemplate", "turbine");
+ assertEquals(3, session.countMeasurementsInTemplate("flatTemplate"));
+ }
+
+ @Test
+ public void testCompatibleInterfaceCreateSchemaTemplate()
+ throws IoTDBConnectionException, StatementExecutionException, IOException {
+ session = new Session("127.0.0.1", 6667, "root", "root", ZoneId.of("+05:00"));
+ session.open();
+
+ List<List<String>> measurementList = new ArrayList<>();
+ measurementList.add(Arrays.asList("s1", "s11", "s12"));
+ measurementList.add(Collections.singletonList("s2"));
+ measurementList.add(Collections.singletonList("s3"));
+
+ List<List<TSDataType>> dataTypeList = new ArrayList<>();
+ dataTypeList.add(Arrays.asList(TSDataType.FLOAT, TSDataType.DOUBLE, TSDataType.INT64));
+ dataTypeList.add(Collections.singletonList(TSDataType.INT64));
+ dataTypeList.add(Collections.singletonList(TSDataType.INT64));
+
+ List<List<TSEncoding>> encodingList = new ArrayList<>();
+ encodingList.add(Arrays.asList(TSEncoding.GORILLA, TSEncoding.RLE, TSEncoding.GORILLA_V1));
+ encodingList.add(Collections.singletonList(TSEncoding.RLE));
+ encodingList.add(Collections.singletonList(TSEncoding.RLE));
+
+ List<CompressionType> compressionTypes = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ compressionTypes.add(CompressionType.SNAPPY);
+ }
+ List<String> schemaNames = new ArrayList<>();
+ schemaNames.add("s1");
+ schemaNames.add("s2");
+ schemaNames.add("s3");
+
+ session.createSchemaTemplate(
+ "cptTemplate", schemaNames, measurementList, dataTypeList, encodingList, compressionTypes);
+ session.setSchemaTemplate("cptTemplate", "root.sg1");
+
+ List<TSDataType> dataTypes =
+ Arrays.asList(TSDataType.FLOAT, TSDataType.FLOAT, TSDataType.DOUBLE);
+ List<TSEncoding> encodings = Arrays.asList(TSEncoding.RLE, TSEncoding.RLE, TSEncoding.GORILLA);
+ List<CompressionType> compressors =
+ Arrays.asList(CompressionType.SNAPPY, CompressionType.SNAPPY, CompressionType.LZ4);
+ try {
+ session.addAlignedMeasurementsInTemplate(
+ "cptTemplate", Arrays.asList("temp", "x", "humidity"), dataTypes, encodings, compressors);
+ // fail();
+ } catch (Exception e) {
+ assertEquals(
+ "315: is not a legal path, because path already exists but not aligned", e.getMessage());
+ }
+
+ assertEquals(3, session.countMeasurementsInTemplate("cptTemplate"));
+ assertEquals(false, session.isPathExistInTemplate("cptTemplate", "s11"));
+ assertEquals(false, session.isPathExistInTemplate("cptTemplate", "s12"));
+ assertEquals(true, session.isMeasurementInTemplate("cptTemplate", "s1"));
+ }
+
+ @Test
public void treeStructuredSchemaTemplateTest()
throws IoTDBConnectionException, StatementExecutionException, IOException {
session = new Session("127.0.0.1", 6667, "root", "root", ZoneId.of("+05:00"));
@@ -308,6 +418,7 @@ public class SessionTest {
try {
session.addAlignedMeasurementsInTemplate(
"treeTemplate", measurementPaths, dataTypes, encodings, compressionTypeList);
+ fail();
} catch (Exception e) {
assertEquals(
"315: GPS is not a legal path, because path already exists but not aligned",
@@ -320,6 +431,7 @@ public class SessionTest {
try {
session.addUnalignedMeasurementInTemplate(
"treeTemplate", "GPS.X", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.SNAPPY);
+ fail();
} catch (Exception e) {
assertEquals("315: Path duplicated: GPS.X is not a legal path", e.getMessage());
}
@@ -385,8 +497,16 @@ public class SessionTest {
Template template = new Template("emptyTemplate");
session.createSchemaTemplate(template);
- session.addAlignedMeasurementInTemplate(
+ session.addUnalignedMeasurementInTemplate(
"emptyTemplate", "speed", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.SNAPPY);
+ try {
+ session.addAlignedMeasurementInTemplate(
+ "emptyTemplate", "speed2", TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.SNAPPY);
+ fail();
+ } catch (Exception e) {
+ assertEquals(
+ "315: is not a legal path, because path already exists but not aligned", e.getMessage());
+ }
}
@Test
diff --git a/session/src/test/java/org/apache/iotdb/session/template/TemplateUT.java b/session/src/test/java/org/apache/iotdb/session/template/TemplateUT.java
index a244b83..b947f2f 100644
--- a/session/src/test/java/org/apache/iotdb/session/template/TemplateUT.java
+++ b/session/src/test/java/org/apache/iotdb/session/template/TemplateUT.java
@@ -37,6 +37,8 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.time.ZoneId;
+import java.util.Arrays;
+import java.util.List;
import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertEquals;
@@ -50,6 +52,8 @@ public class TemplateUT {
System.setProperty(IoTDBConstant.IOTDB_CONF, "src/test/resources/");
EnvironmentUtils.closeStatMonitor();
EnvironmentUtils.envSetUp();
+ session = new Session("127.0.0.1", 6667, "root", "root", ZoneId.of("+05:00"));
+ session.open();
}
@After
@@ -59,10 +63,29 @@ public class TemplateUT {
}
@Test
+ public void testCreateFlatTemplate()
+ throws IOException, StatementExecutionException, IoTDBConnectionException {
+ String tempName = "flatTemplate";
+ List<String> measurements = Arrays.asList("x", "y", "speed");
+ List<TSDataType> dataTypes =
+ Arrays.asList(TSDataType.FLOAT, TSDataType.FLOAT, TSDataType.DOUBLE);
+ List<TSEncoding> encodings = Arrays.asList(TSEncoding.RLE, TSEncoding.RLE, TSEncoding.GORILLA);
+ List<CompressionType> compressors =
+ Arrays.asList(CompressionType.SNAPPY, CompressionType.SNAPPY, CompressionType.LZ4);
+
+ session.createSchemaTemplate(tempName, measurements, dataTypes, encodings, compressors, true);
+ session.setSchemaTemplate("flatTemplate", "root.sg.d0");
+ try {
+ session.setSchemaTemplate("flatTemplate", "root.sg.d0");
+ fail();
+ } catch (StatementExecutionException e) {
+ assertEquals("303: Template already exists on root.sg.d0", e.getMessage());
+ }
+ }
+
+ @Test
public void testTemplateTree()
throws IOException, MetadataException, StatementExecutionException, IoTDBConnectionException {
- session = new Session("127.0.0.1", 6667, "root", "root", ZoneId.of("+05:00"));
- session.open();
Template sessionTemplate = new Template("treeTemplate", true);
TemplateNode iNodeGPS = new InternalNode("GPS", false);
TemplateNode iNodeV = new InternalNode("vehicle", true);