You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2021/05/13 01:15:04 UTC
[iotdb] branch master updated: [IOTDB-1371] Fix NPE when creating
aligned timeseries and inserting with mismatched data type (#3164)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 4852dad [IOTDB-1371] Fix NPE when creating aligned timeseries and inserting with mismatched data type (#3164)
4852dad is described below
commit 4852dad6a39a57abc49998f54fea6473b6abe3f9
Author: Zesong Sun <sz...@mails.tsinghua.edu.cn>
AuthorDate: Thu May 13 09:14:36 2021 +0800
[IOTDB-1371] Fix NPE when creating aligned timeseries and inserting with mismatched data type (#3164)
---
.../org/apache/iotdb/db/metadata/MManager.java | 64 ++++--
.../iotdb/db/metadata/MManagerBasicTest.java | 225 +++++++++++++++++++++
.../apache/iotdb/db/sink/LocalIoTDBSinkTest.java | 3 +-
3 files changed, 278 insertions(+), 14 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index a91d333..53fc126 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -71,6 +71,7 @@ import org.apache.iotdb.db.rescon.PrimitiveArrayManager;
import org.apache.iotdb.db.utils.RandomDeleteCache;
import org.apache.iotdb.db.utils.SchemaUtils;
import org.apache.iotdb.db.utils.TestOnly;
+import org.apache.iotdb.db.utils.TypeInferenceUtils;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.exception.cache.CacheException;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
@@ -2196,9 +2197,8 @@ public class MManager {
Arrays.asList(measurement.replace("(", "").replace(")", "").split(","));
if (measurements.size() == 1) {
internalCreateTimeseries(
- deviceId.concatNode(measurement), plan.getDataTypes()[loc++]);
+ deviceId.concatNode(measurement), plan.getDataTypes()[loc]);
measurementMNode = (MeasurementMNode) deviceMNode.left.getChild(measurement);
-
} else {
int curLoc = loc;
List<TSDataType> dataTypes = new ArrayList<>();
@@ -2221,7 +2221,8 @@ public class MManager {
// check type is match
boolean mismatch = false;
- TSDataType insertDataType = null;
+ TSDataType insertDataType;
+ DataTypeMismatchException mismatchException = null;
if (plan instanceof InsertRowPlan || plan instanceof InsertTabletPlan) {
if (measurementList[i].contains("(") && measurementList[i].contains(",")) {
for (int j = 0; j < measurementList[i].split(",").length; j++) {
@@ -2233,26 +2234,46 @@ public class MManager {
}
if (dataTypeInNode != insertDataType) {
mismatch = true;
- insertDataType = dataTypeInNode;
+ logger.warn(
+ "DataType mismatch, Insert measurement {} in {} type {}, metadata tree type {}",
+ measurementMNode.getSchema().getValueMeasurementIdList().get(j),
+ measurementList[i],
+ insertDataType,
+ dataTypeInNode);
+ mismatchException =
+ new DataTypeMismatchException(
+ measurementList[i], insertDataType, dataTypeInNode);
break;
}
loc++;
}
} else {
- insertDataType = measurementMNode.getSchema().getType();
+ if (plan instanceof InsertRowPlan) {
+ if (!((InsertRowPlan) plan).isNeedInferType()) {
+ // only when InsertRowPlan's values is object[], we should check type
+ insertDataType = getTypeInLoc(plan, loc);
+ } else {
+ insertDataType = measurementMNode.getSchema().getType();
+ }
+ } else {
+ insertDataType = getTypeInLoc(plan, loc);
+ }
mismatch = measurementMNode.getSchema().getType() != insertDataType;
+ if (mismatch) {
+ logger.warn(
+ "DataType mismatch, Insert measurement {} type {}, metadata tree type {}",
+ measurementList[i],
+ insertDataType,
+ measurementMNode.getSchema().getType());
+ mismatchException =
+ new DataTypeMismatchException(
+ measurementList[i], insertDataType, measurementMNode.getSchema().getType());
+ }
+ loc++;
}
}
if (mismatch) {
- logger.warn(
- "DataType mismatch, Insert measurement {} type {}, metadata tree type {}",
- measurementList[i],
- insertDataType,
- measurementMNode.getSchema().getType());
- DataTypeMismatchException mismatchException =
- new DataTypeMismatchException(
- measurementList[i], insertDataType, measurementMNode.getSchema().getType());
if (!config.isEnablePartialInsert()) {
throw mismatchException;
} else {
@@ -2284,6 +2305,23 @@ public class MManager {
return deviceMNode.left;
}
+ /** get dataType of plan, in loc measurements only support InsertRowPlan and InsertTabletPlan */
+ private TSDataType getTypeInLoc(InsertPlan plan, int loc) throws MetadataException {
+ TSDataType dataType;
+ if (plan instanceof InsertRowPlan) {
+ InsertRowPlan tPlan = (InsertRowPlan) plan;
+ dataType =
+ TypeInferenceUtils.getPredictedDataType(tPlan.getValues()[loc], tPlan.isNeedInferType());
+ } else if (plan instanceof InsertTabletPlan) {
+ dataType = (plan).getDataTypes()[loc];
+ } else {
+ throw new MetadataException(
+ String.format(
+ "Only support insert and insertTablet, plan is [%s]", plan.getOperatorType()));
+ }
+ return dataType;
+ }
+
public MNode getMNode(MNode deviceMNode, String measurementName) {
return deviceMNode.getChild(measurementName);
}
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
index 1e86e89..a3b0087 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java
@@ -22,8 +22,10 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.metadata.mnode.MNode;
+import org.apache.iotdb.db.metadata.mnode.MeasurementMNode;
import org.apache.iotdb.db.metadata.template.Template;
import org.apache.iotdb.db.qp.physical.crud.CreateTemplatePlan;
+import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
import org.apache.iotdb.db.qp.physical.crud.SetDeviceTemplatePlan;
import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan;
@@ -55,6 +57,7 @@ import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -1334,4 +1337,226 @@ public class MManagerBasicTest {
assertTrue(manager.isPathExist(new PartialPath("root.group-with-hyphen")));
}
+
+ @Test
+ public void testCreateAlignedTimeseriesAndInsertWithMismatchDataType() {
+ MManager manager = IoTDB.metaManager;
+ try {
+ manager.setStorageGroup(new PartialPath("root.laptop"));
+ manager.createAlignedTimeSeries(
+ new PartialPath("root.laptop.d1"),
+ Arrays.asList("s1", "s2", "s3"),
+ Arrays.asList(
+ TSDataType.valueOf("FLOAT"),
+ TSDataType.valueOf("INT64"),
+ TSDataType.valueOf("INT32")),
+ Arrays.asList(
+ TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE")),
+ compressionType);
+
+ // construct an insertRowPlan with mismatched data type
+ long time = 1L;
+ TSDataType[] dataTypes =
+ new TSDataType[] {TSDataType.FLOAT, TSDataType.DOUBLE, TSDataType.INT32};
+
+ String[] columns = new String[3];
+ columns[0] = 2.0 + "";
+ columns[1] = 10000 + "";
+ columns[2] = 100 + "";
+
+ InsertRowPlan insertRowPlan =
+ new InsertRowPlan(
+ new PartialPath("root.laptop.d1"),
+ time,
+ new String[] {"(s1,s2,s3)"},
+ dataTypes,
+ columns);
+ insertRowPlan.setMeasurementMNodes(
+ new MeasurementMNode[insertRowPlan.getMeasurements().length]);
+
+ // call getSeriesSchemasAndReadLockDevice
+ MNode mNode = manager.getSeriesSchemasAndReadLockDevice(insertRowPlan);
+ assertEquals(4, mNode.getMeasurementMNodeCount());
+ assertNull(insertRowPlan.getMeasurementMNodes()[0]);
+ assertEquals(1, insertRowPlan.getFailedMeasurementNumber());
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCreateTimeseriesAndInsertWithMismatchDataType() {
+ MManager manager = IoTDB.metaManager;
+ try {
+ manager.setStorageGroup(new PartialPath("root.laptop"));
+ manager.createTimeseries(
+ new PartialPath("root.laptop.d1.s0"),
+ TSDataType.valueOf("INT32"),
+ TSEncoding.valueOf("RLE"),
+ compressionType,
+ Collections.emptyMap());
+
+ // construct an insertRowPlan with mismatched data type
+ long time = 1L;
+ TSDataType[] dataTypes = new TSDataType[] {TSDataType.FLOAT};
+
+ String[] columns = new String[1];
+ columns[0] = 2.0 + "";
+
+ InsertRowPlan insertRowPlan =
+ new InsertRowPlan(
+ new PartialPath("root.laptop.d1"), time, new String[] {"s0"}, dataTypes, columns);
+ insertRowPlan.setMeasurementMNodes(
+ new MeasurementMNode[insertRowPlan.getMeasurements().length]);
+
+ // call getSeriesSchemasAndReadLockDevice
+ MNode mNode = manager.getSeriesSchemasAndReadLockDevice(insertRowPlan);
+ assertEquals(1, mNode.getMeasurementMNodeCount());
+ assertNull(insertRowPlan.getMeasurementMNodes()[0]);
+ assertEquals(1, insertRowPlan.getFailedMeasurementNumber());
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCreateMixedTimeseriesAndInsertWithMismatchDataType() {
+ MManager manager = IoTDB.metaManager;
+ try {
+ manager.setStorageGroup(new PartialPath("root.laptop"));
+ manager.createTimeseries(
+ new PartialPath("root.laptop.d1.s0"),
+ TSDataType.valueOf("INT32"),
+ TSEncoding.valueOf("RLE"),
+ compressionType,
+ Collections.emptyMap());
+
+ manager.createAlignedTimeSeries(
+ new PartialPath("root.laptop.d1"),
+ Arrays.asList("s1", "s2", "s3"),
+ Arrays.asList(
+ TSDataType.valueOf("FLOAT"),
+ TSDataType.valueOf("INT64"),
+ TSDataType.valueOf("INT32")),
+ Arrays.asList(
+ TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE")),
+ compressionType);
+
+ // construct an insertRowPlan with correct data type
+ long time = 1L;
+ TSDataType[] dataTypes =
+ new TSDataType[] {TSDataType.INT32, TSDataType.FLOAT, TSDataType.INT64, TSDataType.INT32};
+
+ String[] columns = new String[4];
+ columns[0] = 100 + "";
+ columns[1] = 2.0 + "";
+ columns[2] = 10000 + "";
+ columns[3] = 200 + "";
+
+ InsertRowPlan insertRowPlan =
+ new InsertRowPlan(
+ new PartialPath("root.laptop.d1"),
+ time,
+ new String[] {"s0", "(s1,s2,s3)"},
+ dataTypes,
+ columns);
+ insertRowPlan.setMeasurementMNodes(
+ new MeasurementMNode[insertRowPlan.getMeasurements().length]);
+
+ // call getSeriesSchemasAndReadLockDevice
+ MNode mNode = manager.getSeriesSchemasAndReadLockDevice(insertRowPlan);
+ assertEquals(5, mNode.getMeasurementMNodeCount());
+ assertNotNull(insertRowPlan.getMeasurementMNodes()[0]);
+ assertNotNull(insertRowPlan.getMeasurementMNodes()[1]);
+ assertEquals(0, insertRowPlan.getFailedMeasurementNumber());
+
+ // construct an insertRowPlan with mismatched data type in non-aligned timeseries
+ time = 2L;
+ dataTypes =
+ new TSDataType[] {TSDataType.FLOAT, TSDataType.FLOAT, TSDataType.INT64, TSDataType.INT32};
+
+ columns[0] = 2.0 + "";
+ columns[1] = 2.0 + "";
+ columns[2] = 10000 + "";
+ columns[3] = 200 + "";
+
+ insertRowPlan =
+ new InsertRowPlan(
+ new PartialPath("root.laptop.d1"),
+ time,
+ new String[] {"s0", "(s1,s2,s3)"},
+ dataTypes,
+ columns);
+ insertRowPlan.setMeasurementMNodes(
+ new MeasurementMNode[insertRowPlan.getMeasurements().length]);
+
+ // call getSeriesSchemasAndReadLockDevice
+ mNode = manager.getSeriesSchemasAndReadLockDevice(insertRowPlan);
+ assertEquals(5, mNode.getMeasurementMNodeCount());
+ assertNull(insertRowPlan.getMeasurementMNodes()[0]);
+ assertNotNull(insertRowPlan.getMeasurementMNodes()[1]);
+ assertEquals(1, insertRowPlan.getFailedMeasurementNumber());
+
+ // construct an insertRowPlan with mismatched data type in aligned timeseries
+ time = 3L;
+ dataTypes =
+ new TSDataType[] {TSDataType.INT32, TSDataType.FLOAT, TSDataType.INT32, TSDataType.INT32};
+
+ columns[0] = 100 + "";
+ columns[1] = 2.0 + "";
+ columns[2] = 200 + "";
+ columns[3] = 300 + "";
+
+ insertRowPlan =
+ new InsertRowPlan(
+ new PartialPath("root.laptop.d1"),
+ time,
+ new String[] {"s0", "(s1,s2,s3)"},
+ dataTypes,
+ columns);
+ insertRowPlan.setMeasurementMNodes(
+ new MeasurementMNode[insertRowPlan.getMeasurements().length]);
+
+ // call getSeriesSchemasAndReadLockDevice
+ mNode = manager.getSeriesSchemasAndReadLockDevice(insertRowPlan);
+ assertEquals(5, mNode.getMeasurementMNodeCount());
+ assertNotNull(insertRowPlan.getMeasurementMNodes()[0]);
+ assertNull(insertRowPlan.getMeasurementMNodes()[1]);
+ assertEquals(1, insertRowPlan.getFailedMeasurementNumber());
+
+ // construct an insertRowPlan with mismatched data type in both timeseries
+ time = 4L;
+ dataTypes =
+ new TSDataType[] {TSDataType.FLOAT, TSDataType.FLOAT, TSDataType.INT32, TSDataType.INT32};
+
+ columns[0] = 1.0 + "";
+ columns[1] = 2.0 + "";
+ columns[2] = 200 + "";
+ columns[3] = 300 + "";
+
+ insertRowPlan =
+ new InsertRowPlan(
+ new PartialPath("root.laptop.d1"),
+ time,
+ new String[] {"s0", "(s1,s2,s3)"},
+ dataTypes,
+ columns);
+ insertRowPlan.setMeasurementMNodes(
+ new MeasurementMNode[insertRowPlan.getMeasurements().length]);
+
+ // call getSeriesSchemasAndReadLockDevice
+ mNode = manager.getSeriesSchemasAndReadLockDevice(insertRowPlan);
+ assertEquals(5, mNode.getMeasurementMNodeCount());
+ assertNull(insertRowPlan.getMeasurementMNodes()[0]);
+ assertNull(insertRowPlan.getMeasurementMNodes()[1]);
+ assertEquals(2, insertRowPlan.getFailedMeasurementNumber());
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
}
diff --git a/server/src/test/java/org/apache/iotdb/db/sink/LocalIoTDBSinkTest.java b/server/src/test/java/org/apache/iotdb/db/sink/LocalIoTDBSinkTest.java
index adcd434..0261cf7 100644
--- a/server/src/test/java/org/apache/iotdb/db/sink/LocalIoTDBSinkTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/sink/LocalIoTDBSinkTest.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.sink;
+import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.sink.local.LocalIoTDBConfiguration;
import org.apache.iotdb.db.sink.local.LocalIoTDBEvent;
import org.apache.iotdb.db.sink.local.LocalIoTDBHandler;
@@ -191,7 +192,7 @@ public class LocalIoTDBSinkTest {
}
}
- @Test(expected = ClassCastException.class)
+ @Test(expected = QueryProcessException.class)
public void onEventWithWrongType1() throws Exception {
LocalIoTDBHandler localIoTDBHandler = new LocalIoTDBHandler();
localIoTDBHandler.open(