You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/06/05 13:22:06 UTC
[incubator-iotdb] branch master updated: [IOTDB-745] fix partial
write and sync close bug (#1318)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9d5fc6a [IOTDB-745] fix partial write and sync close bug (#1318)
9d5fc6a is described below
commit 9d5fc6aa0efe0232710ac1910aaf67f845861452
Author: Jialin Qiao <qj...@mails.tsinghua.edu.cn>
AuthorDate: Fri Jun 5 21:21:53 2020 +0800
[IOTDB-745] fix partial write and sync close bug (#1318)
* fix partial write and sync close bug
---
.../iotdb/db/engine/memtable/AbstractMemTable.java | 5 +++-
.../db/engine/storagegroup/TsFileProcessor.java | 4 ++-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 2 +-
.../iotdb/db/qp/physical/crud/InsertPlan.java | 4 +++
.../iotdb/db/integration/IoTDBSimpleQueryIT.java | 33 +++++++++++++++++++++-
5 files changed, 44 insertions(+), 4 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index 4629023..429124a 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -111,7 +111,7 @@ public abstract class AbstractMemTable implements IMemTable {
insertPlan.getSchemas()[i], insertPlan.getTime(), value);
}
- totalPointsNum += insertPlan.getValues().length;
+ totalPointsNum += insertPlan.getMeasurements().length - insertPlan.getFailedMeasurementNumber();
}
@Override
@@ -171,6 +171,9 @@ public abstract class AbstractMemTable implements IMemTable {
@Override
public boolean reachTotalPointNumThreshold() {
+ if (totalPointsNum == 0) {
+ return false;
+ }
return totalPointsNum >= totalPointsNumThreshold;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 31c76f2..6d90ed8 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -378,7 +378,9 @@ public class TsFileProcessor {
//we have to add the memtable into flushingList first and then set the shouldClose tag.
// see https://issues.apache.org/jira/browse/IOTDB-510
- IMemTable tmpMemTable = workMemTable == null ? new NotifyFlushMemTable() : workMemTable;
+ IMemTable tmpMemTable = workMemTable == null || workMemTable.memSize() == 0
+ ? new NotifyFlushMemTable()
+ : workMemTable;
try {
addAMemtableIntoFlushingList(tmpMemTable);
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index 5c1123c..831d269 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -905,7 +905,7 @@ public class PlanExecutor implements IPlanExecutor {
checkType(insertPlan, i, measurementNode.getSchema().getType());
}
} catch (MetadataException e) {
- logger.warn("meet error when check {}.{}", deviceId, measurement, e);
+ logger.warn("meet error when check {}.{}, message: {}", deviceId, measurement, e.getMessage());
if (enablePartialInsert) {
insertPlan.markMeasurementInsertionFailed(i);
} else {
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java
index 5bb84b3..f617a25 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java
@@ -358,6 +358,10 @@ public class InsertPlan extends PhysicalPlan {
return failedMeasurements;
}
+ public int getFailedMeasurementNumber() {
+ return failedMeasurements == null ? 0 : failedMeasurements.size();
+ }
+
public TSDataType[] getTypes() {
return types;
}
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSimpleQueryIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSimpleQueryIT.java
index a4deafc..05bfd78 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSimpleQueryIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSimpleQueryIT.java
@@ -27,6 +27,7 @@ import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.jdbc.IoTDBSQLException;
@@ -183,7 +184,6 @@ public class IoTDBSimpleQueryIT {
statement.execute("CREATE TIMESERIES root.sg1.d0.s0 WITH DATATYPE=INT32,ENCODING=PLAIN");
statement.execute("CREATE TIMESERIES root.sg1.d0.s1 WITH DATATYPE=INT32,ENCODING=PLAIN");
- // seq chunk : [1,10]
try {
statement.execute("INSERT INTO root.sg1.d0(timestamp, s0, s1) VALUES (1, 1, 2.2)");
fail();
@@ -200,6 +200,37 @@ public class IoTDBSimpleQueryIT {
}
}
+
+ @Test
+ public void testPartialInsertionAllFailed() throws SQLException, ClassNotFoundException {
+ Class.forName(Config.JDBC_DRIVER_NAME);
+
+ boolean autoCreateSchemaEnabled = IoTDBDescriptor.getInstance().getConfig()
+ .isAutoCreateSchemaEnabled();
+ boolean enablePartialInsert = IoTDBDescriptor.getInstance().getConfig().isEnablePartialInsert();
+
+ try(Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/",
+ "root", "root");
+ Statement statement = connection.createStatement()){
+ IoTDBDescriptor.getInstance().getConfig().setAutoCreateSchemaEnabled(false);
+ IoTDBDescriptor.getInstance().getConfig().setEnablePartialInsert(true);
+
+ statement.execute("SET STORAGE GROUP TO root.sg1");
+
+ try {
+ statement.execute("INSERT INTO root.sg1(timestamp, s0) VALUES (1, 1)");
+ fail();
+ } catch (IoTDBSQLException e) {
+ assertTrue(e.getMessage().contains("s0"));
+ }
+ }
+
+ IoTDBDescriptor.getInstance().getConfig().setEnablePartialInsert(enablePartialInsert);
+ IoTDBDescriptor.getInstance().getConfig().setAutoCreateSchemaEnabled(autoCreateSchemaEnabled);
+
+ }
+
@Test
public void testOverlappedPagesMerge() throws SQLException, ClassNotFoundException {
Class.forName(Config.JDBC_DRIVER_NAME);