You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ne...@apache.org on 2021/08/31 03:26:20 UTC
[iotdb] branch master updated: [IOTDB-1583] Raft log failed to be
committed in cluster version (#3832)
This is an automated email from the ASF dual-hosted git repository.
neuyilan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new bc696e9 [IOTDB-1583] Raft log failed to be committed in cluster version (#3832)
bc696e9 is described below
commit bc696e9586faada13c9715135a7c1563c792ac0a
Author: lisijia <44...@users.noreply.github.com>
AuthorDate: Tue Aug 31 11:23:18 2021 +0800
[IOTDB-1583] Raft log failed to be committed in cluster version (#3832)
---
.../iotdb/cluster/server/member/RaftMember.java | 17 ++++++++++++++
.../db/writelog/node/ExclusiveWriteLogNode.java | 3 +++
.../apache/iotdb/db/writelog/WriteLogNodeTest.java | 26 ++++++++++++++++++++++
3 files changed, 46 insertions(+)
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java
index a5a58b0..c2c8672 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java
@@ -1032,6 +1032,15 @@ public abstract class RaftMember {
log.setCurrLogTerm(getTerm().get());
log.setCurrLogIndex(logManager.getLastLogIndex() + 1);
+ // if a single log exceeds the threshold
+ // we need to return error code to the client as in server mode
+ if (log.serialize().capacity() + Integer.BYTES
+ >= ClusterDescriptor.getInstance().getConfig().getRaftLogBufferSize()) {
+ logger.error(
+ "Log cannot fit into buffer, please increase raft_log_buffer_size;"
+ + "or reduce the size of requests you send.");
+ return StatusUtils.INTERNAL_ERROR;
+ }
logManager.append(log);
}
Timer.Statistic.RAFT_SENDER_APPEND_LOG.calOperationCostTimeFromStart(startTime);
@@ -1077,6 +1086,14 @@ public abstract class RaftMember {
log.setCurrLogIndex(logManager.getLastLogIndex() + 1);
startTime = Timer.Statistic.RAFT_SENDER_APPEND_LOG_V2.getOperationStartTime();
+ // just like processPlanLocally,we need to check the size of log
+ if (log.serialize().capacity() + Integer.BYTES
+ >= ClusterDescriptor.getInstance().getConfig().getRaftLogBufferSize()) {
+ logger.error(
+ "Log cannot fit into buffer, please increase raft_log_buffer_size;"
+ + "or reduce the size of requests you send.");
+ return StatusUtils.INTERNAL_ERROR;
+ }
// logDispatcher will serialize log, and set log size, and we will use the size after it
logManager.append(log);
Timer.Statistic.RAFT_SENDER_APPEND_LOG_V2.calOperationCostTimeFromStart(startTime);
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java b/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
index 56fafe2..e11a11e 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
@@ -113,6 +113,9 @@ public class ExclusiveWriteLogNode implements WriteLogNode, Comparable<Exclusive
sync();
}
} catch (BufferOverflowException e) {
+ // if the size of a single plan bigger than logBufferWorking
+ // we need to clear the buffer to drop something wrong that has written.
+ logBufferWorking.clear();
throw new IOException("Log cannot fit into the buffer, please increase wal_buffer_size", e);
} finally {
lock.unlock();
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/WriteLogNodeTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/WriteLogNodeTest.java
index 2767813..fd7cd0d 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/WriteLogNodeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/WriteLogNodeTest.java
@@ -316,6 +316,32 @@ public class WriteLogNodeTest {
}
assertTrue(caught);
+ // if last insertplan failed by overflow,it can not take affect the next insertplan
+ InsertRowPlan bwInsertPlan2 =
+ new InsertRowPlan(
+ new PartialPath("root.logTestDevice.oversize"),
+ 100,
+ new String[] {"s1", "s2", "s3", "s4"},
+ new TSDataType[] {
+ TSDataType.DOUBLE, TSDataType.INT64, TSDataType.TEXT, TSDataType.BOOLEAN
+ },
+ // try to apply a insertplan whose size will fill the entire logBufferWorking
+ new String[] {
+ "1.0",
+ "15",
+ new String(
+ new char
+ [(IoTDBDescriptor.getInstance().getConfig().getWalBufferSize() / 2) - 109]),
+ "false"
+ });
+ caught = false;
+ try {
+ logNode.write(bwInsertPlan2);
+ } catch (IOException e) {
+ caught = true;
+ }
+ assertFalse(caught);
+
ByteBuffer[] array = logNode.delete();
for (ByteBuffer byteBuffer : array) {
MmapUtil.clean((MappedByteBuffer) byteBuffer);