You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2022/03/03 08:20:01 UTC
[iotdb] branch expr_plus updated: add LogRelay
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch expr_plus
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/expr_plus by this push:
new 5033ff9 add LogRelay
5033ff9 is described below
commit 5033ff9c57f3002596759178004e28634882c914
Author: jt <jt...@163.com>
AuthorDate: Thu Mar 3 16:19:17 2022 +0800
add LogRelay
---
.../resources/conf/iotdb-cluster.properties | 3 ++
.../iotdb/cluster/client/sync/SyncClientPool.java | 11 ++++-
.../apache/iotdb/cluster/config/ClusterConfig.java | 10 +++++
.../iotdb/cluster/config/ClusterDescriptor.java | 15 +++++++
.../org/apache/iotdb/cluster/expr/ExprMember.java | 16 +++++--
.../org/apache/iotdb/cluster/expr/ExprServer.java | 2 +-
.../iotdb/cluster/log/IndirectLogDispatcher.java | 6 +--
.../java/org/apache/iotdb/cluster/log/Log.java | 2 +-
.../apache/iotdb/cluster/log/LogDispatcher.java | 22 +++++++---
.../cluster/log/manage/CommittedEntryManager.java | 11 +++++
.../iotdb/cluster/log/manage/RaftLogManager.java | 13 +++---
.../log/manage/UnCommittedEntryManager.java | 14 ++++++
.../log/sequencing/AsynchronousSequencer.java | 9 +++-
.../log/sequencing/SynchronousSequencer.java | 6 ++-
.../cluster/log/snapshot/MetaSimpleSnapshot.java | 4 ++
.../handlers/caller/AppendNodeEntryHandler.java | 8 ----
.../server/handlers/caller/HeartbeatHandler.java | 6 +--
.../server/handlers/caller/LogCatchUpHandler.java | 16 ++++++-
.../handlers/caller/LogCatchUpInBatchHandler.java | 5 ++-
.../cluster/server/member/DataGroupMember.java | 6 ++-
.../cluster/server/member/MetaGroupMember.java | 8 +++-
.../iotdb/cluster/server/member/RaftMember.java | 50 ++++++++++++++--------
.../apache/iotdb/cluster/server/monitor/Timer.java | 2 +-
23 files changed, 181 insertions(+), 64 deletions(-)
diff --git a/cluster/src/assembly/resources/conf/iotdb-cluster.properties b/cluster/src/assembly/resources/conf/iotdb-cluster.properties
index 2b01d5b..eb3b937 100644
--- a/cluster/src/assembly/resources/conf/iotdb-cluster.properties
+++ b/cluster/src/assembly/resources/conf/iotdb-cluster.properties
@@ -192,3 +192,6 @@ multi_raft_factor=1
# we need to wait so much time for other connections to be released until timeout,
# or a new connection will be created.
# wait_client_timeout_ms=5000
+
+
+use_indirect_broadcasting=true
\ No newline at end of file
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/client/sync/SyncClientPool.java b/cluster/src/main/java/org/apache/iotdb/cluster/client/sync/SyncClientPool.java
index 2c279c0..6d13be3 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/client/sync/SyncClientPool.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/client/sync/SyncClientPool.java
@@ -30,6 +30,8 @@ import org.apache.thrift.transport.TTransportException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.net.ConnectException;
+import java.net.SocketTimeoutException;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.Map;
@@ -90,13 +92,18 @@ public class SyncClientPool {
try {
client = syncClientFactory.getSyncClient(clusterNode, this);
} catch (TTransportException e) {
- logger.error("Cannot open transport for client {}", node, e);
+ if (!(e.getCause() instanceof ConnectException
+ || e.getCause() instanceof SocketTimeoutException)) {
+ logger.error("Cannot open transport for client {}", node, e);
+ }
return null;
}
nodeClientNumMap.compute(
clusterNode,
(n, oldValue) -> {
- if (oldValue == null) return 1;
+ if (oldValue == null) {
+ return 1;
+ }
return oldValue + 1;
});
return client;
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java b/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java
index 420f7ca..a5eb928 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java
@@ -185,6 +185,8 @@ public class ClusterConfig {
private boolean useAsyncSequencing = true;
+ private int relaySenderNum = 8;
+
/**
* create a clusterConfig class. The internalIP will be set according to the server's hostname. If
* there is something error for getting the ip of the hostname, then set the internalIp as
@@ -559,4 +561,12 @@ public class ClusterConfig {
public void setUseAsyncSequencing(boolean useAsyncSequencing) {
this.useAsyncSequencing = useAsyncSequencing;
}
+
+ public int getRelaySenderNum() {
+ return relaySenderNum;
+ }
+
+ public void setRelaySenderNum(int relaySenderNum) {
+ this.relaySenderNum = relaySenderNum;
+ }
}
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterDescriptor.java b/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterDescriptor.java
index f70f08f..890bbf8 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterDescriptor.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterDescriptor.java
@@ -323,6 +323,21 @@ public class ClusterDescriptor {
properties.getProperty(
"wait_client_timeout_ms", String.valueOf(config.getWaitClientTimeoutMS()))));
+ config.setUseAsyncSequencing(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "use_async_sequencing", String.valueOf(config.isUseAsyncSequencing()))));
+
+ config.setUseIndirectBroadcasting(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "use_indirect_broadcasting", String.valueOf(config.isUseIndirectBroadcasting()))));
+
+ config.setRelaySenderNum(
+ Integer.parseInt(
+ properties.getProperty(
+ "relay_sender_number", String.valueOf(config.getRelaySenderNum()))));
+
String consistencyLevel = properties.getProperty("consistency_level");
if (consistencyLevel != null) {
config.setConsistencyLevel(ConsistencyLevel.getConsistencyLevel(consistencyLevel));
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/expr/ExprMember.java b/cluster/src/main/java/org/apache/iotdb/cluster/expr/ExprMember.java
index bcc26ce..2f92a88 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/expr/ExprMember.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/expr/ExprMember.java
@@ -79,8 +79,7 @@ public class ExprMember extends MetaGroupMember {
public ExprMember(TProtocolFactory factory, Node thisNode, Coordinator coordinator)
throws QueryProcessException {
super(factory, thisNode, coordinator);
- this.firstPosPrevIndex = logManager.getLastLogIndex();
- this.prevTerms[0] = logManager.getLastLogTerm();
+ resetWindow(logManager.getLastLogIndex(), logManager.getLastLogTerm());
bypassPool = Executors.newFixedThreadPool(LogDispatcher.bindingThreadNum);
}
@@ -89,10 +88,19 @@ public class ExprMember extends MetaGroupMember {
// do not start data groups in such experiments
}
+ public void resetWindow(long firstPosPrevIndex, long firstPosPrevTerm) {
+ this.firstPosPrevIndex = firstPosPrevIndex;
+ this.prevTerms[0] = firstPosPrevTerm;
+ logWindow = new Log[windowCapacity];
+ }
+
@Override
public TSStatus executeNonQueryPlan(PhysicalPlan plan) {
try {
if (bypassRaft) {
+ if (allNodes.size() == 1) {
+ return StatusUtils.OK;
+ }
CountDownLatch latch = new CountDownLatch(allNodes.size() / 2);
int bufferSize = 4096;
if (plan instanceof DummyPlan && !((DummyPlan) plan).isNeedForward()) {
@@ -130,7 +138,9 @@ public class ExprMember extends MetaGroupMember {
}));
}
}
- latch.await();
+ if (allNodes.size() > 1) {
+ latch.await();
+ }
return StatusUtils.OK;
}
long startTime = Timer.Statistic.META_GROUP_MEMBER_EXECUTE_NON_QUERY.getOperationStartTime();
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/expr/ExprServer.java b/cluster/src/main/java/org/apache/iotdb/cluster/expr/ExprServer.java
index ab2468e..d245c23 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/expr/ExprServer.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/expr/ExprServer.java
@@ -90,7 +90,7 @@ public class ExprServer extends MetaClusterServer {
ClusterDescriptor.getInstance().getConfig().setMaxClientPerNodePerMember(50000);
// ClusterDescriptor.getInstance().getConfig().setUseBatchInLogCatchUp(false);
RaftMember.USE_LOG_DISPATCHER = true;
- RaftMember.USE_INDIRECT_LOG_DISPATCHER = useIndirectDispatcher;
+ ClusterDescriptor.getInstance().getConfig().setUseIndirectBroadcasting(useIndirectDispatcher);
LogDispatcher.bindingThreadNum = dispatcherThreadNum;
LogDispatcher.maxBatchSize = maxBatchSize;
ExprMember.bypassRaft = bypassRaft;
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/IndirectLogDispatcher.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/IndirectLogDispatcher.java
index 767736d..cc05753 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/IndirectLogDispatcher.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/IndirectLogDispatcher.java
@@ -87,9 +87,9 @@ public class IndirectLogDispatcher extends LogDispatcher {
}
}
- // for (Node node : directToIndirectFollowerMap.keySet()) {
- // nodeLogQueues.add(createQueueAndBindingThread(node));
- // }
+ for (Node node : directToIndirectFollowerMap.keySet()) {
+ nodesLogQueues.put(node, createQueueAndBindingThread(node));
+ }
}
class DispatcherThread extends LogDispatcher.DispatcherThread {
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/Log.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/Log.java
index 01d8541..9c6f4ce 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/Log.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/Log.java
@@ -34,7 +34,7 @@ public abstract class Log implements Comparable<Log> {
// make this configurable or adaptive
public static int DEFAULT_BUFFER_SIZE = 16 * 1024;
- private long currLogIndex;
+ private volatile long currLogIndex = Long.MIN_VALUE;
private long currLogTerm;
// for async application
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/LogDispatcher.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/LogDispatcher.java
index 17b853c..812fee1 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/LogDispatcher.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/LogDispatcher.java
@@ -48,7 +48,10 @@ import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutionException;
@@ -72,7 +75,7 @@ public class LogDispatcher {
RaftMember member;
private ClusterConfig clusterConfig = ClusterDescriptor.getInstance().getConfig();
private boolean useBatchInLogCatchUp = clusterConfig.isUseBatchInLogCatchUp();
- List<BlockingQueue<SendLogRequest>> nodesLogQueues = new ArrayList<>();
+ Map<Node, BlockingQueue<SendLogRequest>> nodesLogQueues = new HashMap<>();
ExecutorService executorService;
private static ExecutorService serializationService =
Executors.newFixedThreadPool(
@@ -90,7 +93,7 @@ public class LogDispatcher {
void createQueueAndBindingThreads() {
for (Node node : member.getAllNodes()) {
if (!ClusterUtils.isNodeEquals(node, member.getThisNode())) {
- nodesLogQueues.add(createQueueAndBindingThread(node));
+ nodesLogQueues.put(node, createQueueAndBindingThread(node));
}
}
}
@@ -115,8 +118,8 @@ public class LogDispatcher {
long startTime = Statistic.LOG_DISPATCHER_LOG_ENQUEUE.getOperationStartTime();
request.getVotingLog().getLog().setEnqueueTime(System.nanoTime());
- for (int i = 0; i < nodesLogQueues.size(); i++) {
- BlockingQueue<SendLogRequest> nodeLogQueue = nodesLogQueues.get(i);
+ for (Entry<Node, BlockingQueue<SendLogRequest>> entry : nodesLogQueues.entrySet()) {
+ BlockingQueue<SendLogRequest> nodeLogQueue = entry.getValue();
try {
boolean addSucceeded;
if (ClusterDescriptor.getInstance().getConfig().isWaitForSlowNode()) {
@@ -131,13 +134,17 @@ public class LogDispatcher {
if (!addSucceeded) {
logger.debug(
- "Log queue[{}] of {} is full, ignore the request to this node", i, member.getName());
+ "Log queue[{}] of {} is full, ignore the request to this node",
+ entry.getKey(),
+ member.getName());
} else {
request.setEnqueueTime(System.nanoTime());
}
} catch (IllegalStateException e) {
logger.debug(
- "Log queue[{}] of {} is full, ignore the request to this node", i, member.getName());
+ "Log queue[{}] of {} is full, ignore the request to this node",
+ entry.getKey(),
+ member.getName());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
@@ -442,6 +449,9 @@ public class LogDispatcher {
try {
long operationStartTime = Statistic.RAFT_SENDER_SEND_LOG.getOperationStartTime();
for (int i = 0; i < retries; i++) {
+ if (client == null) {
+ client = member.getSyncClient(receiver);
+ }
AppendEntryResult result = client.appendEntry(logRequest.appendEntryRequest);
if (result.status == Response.RESPONSE_OUT_OF_WINDOW) {
Thread.sleep(100);
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/CommittedEntryManager.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/CommittedEntryManager.java
index 4ecd167..77be5b3 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/CommittedEntryManager.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/CommittedEntryManager.java
@@ -110,6 +110,17 @@ public class CommittedEntryManager {
return getDummyIndex() + entries.size() - 1;
}
+ Long getLastTerm() {
+ int lastIndex = entries.size() - 1;
+ while (true) {
+ try {
+ return entries.get(lastIndex).getCurrLogTerm();
+ } catch (IndexOutOfBoundsException e) {
+ lastIndex = entries.size() - 1;
+ }
+ }
+ }
+
/**
* Return the entries's size
*
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/RaftLogManager.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/RaftLogManager.java
index 46ee69c..1781032 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/RaftLogManager.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/RaftLogManager.java
@@ -33,7 +33,6 @@ import org.apache.iotdb.cluster.log.StableEntryManager;
import org.apache.iotdb.cluster.log.manage.serializable.LogManagerMeta;
import org.apache.iotdb.cluster.server.monitor.Timer.Statistic;
import org.apache.iotdb.db.utils.TestOnly;
-import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.slf4j.Logger;
@@ -322,13 +321,11 @@ public abstract class RaftLogManager {
* @return last entry's term
*/
public long getLastLogTerm() {
- long term = -1;
- try {
- term = getTerm(getLastLogIndex());
- } catch (Exception e) {
- logger.error("{}: unexpected error when getting the last term : {}", name, e.getMessage());
+ long last = getUnCommittedEntryManager().maybeLastTerm();
+ if (last != -1) {
+ return last;
}
- return term;
+ return getCommittedEntryManager().getLastTerm();
}
/**
@@ -637,7 +634,7 @@ public abstract class RaftLogManager {
"{} should not go here, must be send to the follower, "
+ "so the log has been serialized exclude single node mode",
entry);
- entry.setByteSize((int) RamUsageEstimator.sizeOf(entry));
+ // entry.setByteSize((int) RamUsageEstimator.sizeOf(entry));
}
newEntryMemSize += entry.getByteSize();
}
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/UnCommittedEntryManager.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/UnCommittedEntryManager.java
index 2c79fdf..80bf2f8 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/UnCommittedEntryManager.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/UnCommittedEntryManager.java
@@ -72,6 +72,20 @@ public class UnCommittedEntryManager {
return -1;
}
+ long maybeLastTerm() {
+ int entryNum = entries.size();
+ while (entryNum != 0) {
+ try {
+ return entries.get(entryNum - 1).getCurrLogTerm();
+ } catch (IndexOutOfBoundsException e) {
+ // the exception is thrown when there is a concurrent deletion, which is rare, so a retry
+ // is usually enough and it is not likely that we will get stuck here
+ entryNum = entries.size();
+ }
+ }
+ return -1;
+ }
+
/**
* Return the entry's term for given index. Note that the called should ensure index >= offset.
*
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/AsynchronousSequencer.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/AsynchronousSequencer.java
index 4390def..5f7164f 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/AsynchronousSequencer.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/AsynchronousSequencer.java
@@ -108,8 +108,10 @@ public class AsynchronousSequencer implements LogSequencer {
Statistic.LOG_DISPATCHER_FROM_RECEIVE_TO_CREATE.calOperationCostTimeFromStart(
log.getReceiveTime());
log.setCreateTime(System.nanoTime());
- member.getVotingLogList().insert(sendLogRequest.getVotingLog());
- member.getLogDispatcher().offer(sendLogRequest);
+ if (member.getAllNodes().size() > 1) {
+ member.getVotingLogList().insert(sendLogRequest.getVotingLog());
+ member.getLogDispatcher().offer(sendLogRequest);
+ }
Statistic.RAFT_SENDER_OFFER_LOG.calOperationCostTimeFromStart(startTime);
RAFT_SENDER_SEQUENCE_LOG.calOperationCostTimeFromStart(sequenceStartTime);
}
@@ -131,6 +133,9 @@ public class AsynchronousSequencer implements LogSequencer {
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return;
+ } catch (Exception e) {
+ logger.error("Error in sequencer", e);
+ return;
}
}
}
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/SynchronousSequencer.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/SynchronousSequencer.java
index f7e6704..84b1011 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/SynchronousSequencer.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/SynchronousSequencer.java
@@ -70,8 +70,10 @@ public class SynchronousSequencer implements LogSequencer {
startTime = Statistic.RAFT_SENDER_OFFER_LOG.getOperationStartTime();
log.setCreateTime(System.nanoTime());
- member.getVotingLogList().insert(sendLogRequest.getVotingLog());
- member.getLogDispatcher().offer(sendLogRequest);
+ if (member.getAllNodes().size() > 1) {
+ member.getVotingLogList().insert(sendLogRequest.getVotingLog());
+ member.getLogDispatcher().offer(sendLogRequest);
+ }
Statistic.RAFT_SENDER_OFFER_LOG.calOperationCostTimeFromStart(startTime);
}
return sendLogRequest;
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/snapshot/MetaSimpleSnapshot.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/snapshot/MetaSimpleSnapshot.java
index 1713426..d89bb7e 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/snapshot/MetaSimpleSnapshot.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/snapshot/MetaSimpleSnapshot.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.cluster.log.snapshot;
+import org.apache.iotdb.cluster.expr.ExprMember;
import org.apache.iotdb.cluster.log.Snapshot;
import org.apache.iotdb.cluster.server.member.MetaGroupMember;
import org.apache.iotdb.cluster.server.member.RaftMember;
@@ -236,6 +237,9 @@ public class MetaSimpleSnapshot extends Snapshot {
synchronized (metaGroupMember.getLogManager()) {
metaGroupMember.getLogManager().applySnapshot(snapshot);
+ if (metaGroupMember instanceof ExprMember) {
+ ((ExprMember) metaGroupMember).resetWindow(snapshot.lastLogIndex, snapshot.lastLogTerm);
+ }
}
}
}
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/AppendNodeEntryHandler.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/AppendNodeEntryHandler.java
index 749a3ec..509adb3 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/AppendNodeEntryHandler.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/AppendNodeEntryHandler.java
@@ -88,14 +88,6 @@ public class AppendNodeEntryHandler implements AsyncMethodCallback<AppendEntryRe
long resp = response.status;
if (resp == RESPONSE_STRONG_ACCEPT) {
- synchronized (log.getStronglyAcceptedNodeIds()) {
- if (log.getWeaklyAcceptedNodeIds().size() + log.getStronglyAcceptedNodeIds().size()
- >= quorumSize) {
- log.acceptedTime.set(System.nanoTime());
- }
- log.getStronglyAcceptedNodeIds().add(receiver.nodeIdentifier);
- log.getStronglyAcceptedNodeIds().notifyAll();
- }
member
.getVotingLogList()
.onStronglyAccept(
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/HeartbeatHandler.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/HeartbeatHandler.java
index ba26259..8d47448 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/HeartbeatHandler.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/HeartbeatHandler.java
@@ -104,10 +104,10 @@ public class HeartbeatHandler implements AsyncMethodCallback<HeartBeatResponse>
if (!localMember.getLogManager().isLogUpToDate(lastLogTerm, lastLogIdx)
|| !localMember.getLogManager().matchTerm(lastLogTerm, lastLogIdx)) {
// the follower is not up-to-date
- if (lastLogIdx == -1) {
+ if (lastLogIdx == -1 || lastLogIdx < peer.getMatchIndex()) {
// maybe the follower has restarted, so we need to find its match index again, because
// some logs may be lost due to restart
- peer.setMatchIndex(-1);
+ peer.setMatchIndex(lastLogIdx);
}
// only start a catch up when the follower's lastLogIndex remains stall and unchanged for 3
@@ -115,7 +115,7 @@ public class HeartbeatHandler implements AsyncMethodCallback<HeartBeatResponse>
if (lastLogIdx == peer.getLastHeartBeatIndex()) {
// the follower's lastLogIndex is unchanged, increase inconsistent counter
int inconsistentNum = peer.incInconsistentHeartbeatNum();
- if (inconsistentNum >= 5) {
+ if (inconsistentNum >= 500) {
logger.info(
"{}: catching up node {}, index-term: {}-{}/{}-{}, peer match index {}",
memberName,
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/LogCatchUpHandler.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/LogCatchUpHandler.java
index 62933a7..f523393 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/LogCatchUpHandler.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/LogCatchUpHandler.java
@@ -49,11 +49,20 @@ public class LogCatchUpHandler implements AsyncMethodCallback<AppendEntryResult>
private String memberName;
private RaftMember raftMember;
+ private void processStrongAccept() {
+ raftMember
+ .getVotingLogList()
+ .onStronglyAccept(log.getCurrLogIndex(), log.getCurrLogTerm(), follower.nodeIdentifier);
+ }
+
@Override
public void onComplete(AppendEntryResult response) {
logger.debug("{}: Received a catch-up result of {} from {}", memberName, log, follower);
long resp = response.status;
- if (resp == RESPONSE_AGREE || resp == RESPONSE_STRONG_ACCEPT || resp == RESPONSE_WEAK_ACCEPT) {
+ if (resp == RESPONSE_AGREE || resp == RESPONSE_STRONG_ACCEPT) {
+ if (log.getCurrLogTerm() == raftMember.getTerm().get()) {
+ processStrongAccept();
+ }
synchronized (appendSucceed) {
appendSucceed.set(true);
appendSucceed.notifyAll();
@@ -67,6 +76,11 @@ public class LogCatchUpHandler implements AsyncMethodCallback<AppendEntryResult>
appendSucceed.set(true);
appendSucceed.notifyAll();
}
+ } else if (resp == RESPONSE_WEAK_ACCEPT) {
+ synchronized (appendSucceed) {
+ appendSucceed.notifyAll();
+ }
+ logger.warn("{}: Weak accept in {} catch-up with {}", memberName, follower, log);
} else {
// the follower's term has updated, which means a new leader is elected
logger.debug("{}: Received a rejection because term is updated to: {}", memberName, resp);
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/LogCatchUpInBatchHandler.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/LogCatchUpInBatchHandler.java
index 42656d1..65773c2 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/LogCatchUpInBatchHandler.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/LogCatchUpInBatchHandler.java
@@ -62,7 +62,10 @@ public class LogCatchUpInBatchHandler implements AsyncMethodCallback<AppendEntry
} else if (resp == RESPONSE_LOG_MISMATCH || resp == RESPONSE_WEAK_ACCEPT) {
// this is not probably possible
logger.error(
- "{}: Log mismatch occurred when sending logs, whose size is {}", memberName, logs.size());
+ "{}: Log mismatch occurred when sending logs, whose size is {}, resp: {}",
+ memberName,
+ logs.size(),
+ resp);
synchronized (appendSucceed) {
appendSucceed.notifyAll();
}
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/DataGroupMember.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/DataGroupMember.java
index 292dafe..5319d6e 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/DataGroupMember.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/DataGroupMember.java
@@ -625,8 +625,10 @@ public class DataGroupMember extends RaftMember {
logManager.append(log);
votingLog = buildVotingLog(log);
- votingLogList.insert(votingLog);
- logger.info("Send the close file request of {} to other nodes", log);
+ if (getAllNodes().size() > 1) {
+ votingLogList.insert(votingLog);
+ logger.info("Send the close file request of {} to other nodes", log);
+ }
}
try {
return appendLogInGroup(votingLog);
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/MetaGroupMember.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/MetaGroupMember.java
index ad0f55c..e4b79f6 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/MetaGroupMember.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/MetaGroupMember.java
@@ -956,7 +956,9 @@ public class MetaGroupMember extends RaftMember {
addNodeLog.setNewNode(newNode);
logManager.append(addNodeLog);
- votingLogList.insert(votingLog);
+ if (getAllNodes().size() > 1) {
+ votingLogList.insert(votingLog);
+ }
}
int retryTime = 0;
@@ -1712,7 +1714,9 @@ public class MetaGroupMember extends RaftMember {
removeNodeLog.setRemovedNode(target);
logManager.append(removeNodeLog);
- votingLogList.insert(votingLog);
+ if (getAllNodes().size() > 1) {
+ votingLogList.insert(votingLog);
+ }
}
int retryTime = 0;
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java
index 4a24d13..a8505c0 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java
@@ -38,6 +38,7 @@ import org.apache.iotdb.cluster.log.Log;
import org.apache.iotdb.cluster.log.LogDispatcher;
import org.apache.iotdb.cluster.log.LogDispatcher.SendLogRequest;
import org.apache.iotdb.cluster.log.LogParser;
+import org.apache.iotdb.cluster.log.LogRelay;
import org.apache.iotdb.cluster.log.VotingLog;
import org.apache.iotdb.cluster.log.catchup.CatchUpTask;
import org.apache.iotdb.cluster.log.logtypes.FragmentedLog;
@@ -132,7 +133,6 @@ public abstract class RaftMember {
private static final Logger logger = LoggerFactory.getLogger(RaftMember.class);
public static boolean USE_LOG_DISPATCHER = false;
- public static boolean USE_INDIRECT_LOG_DISPATCHER = false;
public static boolean ENABLE_WEAK_ACCEPTANCE = true;
public static boolean ENABLE_COMMIT_RETURN = false;
public static boolean USE_CRAFT = false;
@@ -279,6 +279,8 @@ public abstract class RaftMember {
protected LogSequencer logSequencer;
+ protected LogRelay logRelay;
+
protected RaftMember(
String name,
AsyncClientPool asyncPool,
@@ -345,6 +347,9 @@ public abstract class RaftMember {
commitLogPool =
Executors.newSingleThreadExecutor(
new ThreadFactoryBuilder().setNameFormat(getName() + "-CommitLog%d").build());
+ if (ClusterDescriptor.getInstance().getConfig().isUseIndirectBroadcasting()) {
+ logRelay = new LogRelay(this);
+ }
}
public String getName() {
@@ -411,6 +416,10 @@ public abstract class RaftMember {
logger.error("Unexpected interruption when waiting for commitLogPool to end", e);
}
}
+
+ if (logRelay != null) {
+ logRelay.stop();
+ }
leader.set(ClusterConstant.EMPTY_NODE);
catchUpService = null;
heartBeatService = null;
@@ -626,11 +635,11 @@ public abstract class RaftMember {
throws UnknownLogTypeException {
AppendEntryResult result = appendEntry(request);
request.entry.rewind();
- appendLogThreadPool.submit(() -> sendLogToSubFollowers(request, subFollowers));
+ logRelay.offer(request, subFollowers);
return result;
}
- private void sendLogToSubFollowers(AppendEntryRequest request, List<Node> subFollowers) {
+ public void sendLogToSubFollowers(AppendEntryRequest request, List<Node> subFollowers) {
request.setIsFromLeader(false);
for (Node subFollower : subFollowers) {
Client syncClient = null;
@@ -1162,7 +1171,9 @@ public abstract class RaftMember {
log.setCurrLogIndex(logManager.getLastLogIndex() + 1);
logManager.append(log);
votingLog = buildVotingLog(log);
- votingLogList.insert(votingLog);
+ if (getAllNodes().size() > 1) {
+ votingLogList.insert(votingLog);
+ }
}
log.setCreateTime(System.nanoTime());
@@ -1612,7 +1623,7 @@ public abstract class RaftMember {
public synchronized LogDispatcher getLogDispatcher() {
if (logDispatcher == null) {
- if (USE_INDIRECT_LOG_DISPATCHER) {
+ if (ClusterDescriptor.getInstance().getConfig().isUseIndirectBroadcasting()) {
logDispatcher = new IndirectLogDispatcher(this);
} else if (USE_CRAFT && allNodes.size() > 2) {
logDispatcher = new FragmentedLogDispatcher(this);
@@ -1628,16 +1639,17 @@ public abstract class RaftMember {
int stronglyAcceptedNodeNum = log.getStronglyAcceptedNodeIds().size();
int weaklyAcceptedNodeNum = log.getWeaklyAcceptedNodeIds().size();
int totalAccepted = stronglyAcceptedNodeNum + weaklyAcceptedNodeNum;
- long nextTimeToPrint = 1000;
+ long nextTimeToPrint = 5000;
long waitStart = System.currentTimeMillis();
long alreadyWait = 0;
- while (stronglyAcceptedNodeNum < quorumSize
- && (!ENABLE_WEAK_ACCEPTANCE
- || (totalAccepted < quorumSize)
- || votingLogList.size() > config.getMaxNumOfLogsInMem())
- && alreadyWait < RaftServer.getWriteOperationTimeoutMS()
- && !log.getStronglyAcceptedNodeIds().contains(Integer.MAX_VALUE)) {
+ while (log.getLog().getCurrLogIndex() == Long.MIN_VALUE
+ || (stronglyAcceptedNodeNum < quorumSize
+ && (!ENABLE_WEAK_ACCEPTANCE
+ || (totalAccepted < quorumSize)
+ || votingLogList.size() > config.getMaxNumOfLogsInMem())
+ && alreadyWait < RaftServer.getWriteOperationTimeoutMS()
+ && !log.getStronglyAcceptedNodeIds().contains(Integer.MAX_VALUE))) {
long singleWaitTime = 0;
long singleWaitStart = System.nanoTime();
try {
@@ -1645,7 +1657,6 @@ public abstract class RaftMember {
log.getStronglyAcceptedNodeIds().wait(1);
}
singleWaitTime = (System.nanoTime() - singleWaitStart) / 1000000;
- logger.debug("{} ends waiting", log);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
logger.warn("Unexpected interruption when sending a log", e);
@@ -1695,11 +1706,12 @@ public abstract class RaftMember {
int weaklyAcceptedNodeNum = log.getWeaklyAcceptedNodeIds().size();
int totalAccepted = stronglyAcceptedNodeNum + weaklyAcceptedNodeNum;
- if (stronglyAcceptedNodeNum < quorumSize
- && (!ENABLE_WEAK_ACCEPTANCE
- || (totalAccepted < quorumSize)
- || votingLogList.size() > config.getMaxNumOfLogsInMem())
- && !log.getStronglyAcceptedNodeIds().contains(Integer.MAX_VALUE)) {
+ if (log.getLog().getCurrLogIndex() == Long.MIN_VALUE
+ || (stronglyAcceptedNodeNum < quorumSize
+ && (!ENABLE_WEAK_ACCEPTANCE
+ || (totalAccepted < quorumSize)
+ || votingLogList.size() > config.getMaxNumOfLogsInMem())
+ && !log.getStronglyAcceptedNodeIds().contains(Integer.MAX_VALUE))) {
waitAppendResultLoop(log, quorumSize);
}
@@ -2134,6 +2146,8 @@ public abstract class RaftMember {
} finally {
ClientUtils.putBackSyncClient(client);
}
+ } else {
+ logger.debug("Skip sending {} because cannot get client of {}", log, node);
}
}
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/monitor/Timer.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/monitor/Timer.java
index c72550e..5a59d6e 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/monitor/Timer.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/monitor/Timer.java
@@ -334,7 +334,7 @@ public class Timer {
* method to avoid unnecessary calls when instrumenting is disabled.
*/
public void calOperationCostTimeFromStart(long startTime) {
- if (ENABLE_INSTRUMENTING && startTime != Long.MIN_VALUE) {
+ if (ENABLE_INSTRUMENTING && startTime != Long.MIN_VALUE && startTime != 0) {
long consumed = System.nanoTime() - startTime;
add(consumed);
if (consumed >= warningThreshold && logger.isWarnEnabled()) {