You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2022/03/03 08:20:01 UTC

[iotdb] branch expr_plus updated: add LogRelay

This is an automated email from the ASF dual-hosted git repository.

jiangtian pushed a commit to branch expr_plus
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/expr_plus by this push:
     new 5033ff9  add LogRelay
5033ff9 is described below

commit 5033ff9c57f3002596759178004e28634882c914
Author: jt <jt...@163.com>
AuthorDate: Thu Mar 3 16:19:17 2022 +0800

    add LogRelay
---
 .../resources/conf/iotdb-cluster.properties        |  3 ++
 .../iotdb/cluster/client/sync/SyncClientPool.java  | 11 ++++-
 .../apache/iotdb/cluster/config/ClusterConfig.java | 10 +++++
 .../iotdb/cluster/config/ClusterDescriptor.java    | 15 +++++++
 .../org/apache/iotdb/cluster/expr/ExprMember.java  | 16 +++++--
 .../org/apache/iotdb/cluster/expr/ExprServer.java  |  2 +-
 .../iotdb/cluster/log/IndirectLogDispatcher.java   |  6 +--
 .../java/org/apache/iotdb/cluster/log/Log.java     |  2 +-
 .../apache/iotdb/cluster/log/LogDispatcher.java    | 22 +++++++---
 .../cluster/log/manage/CommittedEntryManager.java  | 11 +++++
 .../iotdb/cluster/log/manage/RaftLogManager.java   | 13 +++---
 .../log/manage/UnCommittedEntryManager.java        | 14 ++++++
 .../log/sequencing/AsynchronousSequencer.java      |  9 +++-
 .../log/sequencing/SynchronousSequencer.java       |  6 ++-
 .../cluster/log/snapshot/MetaSimpleSnapshot.java   |  4 ++
 .../handlers/caller/AppendNodeEntryHandler.java    |  8 ----
 .../server/handlers/caller/HeartbeatHandler.java   |  6 +--
 .../server/handlers/caller/LogCatchUpHandler.java  | 16 ++++++-
 .../handlers/caller/LogCatchUpInBatchHandler.java  |  5 ++-
 .../cluster/server/member/DataGroupMember.java     |  6 ++-
 .../cluster/server/member/MetaGroupMember.java     |  8 +++-
 .../iotdb/cluster/server/member/RaftMember.java    | 50 ++++++++++++++--------
 .../apache/iotdb/cluster/server/monitor/Timer.java |  2 +-
 23 files changed, 181 insertions(+), 64 deletions(-)

diff --git a/cluster/src/assembly/resources/conf/iotdb-cluster.properties b/cluster/src/assembly/resources/conf/iotdb-cluster.properties
index 2b01d5b..eb3b937 100644
--- a/cluster/src/assembly/resources/conf/iotdb-cluster.properties
+++ b/cluster/src/assembly/resources/conf/iotdb-cluster.properties
@@ -192,3 +192,6 @@ multi_raft_factor=1
 # we need to wait so much time for other connections to be released until timeout,
 # or a new connection will be created.
 # wait_client_timeout_ms=5000
+
+
+use_indirect_broadcasting=true
\ No newline at end of file
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/client/sync/SyncClientPool.java b/cluster/src/main/java/org/apache/iotdb/cluster/client/sync/SyncClientPool.java
index 2c279c0..6d13be3 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/client/sync/SyncClientPool.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/client/sync/SyncClientPool.java
@@ -30,6 +30,8 @@ import org.apache.thrift.transport.TTransportException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.net.ConnectException;
+import java.net.SocketTimeoutException;
 import java.util.ArrayDeque;
 import java.util.Deque;
 import java.util.Map;
@@ -90,13 +92,18 @@ public class SyncClientPool {
           try {
             client = syncClientFactory.getSyncClient(clusterNode, this);
           } catch (TTransportException e) {
-            logger.error("Cannot open transport for client {}", node, e);
+            if (!(e.getCause() instanceof ConnectException
+                || e.getCause() instanceof SocketTimeoutException)) {
+              logger.error("Cannot open transport for client {}", node, e);
+            }
             return null;
           }
           nodeClientNumMap.compute(
               clusterNode,
               (n, oldValue) -> {
-                if (oldValue == null) return 1;
+                if (oldValue == null) {
+                  return 1;
+                }
                 return oldValue + 1;
               });
           return client;
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java b/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java
index 420f7ca..a5eb928 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java
@@ -185,6 +185,8 @@ public class ClusterConfig {
 
   private boolean useAsyncSequencing = true;
 
+  private int relaySenderNum = 8;
+
   /**
    * create a clusterConfig class. The internalIP will be set according to the server's hostname. If
    * there is something error for getting the ip of the hostname, then set the internalIp as
@@ -559,4 +561,12 @@ public class ClusterConfig {
   public void setUseAsyncSequencing(boolean useAsyncSequencing) {
     this.useAsyncSequencing = useAsyncSequencing;
   }
+
+  public int getRelaySenderNum() {
+    return relaySenderNum;
+  }
+
+  public void setRelaySenderNum(int relaySenderNum) {
+    this.relaySenderNum = relaySenderNum;
+  }
 }
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterDescriptor.java b/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterDescriptor.java
index f70f08f..890bbf8 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterDescriptor.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterDescriptor.java
@@ -323,6 +323,21 @@ public class ClusterDescriptor {
             properties.getProperty(
                 "wait_client_timeout_ms", String.valueOf(config.getWaitClientTimeoutMS()))));
 
+    config.setUseAsyncSequencing(
+        Boolean.parseBoolean(
+            properties.getProperty(
+                "use_async_sequencing", String.valueOf(config.isUseAsyncSequencing()))));
+
+    config.setUseIndirectBroadcasting(
+        Boolean.parseBoolean(
+            properties.getProperty(
+                "use_indirect_broadcasting", String.valueOf(config.isUseIndirectBroadcasting()))));
+
+    config.setRelaySenderNum(
+        Integer.parseInt(
+            properties.getProperty(
+                "relay_sender_number", String.valueOf(config.getRelaySenderNum()))));
+
     String consistencyLevel = properties.getProperty("consistency_level");
     if (consistencyLevel != null) {
       config.setConsistencyLevel(ConsistencyLevel.getConsistencyLevel(consistencyLevel));
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/expr/ExprMember.java b/cluster/src/main/java/org/apache/iotdb/cluster/expr/ExprMember.java
index bcc26ce..2f92a88 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/expr/ExprMember.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/expr/ExprMember.java
@@ -79,8 +79,7 @@ public class ExprMember extends MetaGroupMember {
   public ExprMember(TProtocolFactory factory, Node thisNode, Coordinator coordinator)
       throws QueryProcessException {
     super(factory, thisNode, coordinator);
-    this.firstPosPrevIndex = logManager.getLastLogIndex();
-    this.prevTerms[0] = logManager.getLastLogTerm();
+    resetWindow(logManager.getLastLogIndex(), logManager.getLastLogTerm());
     bypassPool = Executors.newFixedThreadPool(LogDispatcher.bindingThreadNum);
   }
 
@@ -89,10 +88,19 @@ public class ExprMember extends MetaGroupMember {
     // do not start data groups in such experiments
   }
 
+  public void resetWindow(long firstPosPrevIndex, long firstPosPrevTerm) {
+    this.firstPosPrevIndex = firstPosPrevIndex;
+    this.prevTerms[0] = firstPosPrevTerm;
+    logWindow = new Log[windowCapacity];
+  }
+
   @Override
   public TSStatus executeNonQueryPlan(PhysicalPlan plan) {
     try {
       if (bypassRaft) {
+        if (allNodes.size() == 1) {
+          return StatusUtils.OK;
+        }
         CountDownLatch latch = new CountDownLatch(allNodes.size() / 2);
         int bufferSize = 4096;
         if (plan instanceof DummyPlan && !((DummyPlan) plan).isNeedForward()) {
@@ -130,7 +138,9 @@ public class ExprMember extends MetaGroupMember {
                     }));
           }
         }
-        latch.await();
+        if (allNodes.size() > 1) {
+          latch.await();
+        }
         return StatusUtils.OK;
       }
       long startTime = Timer.Statistic.META_GROUP_MEMBER_EXECUTE_NON_QUERY.getOperationStartTime();
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/expr/ExprServer.java b/cluster/src/main/java/org/apache/iotdb/cluster/expr/ExprServer.java
index ab2468e..d245c23 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/expr/ExprServer.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/expr/ExprServer.java
@@ -90,7 +90,7 @@ public class ExprServer extends MetaClusterServer {
     ClusterDescriptor.getInstance().getConfig().setMaxClientPerNodePerMember(50000);
     // ClusterDescriptor.getInstance().getConfig().setUseBatchInLogCatchUp(false);
     RaftMember.USE_LOG_DISPATCHER = true;
-    RaftMember.USE_INDIRECT_LOG_DISPATCHER = useIndirectDispatcher;
+    ClusterDescriptor.getInstance().getConfig().setUseIndirectBroadcasting(useIndirectDispatcher);
     LogDispatcher.bindingThreadNum = dispatcherThreadNum;
     LogDispatcher.maxBatchSize = maxBatchSize;
     ExprMember.bypassRaft = bypassRaft;
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/IndirectLogDispatcher.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/IndirectLogDispatcher.java
index 767736d..cc05753 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/IndirectLogDispatcher.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/IndirectLogDispatcher.java
@@ -87,9 +87,9 @@ public class IndirectLogDispatcher extends LogDispatcher {
       }
     }
 
-    //    for (Node node : directToIndirectFollowerMap.keySet()) {
-    //      nodeLogQueues.add(createQueueAndBindingThread(node));
-    //    }
+    for (Node node : directToIndirectFollowerMap.keySet()) {
+      nodesLogQueues.put(node, createQueueAndBindingThread(node));
+    }
   }
 
   class DispatcherThread extends LogDispatcher.DispatcherThread {
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/Log.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/Log.java
index 01d8541..9c6f4ce 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/Log.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/Log.java
@@ -34,7 +34,7 @@ public abstract class Log implements Comparable<Log> {
 
   // make this configurable or adaptive
   public static int DEFAULT_BUFFER_SIZE = 16 * 1024;
-  private long currLogIndex;
+  private volatile long currLogIndex = Long.MIN_VALUE;
   private long currLogTerm;
 
   // for async application
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/LogDispatcher.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/LogDispatcher.java
index 17b853c..812fee1 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/LogDispatcher.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/LogDispatcher.java
@@ -48,7 +48,10 @@ import org.slf4j.LoggerFactory;
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ExecutionException;
@@ -72,7 +75,7 @@ public class LogDispatcher {
   RaftMember member;
   private ClusterConfig clusterConfig = ClusterDescriptor.getInstance().getConfig();
   private boolean useBatchInLogCatchUp = clusterConfig.isUseBatchInLogCatchUp();
-  List<BlockingQueue<SendLogRequest>> nodesLogQueues = new ArrayList<>();
+  Map<Node, BlockingQueue<SendLogRequest>> nodesLogQueues = new HashMap<>();
   ExecutorService executorService;
   private static ExecutorService serializationService =
       Executors.newFixedThreadPool(
@@ -90,7 +93,7 @@ public class LogDispatcher {
   void createQueueAndBindingThreads() {
     for (Node node : member.getAllNodes()) {
       if (!ClusterUtils.isNodeEquals(node, member.getThisNode())) {
-        nodesLogQueues.add(createQueueAndBindingThread(node));
+        nodesLogQueues.put(node, createQueueAndBindingThread(node));
       }
     }
   }
@@ -115,8 +118,8 @@ public class LogDispatcher {
 
     long startTime = Statistic.LOG_DISPATCHER_LOG_ENQUEUE.getOperationStartTime();
     request.getVotingLog().getLog().setEnqueueTime(System.nanoTime());
-    for (int i = 0; i < nodesLogQueues.size(); i++) {
-      BlockingQueue<SendLogRequest> nodeLogQueue = nodesLogQueues.get(i);
+    for (Entry<Node, BlockingQueue<SendLogRequest>> entry : nodesLogQueues.entrySet()) {
+      BlockingQueue<SendLogRequest> nodeLogQueue = entry.getValue();
       try {
         boolean addSucceeded;
         if (ClusterDescriptor.getInstance().getConfig().isWaitForSlowNode()) {
@@ -131,13 +134,17 @@ public class LogDispatcher {
 
         if (!addSucceeded) {
           logger.debug(
-              "Log queue[{}] of {} is full, ignore the request to this node", i, member.getName());
+              "Log queue[{}] of {} is full, ignore the request to this node",
+              entry.getKey(),
+              member.getName());
         } else {
           request.setEnqueueTime(System.nanoTime());
         }
       } catch (IllegalStateException e) {
         logger.debug(
-            "Log queue[{}] of {} is full, ignore the request to this node", i, member.getName());
+            "Log queue[{}] of {} is full, ignore the request to this node",
+            entry.getKey(),
+            member.getName());
       } catch (InterruptedException e) {
         Thread.currentThread().interrupt();
       }
@@ -442,6 +449,9 @@ public class LogDispatcher {
       try {
         long operationStartTime = Statistic.RAFT_SENDER_SEND_LOG.getOperationStartTime();
         for (int i = 0; i < retries; i++) {
+          if (client == null) {
+            client = member.getSyncClient(receiver);
+          }
           AppendEntryResult result = client.appendEntry(logRequest.appendEntryRequest);
           if (result.status == Response.RESPONSE_OUT_OF_WINDOW) {
             Thread.sleep(100);
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/CommittedEntryManager.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/CommittedEntryManager.java
index 4ecd167..77be5b3 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/CommittedEntryManager.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/CommittedEntryManager.java
@@ -110,6 +110,17 @@ public class CommittedEntryManager {
     return getDummyIndex() + entries.size() - 1;
   }
 
+  Long getLastTerm() {
+    int lastIndex = entries.size() - 1;
+    while (true) {
+      try {
+        return entries.get(lastIndex).getCurrLogTerm();
+      } catch (IndexOutOfBoundsException e) {
+        lastIndex = entries.size() - 1;
+      }
+    }
+  }
+
   /**
    * Return the entries's size
    *
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/RaftLogManager.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/RaftLogManager.java
index 46ee69c..1781032 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/RaftLogManager.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/RaftLogManager.java
@@ -33,7 +33,6 @@ import org.apache.iotdb.cluster.log.StableEntryManager;
 import org.apache.iotdb.cluster.log.manage.serializable.LogManagerMeta;
 import org.apache.iotdb.cluster.server.monitor.Timer.Statistic;
 import org.apache.iotdb.db.utils.TestOnly;
-import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
 
 import org.apache.commons.lang3.concurrent.BasicThreadFactory;
 import org.slf4j.Logger;
@@ -322,13 +321,11 @@ public abstract class RaftLogManager {
    * @return last entry's term
    */
   public long getLastLogTerm() {
-    long term = -1;
-    try {
-      term = getTerm(getLastLogIndex());
-    } catch (Exception e) {
-      logger.error("{}: unexpected error when getting the last term : {}", name, e.getMessage());
+    long last = getUnCommittedEntryManager().maybeLastTerm();
+    if (last != -1) {
+      return last;
     }
-    return term;
+    return getCommittedEntryManager().getLastTerm();
   }
 
   /**
@@ -637,7 +634,7 @@ public abstract class RaftLogManager {
             "{} should not go here, must be send to the follower, "
                 + "so the log has been serialized exclude single node mode",
             entry);
-        entry.setByteSize((int) RamUsageEstimator.sizeOf(entry));
+        // entry.setByteSize((int) RamUsageEstimator.sizeOf(entry));
       }
       newEntryMemSize += entry.getByteSize();
     }
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/UnCommittedEntryManager.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/UnCommittedEntryManager.java
index 2c79fdf..80bf2f8 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/UnCommittedEntryManager.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/manage/UnCommittedEntryManager.java
@@ -72,6 +72,20 @@ public class UnCommittedEntryManager {
     return -1;
   }
 
+  long maybeLastTerm() {
+    int entryNum = entries.size();
+    while (entryNum != 0) {
+      try {
+        return entries.get(entryNum - 1).getCurrLogTerm();
+      } catch (IndexOutOfBoundsException e) {
+        // the exception is thrown when there is a concurrent deletion, which is rare, so a retry
+        // is usually enough and it is not likely that we will get stuck here
+        entryNum = entries.size();
+      }
+    }
+    return -1;
+  }
+
   /**
    * Return the entry's term for given index. Note that the called should ensure index >= offset.
    *
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/AsynchronousSequencer.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/AsynchronousSequencer.java
index 4390def..5f7164f 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/AsynchronousSequencer.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/AsynchronousSequencer.java
@@ -108,8 +108,10 @@ public class AsynchronousSequencer implements LogSequencer {
         Statistic.LOG_DISPATCHER_FROM_RECEIVE_TO_CREATE.calOperationCostTimeFromStart(
             log.getReceiveTime());
         log.setCreateTime(System.nanoTime());
-        member.getVotingLogList().insert(sendLogRequest.getVotingLog());
-        member.getLogDispatcher().offer(sendLogRequest);
+        if (member.getAllNodes().size() > 1) {
+          member.getVotingLogList().insert(sendLogRequest.getVotingLog());
+          member.getLogDispatcher().offer(sendLogRequest);
+        }
         Statistic.RAFT_SENDER_OFFER_LOG.calOperationCostTimeFromStart(startTime);
         RAFT_SENDER_SEQUENCE_LOG.calOperationCostTimeFromStart(sequenceStartTime);
       }
@@ -131,6 +133,9 @@ public class AsynchronousSequencer implements LogSequencer {
       } catch (InterruptedException e) {
         Thread.currentThread().interrupt();
         return;
+      } catch (Exception e) {
+        logger.error("Error in sequencer", e);
+        return;
       }
     }
   }
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/SynchronousSequencer.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/SynchronousSequencer.java
index f7e6704..84b1011 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/SynchronousSequencer.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/sequencing/SynchronousSequencer.java
@@ -70,8 +70,10 @@ public class SynchronousSequencer implements LogSequencer {
 
       startTime = Statistic.RAFT_SENDER_OFFER_LOG.getOperationStartTime();
       log.setCreateTime(System.nanoTime());
-      member.getVotingLogList().insert(sendLogRequest.getVotingLog());
-      member.getLogDispatcher().offer(sendLogRequest);
+      if (member.getAllNodes().size() > 1) {
+        member.getVotingLogList().insert(sendLogRequest.getVotingLog());
+        member.getLogDispatcher().offer(sendLogRequest);
+      }
       Statistic.RAFT_SENDER_OFFER_LOG.calOperationCostTimeFromStart(startTime);
     }
     return sendLogRequest;
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/log/snapshot/MetaSimpleSnapshot.java b/cluster/src/main/java/org/apache/iotdb/cluster/log/snapshot/MetaSimpleSnapshot.java
index 1713426..d89bb7e 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/log/snapshot/MetaSimpleSnapshot.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/log/snapshot/MetaSimpleSnapshot.java
@@ -19,6 +19,7 @@
 
 package org.apache.iotdb.cluster.log.snapshot;
 
+import org.apache.iotdb.cluster.expr.ExprMember;
 import org.apache.iotdb.cluster.log.Snapshot;
 import org.apache.iotdb.cluster.server.member.MetaGroupMember;
 import org.apache.iotdb.cluster.server.member.RaftMember;
@@ -236,6 +237,9 @@ public class MetaSimpleSnapshot extends Snapshot {
 
         synchronized (metaGroupMember.getLogManager()) {
           metaGroupMember.getLogManager().applySnapshot(snapshot);
+          if (metaGroupMember instanceof ExprMember) {
+            ((ExprMember) metaGroupMember).resetWindow(snapshot.lastLogIndex, snapshot.lastLogTerm);
+          }
         }
       }
     }
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/AppendNodeEntryHandler.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/AppendNodeEntryHandler.java
index 749a3ec..509adb3 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/AppendNodeEntryHandler.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/AppendNodeEntryHandler.java
@@ -88,14 +88,6 @@ public class AppendNodeEntryHandler implements AsyncMethodCallback<AppendEntryRe
     long resp = response.status;
 
     if (resp == RESPONSE_STRONG_ACCEPT) {
-      synchronized (log.getStronglyAcceptedNodeIds()) {
-        if (log.getWeaklyAcceptedNodeIds().size() + log.getStronglyAcceptedNodeIds().size()
-            >= quorumSize) {
-          log.acceptedTime.set(System.nanoTime());
-        }
-        log.getStronglyAcceptedNodeIds().add(receiver.nodeIdentifier);
-        log.getStronglyAcceptedNodeIds().notifyAll();
-      }
       member
           .getVotingLogList()
           .onStronglyAccept(
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/HeartbeatHandler.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/HeartbeatHandler.java
index ba26259..8d47448 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/HeartbeatHandler.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/HeartbeatHandler.java
@@ -104,10 +104,10 @@ public class HeartbeatHandler implements AsyncMethodCallback<HeartBeatResponse>
     if (!localMember.getLogManager().isLogUpToDate(lastLogTerm, lastLogIdx)
         || !localMember.getLogManager().matchTerm(lastLogTerm, lastLogIdx)) {
       // the follower is not up-to-date
-      if (lastLogIdx == -1) {
+      if (lastLogIdx == -1 || lastLogIdx < peer.getMatchIndex()) {
         // maybe the follower has restarted, so we need to find its match index again, because
         // some logs may be lost due to restart
-        peer.setMatchIndex(-1);
+        peer.setMatchIndex(lastLogIdx);
       }
 
       // only start a catch up when the follower's lastLogIndex remains stall and unchanged for 3
@@ -115,7 +115,7 @@ public class HeartbeatHandler implements AsyncMethodCallback<HeartBeatResponse>
       if (lastLogIdx == peer.getLastHeartBeatIndex()) {
         // the follower's lastLogIndex is unchanged, increase inconsistent counter
         int inconsistentNum = peer.incInconsistentHeartbeatNum();
-        if (inconsistentNum >= 5) {
+        if (inconsistentNum >= 500) {
           logger.info(
               "{}: catching up node {}, index-term: {}-{}/{}-{}, peer match index {}",
               memberName,
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/LogCatchUpHandler.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/LogCatchUpHandler.java
index 62933a7..f523393 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/LogCatchUpHandler.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/LogCatchUpHandler.java
@@ -49,11 +49,20 @@ public class LogCatchUpHandler implements AsyncMethodCallback<AppendEntryResult>
   private String memberName;
   private RaftMember raftMember;
 
+  private void processStrongAccept() {
+    raftMember
+        .getVotingLogList()
+        .onStronglyAccept(log.getCurrLogIndex(), log.getCurrLogTerm(), follower.nodeIdentifier);
+  }
+
   @Override
   public void onComplete(AppendEntryResult response) {
     logger.debug("{}: Received a catch-up result of {} from {}", memberName, log, follower);
     long resp = response.status;
-    if (resp == RESPONSE_AGREE || resp == RESPONSE_STRONG_ACCEPT || resp == RESPONSE_WEAK_ACCEPT) {
+    if (resp == RESPONSE_AGREE || resp == RESPONSE_STRONG_ACCEPT) {
+      if (log.getCurrLogTerm() == raftMember.getTerm().get()) {
+        processStrongAccept();
+      }
       synchronized (appendSucceed) {
         appendSucceed.set(true);
         appendSucceed.notifyAll();
@@ -67,6 +76,11 @@ public class LogCatchUpHandler implements AsyncMethodCallback<AppendEntryResult>
         appendSucceed.set(true);
         appendSucceed.notifyAll();
       }
+    } else if (resp == RESPONSE_WEAK_ACCEPT) {
+      synchronized (appendSucceed) {
+        appendSucceed.notifyAll();
+      }
+      logger.warn("{}: Weak accept in {} catch-up with {}", memberName, follower, log);
     } else {
       // the follower's term has updated, which means a new leader is elected
       logger.debug("{}: Received a rejection because term is updated to: {}", memberName, resp);
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/LogCatchUpInBatchHandler.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/LogCatchUpInBatchHandler.java
index 42656d1..65773c2 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/LogCatchUpInBatchHandler.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/handlers/caller/LogCatchUpInBatchHandler.java
@@ -62,7 +62,10 @@ public class LogCatchUpInBatchHandler implements AsyncMethodCallback<AppendEntry
     } else if (resp == RESPONSE_LOG_MISMATCH || resp == RESPONSE_WEAK_ACCEPT) {
       // this is not probably possible
       logger.error(
-          "{}: Log mismatch occurred when sending logs, whose size is {}", memberName, logs.size());
+          "{}: Log mismatch occurred when sending logs, whose size is {}, resp: {}",
+          memberName,
+          logs.size(),
+          resp);
       synchronized (appendSucceed) {
         appendSucceed.notifyAll();
       }
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/DataGroupMember.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/DataGroupMember.java
index 292dafe..5319d6e 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/DataGroupMember.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/DataGroupMember.java
@@ -625,8 +625,10 @@ public class DataGroupMember extends RaftMember {
 
       logManager.append(log);
       votingLog = buildVotingLog(log);
-      votingLogList.insert(votingLog);
-      logger.info("Send the close file request of {} to other nodes", log);
+      if (getAllNodes().size() > 1) {
+        votingLogList.insert(votingLog);
+        logger.info("Send the close file request of {} to other nodes", log);
+      }
     }
     try {
       return appendLogInGroup(votingLog);
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/MetaGroupMember.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/MetaGroupMember.java
index ad0f55c..e4b79f6 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/MetaGroupMember.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/MetaGroupMember.java
@@ -956,7 +956,9 @@ public class MetaGroupMember extends RaftMember {
       addNodeLog.setNewNode(newNode);
 
       logManager.append(addNodeLog);
-      votingLogList.insert(votingLog);
+      if (getAllNodes().size() > 1) {
+        votingLogList.insert(votingLog);
+      }
     }
 
     int retryTime = 0;
@@ -1712,7 +1714,9 @@ public class MetaGroupMember extends RaftMember {
       removeNodeLog.setRemovedNode(target);
 
       logManager.append(removeNodeLog);
-      votingLogList.insert(votingLog);
+      if (getAllNodes().size() > 1) {
+        votingLogList.insert(votingLog);
+      }
     }
 
     int retryTime = 0;
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java
index 4a24d13..a8505c0 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java
@@ -38,6 +38,7 @@ import org.apache.iotdb.cluster.log.Log;
 import org.apache.iotdb.cluster.log.LogDispatcher;
 import org.apache.iotdb.cluster.log.LogDispatcher.SendLogRequest;
 import org.apache.iotdb.cluster.log.LogParser;
+import org.apache.iotdb.cluster.log.LogRelay;
 import org.apache.iotdb.cluster.log.VotingLog;
 import org.apache.iotdb.cluster.log.catchup.CatchUpTask;
 import org.apache.iotdb.cluster.log.logtypes.FragmentedLog;
@@ -132,7 +133,6 @@ public abstract class RaftMember {
 
   private static final Logger logger = LoggerFactory.getLogger(RaftMember.class);
   public static boolean USE_LOG_DISPATCHER = false;
-  public static boolean USE_INDIRECT_LOG_DISPATCHER = false;
   public static boolean ENABLE_WEAK_ACCEPTANCE = true;
   public static boolean ENABLE_COMMIT_RETURN = false;
   public static boolean USE_CRAFT = false;
@@ -279,6 +279,8 @@ public abstract class RaftMember {
 
   protected LogSequencer logSequencer;
 
+  protected LogRelay logRelay;
+
   protected RaftMember(
       String name,
       AsyncClientPool asyncPool,
@@ -345,6 +347,9 @@ public abstract class RaftMember {
     commitLogPool =
         Executors.newSingleThreadExecutor(
             new ThreadFactoryBuilder().setNameFormat(getName() + "-CommitLog%d").build());
+    if (ClusterDescriptor.getInstance().getConfig().isUseIndirectBroadcasting()) {
+      logRelay = new LogRelay(this);
+    }
   }
 
   public String getName() {
@@ -411,6 +416,10 @@ public abstract class RaftMember {
         logger.error("Unexpected interruption when waiting for commitLogPool to end", e);
       }
     }
+
+    if (logRelay != null) {
+      logRelay.stop();
+    }
     leader.set(ClusterConstant.EMPTY_NODE);
     catchUpService = null;
     heartBeatService = null;
@@ -626,11 +635,11 @@ public abstract class RaftMember {
       throws UnknownLogTypeException {
     AppendEntryResult result = appendEntry(request);
     request.entry.rewind();
-    appendLogThreadPool.submit(() -> sendLogToSubFollowers(request, subFollowers));
+    logRelay.offer(request, subFollowers);
     return result;
   }
 
-  private void sendLogToSubFollowers(AppendEntryRequest request, List<Node> subFollowers) {
+  public void sendLogToSubFollowers(AppendEntryRequest request, List<Node> subFollowers) {
     request.setIsFromLeader(false);
     for (Node subFollower : subFollowers) {
       Client syncClient = null;
@@ -1162,7 +1171,9 @@ public abstract class RaftMember {
       log.setCurrLogIndex(logManager.getLastLogIndex() + 1);
       logManager.append(log);
       votingLog = buildVotingLog(log);
-      votingLogList.insert(votingLog);
+      if (getAllNodes().size() > 1) {
+        votingLogList.insert(votingLog);
+      }
     }
     log.setCreateTime(System.nanoTime());
 
@@ -1612,7 +1623,7 @@ public abstract class RaftMember {
 
   public synchronized LogDispatcher getLogDispatcher() {
     if (logDispatcher == null) {
-      if (USE_INDIRECT_LOG_DISPATCHER) {
+      if (ClusterDescriptor.getInstance().getConfig().isUseIndirectBroadcasting()) {
         logDispatcher = new IndirectLogDispatcher(this);
       } else if (USE_CRAFT && allNodes.size() > 2) {
         logDispatcher = new FragmentedLogDispatcher(this);
@@ -1628,16 +1639,17 @@ public abstract class RaftMember {
     int stronglyAcceptedNodeNum = log.getStronglyAcceptedNodeIds().size();
     int weaklyAcceptedNodeNum = log.getWeaklyAcceptedNodeIds().size();
     int totalAccepted = stronglyAcceptedNodeNum + weaklyAcceptedNodeNum;
-    long nextTimeToPrint = 1000;
+    long nextTimeToPrint = 5000;
 
     long waitStart = System.currentTimeMillis();
     long alreadyWait = 0;
-    while (stronglyAcceptedNodeNum < quorumSize
-        && (!ENABLE_WEAK_ACCEPTANCE
-            || (totalAccepted < quorumSize)
-            || votingLogList.size() > config.getMaxNumOfLogsInMem())
-        && alreadyWait < RaftServer.getWriteOperationTimeoutMS()
-        && !log.getStronglyAcceptedNodeIds().contains(Integer.MAX_VALUE)) {
+    while (log.getLog().getCurrLogIndex() == Long.MIN_VALUE
+        || (stronglyAcceptedNodeNum < quorumSize
+            && (!ENABLE_WEAK_ACCEPTANCE
+                || (totalAccepted < quorumSize)
+                || votingLogList.size() > config.getMaxNumOfLogsInMem())
+            && alreadyWait < RaftServer.getWriteOperationTimeoutMS()
+            && !log.getStronglyAcceptedNodeIds().contains(Integer.MAX_VALUE))) {
       long singleWaitTime = 0;
       long singleWaitStart = System.nanoTime();
       try {
@@ -1645,7 +1657,6 @@ public abstract class RaftMember {
           log.getStronglyAcceptedNodeIds().wait(1);
         }
         singleWaitTime = (System.nanoTime() - singleWaitStart) / 1000000;
-        logger.debug("{} ends waiting", log);
       } catch (InterruptedException e) {
         Thread.currentThread().interrupt();
         logger.warn("Unexpected interruption when sending a log", e);
@@ -1695,11 +1706,12 @@ public abstract class RaftMember {
     int weaklyAcceptedNodeNum = log.getWeaklyAcceptedNodeIds().size();
     int totalAccepted = stronglyAcceptedNodeNum + weaklyAcceptedNodeNum;
 
-    if (stronglyAcceptedNodeNum < quorumSize
-        && (!ENABLE_WEAK_ACCEPTANCE
-            || (totalAccepted < quorumSize)
-            || votingLogList.size() > config.getMaxNumOfLogsInMem())
-        && !log.getStronglyAcceptedNodeIds().contains(Integer.MAX_VALUE)) {
+    if (log.getLog().getCurrLogIndex() == Long.MIN_VALUE
+        || (stronglyAcceptedNodeNum < quorumSize
+            && (!ENABLE_WEAK_ACCEPTANCE
+                || (totalAccepted < quorumSize)
+                || votingLogList.size() > config.getMaxNumOfLogsInMem())
+            && !log.getStronglyAcceptedNodeIds().contains(Integer.MAX_VALUE))) {
       waitAppendResultLoop(log, quorumSize);
     }
 
@@ -2134,6 +2146,8 @@ public abstract class RaftMember {
       } finally {
         ClientUtils.putBackSyncClient(client);
       }
+    } else {
+      logger.debug("Skip sending {} because cannot get client of {}", log, node);
     }
   }
 
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/server/monitor/Timer.java b/cluster/src/main/java/org/apache/iotdb/cluster/server/monitor/Timer.java
index c72550e..5a59d6e 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/server/monitor/Timer.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/monitor/Timer.java
@@ -334,7 +334,7 @@ public class Timer {
      * method to avoid unnecessary calls when instrumenting is disabled.
      */
     public void calOperationCostTimeFromStart(long startTime) {
-      if (ENABLE_INSTRUMENTING && startTime != Long.MIN_VALUE) {
+      if (ENABLE_INSTRUMENTING && startTime != Long.MIN_VALUE && startTime != 0) {
         long consumed = System.nanoTime() - startTime;
         add(consumed);
         if (consumed >= warningThreshold && logger.isWarnEnabled()) {