You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ratis.apache.org by wi...@apache.org on 2024/03/27 06:10:59 UTC

(ratis) branch snapshot-3 updated (e6f255461 -> 467a27ac6)

This is an automated email from the ASF dual-hosted git repository.

williamsong pushed a change to branch snapshot-3
in repository https://gitbox.apache.org/repos/asf/ratis.git


    from e6f255461 Change version for the version 3.0.1 -rc0
     new cc038aea9 RATIS-2013. OrderedAsync retry results an IllegalStateException in GrpcClientProtocolService. (#1026)
     new 1691a3c0f RATIS-2012. Client should not retry after close. (#1025)
     new f8f12d890 RATIS-2008. Follower should recognize candidate if the candidate is the same peer as the current valid leader (#1024)
     new 5baff19ff RATIS-2017: checkAndEvictCache must be done in writeLock. (#1031)
     new 5bb9e32e3 RATIS-2019 Fixed abnormal exit of StateMachineUpdater (#1033)
     new d2e114eda RATIS-2044. Fix ReadIndex loss caused by data race in AppendEntriesListeners (#1052)
     new 50a5f18a0 RATIS-2047. Avoid unnecessary warn log when creating raft group (#1054)
     new ba6e1604d RATIS-2048. Fix SimpleStateMachineStorage potential NPE exception (#1056)
     new 21271d600 RATIS-2036. Avoid trigger snapshot when removing raftGroup (#1055)
     new cda87cee9 RATIS-2049. Suppress Sonar warning for volatile objects (#1057)
     new 52e8db697 RATIS-2050. Add creationGap param to snapshot management API (#1058)
     new 467a27ac6 update snapshot version

The 12 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 pom.xml                                            |   4 +-
 ratis-assembly/pom.xml                             |   2 +-
 ratis-client/pom.xml                               |   2 +-
 .../ratis/client/api/SnapshotManagementApi.java    |  22 ++-
 .../org/apache/ratis/client/impl/BlockingImpl.java |  12 +-
 .../apache/ratis/client/impl/ClientProtoUtils.java |   5 +-
 .../org/apache/ratis/client/impl/OrderedAsync.java | 101 ++++------
 .../apache/ratis/client/impl/RaftClientImpl.java   |  49 +++--
 .../ratis/client/impl/SnapshotManagementImpl.java  |   5 +-
 .../apache/ratis/client/impl/UnorderedAsync.java   |  14 +-
 .../ratis/client/retry/ClientRetryEvent.java       |  24 +--
 ratis-common/pom.xml                               |   2 +-
 .../ratis/protocol/SnapshotManagementRequest.java  |  18 +-
 .../main/java/org/apache/ratis/util/JavaUtils.java |   4 +
 .../apache/ratis/util/MemoizedCheckedSupplier.java |   2 +
 .../org/apache/ratis/util/MemoizedSupplier.java    |   1 +
 .../java/org/apache/ratis/util/PeerProxyMap.java   |  17 +-
 .../java/org/apache/ratis/util/SlidingWindow.java  |  18 +-
 ratis-docs/pom.xml                                 |   2 +-
 ratis-examples/pom.xml                             |   2 +-
 .../apache/ratis/examples/debug/server/Server.java |   1 +
 .../ratis/examples/filestore/cli/Client.java       |   3 +
 .../ratis/examples/membership/server/CServer.java  |   4 +-
 ratis-experiments/pom.xml                          |   2 +-
 ratis-grpc/pom.xml                                 |   2 +-
 .../apache/ratis/grpc/server/GrpcLogAppender.java  |   4 +-
 ratis-metrics-api/pom.xml                          |   2 +-
 ratis-metrics-default/pom.xml                      |   2 +-
 ratis-metrics-dropwizard3/pom.xml                  |   2 +-
 ratis-netty/pom.xml                                |   2 +-
 ratis-proto/pom.xml                                |   2 +-
 ratis-proto/src/main/proto/Raft.proto              |   2 +-
 ratis-replicated-map/pom.xml                       |   2 +-
 ratis-resource-bundle/pom.xml                      |   2 +-
 ratis-server-api/pom.xml                           |   2 +-
 .../apache/ratis/server/RaftServerConfigKeys.java  |  12 ++
 .../apache/ratis/server/leader/LogAppender.java    |   3 +-
 ratis-server/pom.xml                               |   2 +-
 .../apache/ratis/server/impl/LeaderStateImpl.java  |  26 ++-
 .../apache/ratis/server/impl/RaftServerImpl.java   |   6 +-
 .../ratis/server/impl/ReadIndexHeartbeats.java     |  21 +-
 .../apache/ratis/server/impl/RetryCacheImpl.java   |   2 +-
 .../ratis/server/impl/StateMachineUpdater.java     |  28 ++-
 .../org/apache/ratis/server/impl/VoteContext.java  |   4 +-
 .../server/raftlog/segmented/SegmentedRaftLog.java |  12 +-
 .../ratis/server/storage/RaftStorageImpl.java      |  17 +-
 .../impl/SimpleStateMachineStorage.java            |   2 +-
 .../test/java/org/apache/ratis/RaftAsyncTests.java |   2 +-
 .../test/java/org/apache/ratis/RaftBasicTests.java |  14 +-
 .../test/java/org/apache/ratis/RaftTestUtil.java   |  30 ++-
 ratis-shell/pom.xml                                |   2 +-
 ratis-test/pom.xml                                 |   2 +-
 .../org/apache/ratis/RaftLogTruncateTests.java     | 216 +++++++++++++++++++++
 ...hGrpc.java => TestRaftLogTruncateWithGrpc.java} |   6 +-
 .../org/apache/ratis/grpc/TestRaftWithGrpc.java    |   6 -
 .../ratis/retry/TestExceptionDependentRetry.java   |  11 +-
 .../org/apache/ratis/retry/TestRetryPolicy.java    |  20 +-
 .../apache/ratis/server/ServerRestartTests.java    |   2 +-
 .../ratis/server/storage/TestRaftStorage.java      |   1 +
 ratis-tools/pom.xml                                |   2 +-
 60 files changed, 556 insertions(+), 233 deletions(-)
 create mode 100644 ratis-test/src/test/java/org/apache/ratis/RaftLogTruncateTests.java
 copy ratis-test/src/test/java/org/apache/ratis/grpc/{TestRaftAsyncWithGrpc.java => TestRaftLogTruncateWithGrpc.java} (86%)


(ratis) 10/12: RATIS-2049. Suppress Sonar warning for volatile objects (#1057)

Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

williamsong pushed a commit to branch snapshot-3
in repository https://gitbox.apache.org/repos/asf/ratis.git

commit cda87cee939dad462da08150c74bd19d11bba56a
Author: Potato <ta...@apache.org>
AuthorDate: Mon Mar 25 23:12:57 2024 +0800

    RATIS-2049. Suppress Sonar warning for volatile objects (#1057)
---
 .../src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java    | 1 +
 .../src/main/java/org/apache/ratis/util/MemoizedCheckedSupplier.java  | 2 ++
 .../src/main/java/org/apache/ratis/util/MemoizedSupplier.java         | 1 +
 .../src/main/java/org/apache/ratis/examples/debug/server/Server.java  | 1 +
 .../src/main/java/org/apache/ratis/examples/filestore/cli/Client.java | 3 +++
 .../java/org/apache/ratis/examples/membership/server/CServer.java     | 4 +---
 .../src/main/java/org/apache/ratis/grpc/server/GrpcLogAppender.java   | 4 ++--
 .../src/main/java/org/apache/ratis/server/leader/LogAppender.java     | 3 ++-
 .../test/java/org/apache/ratis/server/storage/TestRaftStorage.java    | 1 +
 9 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java
index 1b82709da..db789aef2 100644
--- a/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java
+++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java
@@ -172,6 +172,7 @@ public final class RaftClientImpl implements RaftClient {
   private final RaftGroupId groupId;
   private final RetryPolicy retryPolicy;
 
+  @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type
   private volatile RaftPeerId leaderId;
   /** The callIds of the replied requests. */
   private final RepliedCallIds repliedCallIds;
diff --git a/ratis-common/src/main/java/org/apache/ratis/util/MemoizedCheckedSupplier.java b/ratis-common/src/main/java/org/apache/ratis/util/MemoizedCheckedSupplier.java
index cf2d06023..0e9ae44fa 100644
--- a/ratis-common/src/main/java/org/apache/ratis/util/MemoizedCheckedSupplier.java
+++ b/ratis-common/src/main/java/org/apache/ratis/util/MemoizedCheckedSupplier.java
@@ -44,6 +44,8 @@ public final class MemoizedCheckedSupplier<RETURN, THROW extends Throwable>
   }
 
   private final CheckedSupplier<RETURN, THROW> initializer;
+
+  @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type
   private volatile RETURN value = null;
 
   /**
diff --git a/ratis-common/src/main/java/org/apache/ratis/util/MemoizedSupplier.java b/ratis-common/src/main/java/org/apache/ratis/util/MemoizedSupplier.java
index f179d2dcd..5c2754a30 100644
--- a/ratis-common/src/main/java/org/apache/ratis/util/MemoizedSupplier.java
+++ b/ratis-common/src/main/java/org/apache/ratis/util/MemoizedSupplier.java
@@ -40,6 +40,7 @@ public final class MemoizedSupplier<T> implements Supplier<T> {
   }
 
   private final Supplier<T> initializer;
+  @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type
   private volatile T value = null;
 
   /**
diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/debug/server/Server.java b/ratis-examples/src/main/java/org/apache/ratis/examples/debug/server/Server.java
index 4377a1420..07f7b20f4 100644
--- a/ratis-examples/src/main/java/org/apache/ratis/examples/debug/server/Server.java
+++ b/ratis-examples/src/main/java/org/apache/ratis/examples/debug/server/Server.java
@@ -34,6 +34,7 @@ public final class Server {
   private Server(){
   }
 
+  @SuppressWarnings({"squid:S2095"}) // Suppress closeable  warning
   public static void main(String[] args) throws IOException {
     if (args.length < 1) {
       System.err.println("The arguments should be <ip:port>");
diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Client.java b/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Client.java
index 086b791e4..9e2dbd0cc 100644
--- a/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Client.java
+++ b/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Client.java
@@ -162,6 +162,9 @@ public abstract class Client extends SubCommandBase {
       Process pro = Runtime.getRuntime().exec(cmds);
       pro.waitFor();
     } catch (Throwable t) {
+      if (t instanceof InterruptedException) {
+        Thread.currentThread().interrupt();
+      }
       System.err.println("Failed to run command:" + Arrays.toString(cmds) + ":" + t.getMessage());
     }
   }
diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/membership/server/CServer.java b/ratis-examples/src/main/java/org/apache/ratis/examples/membership/server/CServer.java
index a846cd1e7..2145412e0 100644
--- a/ratis-examples/src/main/java/org/apache/ratis/examples/membership/server/CServer.java
+++ b/ratis-examples/src/main/java/org/apache/ratis/examples/membership/server/CServer.java
@@ -32,7 +32,6 @@ import org.apache.ratis.server.storage.RaftStorage;
 import org.apache.ratis.thirdparty.com.google.common.base.MoreObjects;
 import org.apache.ratis.util.FileUtils;
 
-import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
 import java.util.Collections;
@@ -40,7 +39,7 @@ import java.util.Collections;
 /**
  * A simple raft server using {@link CounterStateMachine}.
  */
-public class CServer implements Closeable {
+public class CServer {
   public static final RaftGroupId GROUP_ID = RaftGroupId.randomId();
   public static final String LOCAL_ADDR = "0.0.0.0";
 
@@ -78,7 +77,6 @@ public class CServer implements Closeable {
     return server.getPeer();
   }
 
-  @Override
   public void close() throws IOException {
     server.close();
     FileUtils.deleteFully(storageDir);
diff --git a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcLogAppender.java b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcLogAppender.java
index 9a2e84f21..bdf9fdc35 100644
--- a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcLogAppender.java
+++ b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcLogAppender.java
@@ -152,7 +152,6 @@ public class GrpcLogAppender extends LogAppenderBase {
   private final TimeDuration logMessageBatchDuration;
   private final int maxOutstandingInstallSnapshots;
   private final TimeoutExecutor scheduler = TimeoutExecutor.getInstance();
-
   private volatile StreamObservers appendLogRequestObserver;
   private final boolean useSeparateHBChannel;
 
@@ -839,6 +838,7 @@ public class GrpcLogAppender extends LogAppenderBase {
 
   static class AppendEntriesRequest {
     private final Timekeeper timer;
+    @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type
     private volatile Timekeeper.Context timerContext;
 
     private final long callId;
@@ -847,7 +847,7 @@ public class GrpcLogAppender extends LogAppenderBase {
 
     private final TermIndex firstEntry;
     private final TermIndex lastEntry;
-
+    @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type
     private volatile Timestamp sendTime;
 
     AppendEntriesRequest(AppendEntriesRequestProto proto, RaftPeerId followerId, GrpcServerMetrics grpcServerMetrics) {
diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/leader/LogAppender.java b/ratis-server-api/src/main/java/org/apache/ratis/server/leader/LogAppender.java
index 020a352c0..36331e3ab 100644
--- a/ratis-server-api/src/main/java/org/apache/ratis/server/leader/LogAppender.java
+++ b/ratis-server-api/src/main/java/org/apache/ratis/server/leader/LogAppender.java
@@ -87,10 +87,11 @@ public interface LogAppender {
   default CompletableFuture<?> stopAsync() {
     stop();
     return CompletableFuture.supplyAsync(() -> {
-      for (; isRunning(); ) {
+      while (isRunning()) {
         try {
           Thread.sleep(10);
         } catch (InterruptedException e) {
+          Thread.currentThread().interrupt();
           throw new CompletionException("stopAsync interrupted", e);
         }
       }
diff --git a/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java b/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java
index 4465cfdf3..5f48e43f7 100644
--- a/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java
+++ b/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java
@@ -83,6 +83,7 @@ public class TestRaftStorage extends BaseTest {
     return impl;
   }
 
+  @SuppressWarnings({"squid:S5783"}) // Suppress same exception warning
   @Test
   public void testNotExistent() throws IOException {
     FileUtils.deleteFully(storageDir);


(ratis) 06/12: RATIS-2044. Fix ReadIndex loss caused by data race in AppendEntriesListeners (#1052)

Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

williamsong pushed a commit to branch snapshot-3
in repository https://gitbox.apache.org/repos/asf/ratis.git

commit d2e114eda3219d0e114fe1f58b81d3da28e2c37e
Author: William Song <48...@users.noreply.github.com>
AuthorDate: Mon Mar 18 23:15:14 2024 +0800

    RATIS-2044. Fix ReadIndex loss caused by data race in AppendEntriesListeners (#1052)
---
 .../org/apache/ratis/server/impl/ReadIndexHeartbeats.java    | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/ReadIndexHeartbeats.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/ReadIndexHeartbeats.java
index d08a1ea40..4ff1460d7 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/ReadIndexHeartbeats.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/ReadIndexHeartbeats.java
@@ -23,6 +23,7 @@ import org.apache.ratis.server.leader.LogAppender;
 import org.apache.ratis.server.raftlog.RaftLog;
 import org.apache.ratis.server.raftlog.RaftLogIndex;
 import org.apache.ratis.util.JavaUtils;
+import org.apache.ratis.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -123,8 +124,15 @@ class ReadIndexHeartbeats {
 
   class AppendEntriesListeners {
     private final NavigableMap<Long, AppendEntriesListener> sorted = new TreeMap<>();
+    private Exception exception = null;
 
     synchronized AppendEntriesListener add(long commitIndex, Function<Long, AppendEntriesListener> constructor) {
+      if (exception != null) {
+        Preconditions.assertTrue(sorted.isEmpty());
+        final AppendEntriesListener listener = constructor.apply(commitIndex);
+        listener.getFuture().completeExceptionally(exception);
+        return listener;
+      }
       return sorted.computeIfAbsent(commitIndex, constructor);
     }
 
@@ -152,6 +160,10 @@ class ReadIndexHeartbeats {
     }
 
     synchronized void failAll(Exception e) {
+      if (exception != null) {
+        return;
+      }
+      exception = e;
       sorted.forEach((index, listener) -> listener.getFuture().completeExceptionally(e));
       sorted.clear();
     }


(ratis) 12/12: update snapshot version

Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

williamsong pushed a commit to branch snapshot-3
in repository https://gitbox.apache.org/repos/asf/ratis.git

commit 467a27ac668ef1e109d9462b3b2d3cda9fcbbeb5
Author: szywilliam <sz...@163.com>
AuthorDate: Wed Mar 27 14:10:49 2024 +0800

    update snapshot version
---
 pom.xml                           | 4 ++--
 ratis-assembly/pom.xml            | 2 +-
 ratis-client/pom.xml              | 2 +-
 ratis-common/pom.xml              | 2 +-
 ratis-docs/pom.xml                | 2 +-
 ratis-examples/pom.xml            | 2 +-
 ratis-experiments/pom.xml         | 2 +-
 ratis-grpc/pom.xml                | 2 +-
 ratis-metrics-api/pom.xml         | 2 +-
 ratis-metrics-default/pom.xml     | 2 +-
 ratis-metrics-dropwizard3/pom.xml | 2 +-
 ratis-netty/pom.xml               | 2 +-
 ratis-proto/pom.xml               | 2 +-
 ratis-replicated-map/pom.xml      | 2 +-
 ratis-resource-bundle/pom.xml     | 2 +-
 ratis-server-api/pom.xml          | 2 +-
 ratis-server/pom.xml              | 2 +-
 ratis-shell/pom.xml               | 2 +-
 ratis-test/pom.xml                | 2 +-
 ratis-tools/pom.xml               | 2 +-
 20 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/pom.xml b/pom.xml
index dae8c9c2f..33d2d43c9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -24,7 +24,7 @@
 
   <artifactId>ratis</artifactId>
   <groupId>org.apache.ratis</groupId>
-  <version>3.0.1</version>
+  <version>3.1.0-52e8db6-SNAPSHOT</version>
   <name>Apache Ratis</name>
   <packaging>pom</packaging>
   <description>
@@ -162,7 +162,7 @@
 
 
   <properties>
-    <project.build.outputTimestamp>2023-12-29T08:16:24Z</project.build.outputTimestamp>
+    <project.build.outputTimestamp>2024-03-27T06:10:39Z</project.build.outputTimestamp>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
      <!-- where to find the generated LICENSE files -->
diff --git a/ratis-assembly/pom.xml b/ratis-assembly/pom.xml
index f779af98b..f8e7a2ab4 100644
--- a/ratis-assembly/pom.xml
+++ b/ratis-assembly/pom.xml
@@ -17,7 +17,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-assembly</artifactId>
diff --git a/ratis-client/pom.xml b/ratis-client/pom.xml
index 0383e867a..d3935f1f0 100644
--- a/ratis-client/pom.xml
+++ b/ratis-client/pom.xml
@@ -17,7 +17,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-client</artifactId>
diff --git a/ratis-common/pom.xml b/ratis-common/pom.xml
index 4fca64bad..ae8a00bb6 100644
--- a/ratis-common/pom.xml
+++ b/ratis-common/pom.xml
@@ -17,7 +17,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-common</artifactId>
diff --git a/ratis-docs/pom.xml b/ratis-docs/pom.xml
index 0371a840c..53f143cf3 100644
--- a/ratis-docs/pom.xml
+++ b/ratis-docs/pom.xml
@@ -20,7 +20,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-docs</artifactId>
diff --git a/ratis-examples/pom.xml b/ratis-examples/pom.xml
index 2c18a3efb..060ede244 100644
--- a/ratis-examples/pom.xml
+++ b/ratis-examples/pom.xml
@@ -17,7 +17,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-examples</artifactId>
diff --git a/ratis-experiments/pom.xml b/ratis-experiments/pom.xml
index eb8df3520..32e9af5c3 100644
--- a/ratis-experiments/pom.xml
+++ b/ratis-experiments/pom.xml
@@ -17,7 +17,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-experiments</artifactId>
diff --git a/ratis-grpc/pom.xml b/ratis-grpc/pom.xml
index 56727129c..bcc0b816d 100644
--- a/ratis-grpc/pom.xml
+++ b/ratis-grpc/pom.xml
@@ -17,7 +17,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-grpc</artifactId>
diff --git a/ratis-metrics-api/pom.xml b/ratis-metrics-api/pom.xml
index cb77992be..a81b21b4e 100644
--- a/ratis-metrics-api/pom.xml
+++ b/ratis-metrics-api/pom.xml
@@ -18,7 +18,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-metrics-api</artifactId>
diff --git a/ratis-metrics-default/pom.xml b/ratis-metrics-default/pom.xml
index de5b48ae2..fb2961e82 100644
--- a/ratis-metrics-default/pom.xml
+++ b/ratis-metrics-default/pom.xml
@@ -18,7 +18,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-metrics-default</artifactId>
diff --git a/ratis-metrics-dropwizard3/pom.xml b/ratis-metrics-dropwizard3/pom.xml
index e87b66fea..5cc53a113 100644
--- a/ratis-metrics-dropwizard3/pom.xml
+++ b/ratis-metrics-dropwizard3/pom.xml
@@ -18,7 +18,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-metrics-dropwizard3</artifactId>
diff --git a/ratis-netty/pom.xml b/ratis-netty/pom.xml
index faf80be2d..c3b20570e 100644
--- a/ratis-netty/pom.xml
+++ b/ratis-netty/pom.xml
@@ -17,7 +17,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-netty</artifactId>
diff --git a/ratis-proto/pom.xml b/ratis-proto/pom.xml
index 90e6a0546..1ccfbd6ca 100644
--- a/ratis-proto/pom.xml
+++ b/ratis-proto/pom.xml
@@ -17,7 +17,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-proto</artifactId>
diff --git a/ratis-replicated-map/pom.xml b/ratis-replicated-map/pom.xml
index 9b14023dd..52ad2a59d 100644
--- a/ratis-replicated-map/pom.xml
+++ b/ratis-replicated-map/pom.xml
@@ -17,7 +17,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-replicated-map</artifactId>
diff --git a/ratis-resource-bundle/pom.xml b/ratis-resource-bundle/pom.xml
index c1721bed6..84bc6670d 100644
--- a/ratis-resource-bundle/pom.xml
+++ b/ratis-resource-bundle/pom.xml
@@ -23,7 +23,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
     <relativePath>..</relativePath>
   </parent>
 
diff --git a/ratis-server-api/pom.xml b/ratis-server-api/pom.xml
index 4368d8cf0..a93a82432 100644
--- a/ratis-server-api/pom.xml
+++ b/ratis-server-api/pom.xml
@@ -17,7 +17,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-server-api</artifactId>
diff --git a/ratis-server/pom.xml b/ratis-server/pom.xml
index f76bd5a11..aafd1d567 100644
--- a/ratis-server/pom.xml
+++ b/ratis-server/pom.xml
@@ -17,7 +17,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-server</artifactId>
diff --git a/ratis-shell/pom.xml b/ratis-shell/pom.xml
index 0b0e35c74..036f5795f 100644
--- a/ratis-shell/pom.xml
+++ b/ratis-shell/pom.xml
@@ -17,7 +17,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-shell</artifactId>
diff --git a/ratis-test/pom.xml b/ratis-test/pom.xml
index 20f281de9..0807d4df9 100644
--- a/ratis-test/pom.xml
+++ b/ratis-test/pom.xml
@@ -17,7 +17,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-test</artifactId>
diff --git a/ratis-tools/pom.xml b/ratis-tools/pom.xml
index fa54b0fa1..d03c37995 100644
--- a/ratis-tools/pom.xml
+++ b/ratis-tools/pom.xml
@@ -17,7 +17,7 @@
   <parent>
     <artifactId>ratis</artifactId>
     <groupId>org.apache.ratis</groupId>
-    <version>3.0.1</version>
+    <version>3.1.0-52e8db6-SNAPSHOT</version>
   </parent>
 
   <artifactId>ratis-tools</artifactId>


(ratis) 09/12: RATIS-2036. Avoid trigger snapshot when removing raftGroup (#1055)

Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

williamsong pushed a commit to branch snapshot-3
in repository https://gitbox.apache.org/repos/asf/ratis.git

commit 21271d60029e6e3f03d7651378974ece7bef519a
Author: Potato <ta...@apache.org>
AuthorDate: Sun Mar 24 02:11:39 2024 +0800

    RATIS-2036. Avoid trigger snapshot when removing raftGroup (#1055)
---
 .../apache/ratis/server/RaftServerConfigKeys.java  | 12 ++++++++++
 .../apache/ratis/server/impl/RaftServerImpl.java   |  1 +
 .../ratis/server/impl/StateMachineUpdater.java     | 28 +++++++++++++++++++++-
 3 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java b/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java
index 565e88126..7419ca095 100644
--- a/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java
+++ b/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java
@@ -647,6 +647,18 @@ public interface RaftServerConfigKeys {
       setBoolean(properties::setBoolean, TRIGGER_WHEN_STOP_ENABLED_KEY, triggerWhenStopEnabled);
     }
 
+    /** whether trigger snapshot when remove raft server */
+    String TRIGGER_WHEN_REMOVE_ENABLED_KEY = PREFIX + ".trigger-when-remove.enabled";
+    /** by default let the state machine to trigger snapshot when remove */
+    boolean TRIGGER_WHEN_REMOVE_ENABLED_DEFAULT = true;
+    static boolean triggerWhenRemoveEnabled(RaftProperties properties) {
+      return getBoolean(properties::getBoolean,
+          TRIGGER_WHEN_REMOVE_ENABLED_KEY, TRIGGER_WHEN_REMOVE_ENABLED_DEFAULT, getDefaultLog());
+    }
+    static void setTriggerWhenRemoveEnabled(RaftProperties properties, boolean triggerWhenRemoveEnabled) {
+      setBoolean(properties::setBoolean, TRIGGER_WHEN_REMOVE_ENABLED_KEY, triggerWhenRemoveEnabled);
+    }
+
     /** The log index gap between to two snapshot creations. */
     String CREATION_GAP_KEY = PREFIX + ".creation.gap";
     long CREATION_GAP_DEFAULT = 1024;
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
index ed5457b3d..4ae60acaa 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
@@ -487,6 +487,7 @@ class RaftServerImpl implements RaftServer.Division,
     final RaftStorageDirectory dir = state.getStorage().getStorageDir();
 
     /* Shutdown is triggered here inorder to avoid any locked files. */
+    state.getStateMachineUpdater().setRemoving();
     close();
     getStateMachine().event().notifyGroupRemove();
     if (deleteDirectory) {
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/StateMachineUpdater.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/StateMachineUpdater.java
index 43fbdd884..5affbc3a7 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/StateMachineUpdater.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/StateMachineUpdater.java
@@ -74,6 +74,8 @@ class StateMachineUpdater implements Runnable {
 
   private final boolean triggerSnapshotWhenStopEnabled;
 
+  private final boolean triggerSnapshotWhenRemoveEnabled;
+
   private final Long autoSnapshotThreshold;
   private final boolean purgeUptoSnapshotIndex;
 
@@ -91,6 +93,8 @@ class StateMachineUpdater implements Runnable {
 
   private final Consumer<Long> appliedIndexConsumer;
 
+  private volatile boolean isRemoving;
+
   StateMachineUpdater(StateMachine stateMachine, RaftServerImpl server,
       ServerState serverState, long lastAppliedIndex, RaftProperties properties, Consumer<Long> appliedIndexConsumer) {
     this.name = serverState.getMemberId() + "-" + JavaUtils.getClassSimpleName(getClass());
@@ -106,6 +110,7 @@ class StateMachineUpdater implements Runnable {
     this.snapshotIndex = new RaftLogIndex("snapshotIndex", lastAppliedIndex);
 
     this.triggerSnapshotWhenStopEnabled = RaftServerConfigKeys.Snapshot.triggerWhenStopEnabled(properties);
+    this.triggerSnapshotWhenRemoveEnabled = RaftServerConfigKeys.Snapshot.triggerWhenRemoveEnabled(properties);
     final boolean autoSnapshot = RaftServerConfigKeys.Snapshot.autoTriggerEnabled(properties);
     this.autoSnapshotThreshold = autoSnapshot? RaftServerConfigKeys.Snapshot.autoTriggerThreshold(properties): null;
     final int numSnapshotFilesRetained = RaftServerConfigKeys.Snapshot.retentionFileNum(properties);
@@ -320,12 +325,33 @@ class StateMachineUpdater implements Runnable {
     if (autoSnapshotThreshold == null) {
       return false;
     } else if (shouldStop()) {
-      return triggerSnapshotWhenStopEnabled && getLastAppliedIndex() - snapshotIndex.get() > 0;
+      return shouldTakeSnapshotAtStop() && getLastAppliedIndex() - snapshotIndex.get() > 0;
     }
     return state == State.RUNNING &&
         getStateMachineLastAppliedIndex() - snapshotIndex.get() >= autoSnapshotThreshold;
   }
 
+  /**
+   * In view of the three variables triggerSnapshotWhenStopEnabled, triggerSnapshotWhenRemoveEnabled and isRemoving,
+   * we can draw the following 8 combination:
+   * true true true => true
+   * true true false => true
+   * true false true => false
+   * true false false => true
+   * false true true => true
+   * false true false =>  false
+   * false false true => false
+   * false false false => false
+   * @return result
+   */
+  private boolean shouldTakeSnapshotAtStop() {
+    return isRemoving ? triggerSnapshotWhenRemoveEnabled : triggerSnapshotWhenStopEnabled;
+  }
+
+  void setRemoving() {
+    this.isRemoving = true;
+  }
+
   private long getLastAppliedIndex() {
     return appliedIndex.get();
   }


(ratis) 05/12: RATIS-2019 Fixed abnormal exit of StateMachineUpdater (#1033)

Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

williamsong pushed a commit to branch snapshot-3
in repository https://gitbox.apache.org/repos/asf/ratis.git

commit 5bb9e32e3bbf26eeb04b8f759544e4fa100d38a9
Author: Potato <ta...@apache.org>
AuthorDate: Wed Jan 31 00:12:23 2024 +0800

    RATIS-2019 Fixed abnormal exit of StateMachineUpdater (#1033)
---
 .../main/java/org/apache/ratis/util/JavaUtils.java |  4 ++++
 .../apache/ratis/server/impl/LeaderStateImpl.java  | 26 +++++++++-------------
 .../ratis/server/impl/ReadIndexHeartbeats.java     |  9 +++-----
 .../apache/ratis/server/impl/RetryCacheImpl.java   |  2 +-
 4 files changed, 19 insertions(+), 22 deletions(-)

diff --git a/ratis-common/src/main/java/org/apache/ratis/util/JavaUtils.java b/ratis-common/src/main/java/org/apache/ratis/util/JavaUtils.java
index 00725903a..5a977a45b 100644
--- a/ratis-common/src/main/java/org/apache/ratis/util/JavaUtils.java
+++ b/ratis-common/src/main/java/org/apache/ratis/util/JavaUtils.java
@@ -283,6 +283,10 @@ public interface JavaUtils {
     return future;
   }
 
+  static boolean isCompletedNormally(CompletableFuture<?> future) {
+    return future.isDone() && !future.isCancelled() && !future.isCompletedExceptionally();
+  }
+
   static Throwable unwrapCompletionException(Throwable t) {
     Objects.requireNonNull(t, "t == null");
     return t instanceof CompletionException && t.getCause() != null? t.getCause(): t;
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderStateImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderStateImpl.java
index c6983e331..34a28abe4 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderStateImpl.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderStateImpl.java
@@ -310,19 +310,16 @@ class LeaderStateImpl implements LeaderState {
       return appliedIndexFuture;
     }
 
-    boolean isApplied(LogEntryProto logEntry) {
-      if (appliedIndexFuture.isDone()) {
-        return true;
-      }
-      final long appliedIndex = logEntry != null? logEntry.getIndex(): server.getState().getLastAppliedIndex();
-      if (appliedIndex >= startIndex) {
-        appliedIndexFuture.complete(appliedIndex);
-        LOG.info("leader is ready since appliedIndex == {} >= startIndex == {}",
-            appliedIndex, startIndex);
-        return true;
-      } else {
-        return false;
+    boolean checkStartIndex(LogEntryProto logEntry) {
+      final boolean completed = logEntry.getIndex() == startIndex && appliedIndexFuture.complete(startIndex);
+      if (completed) {
+        LOG.info("Leader {} is ready since appliedIndex == startIndex == {}", LeaderStateImpl.this, startIndex);
       }
+      return completed;
+    }
+
+    boolean isApplied() {
+      return JavaUtils.isCompletedNormally(appliedIndexFuture);
     }
   }
 
@@ -421,12 +418,11 @@ class LeaderStateImpl implements LeaderState {
   }
 
   boolean isReady() {
-    return startupLogEntry.isInitialized() && startupLogEntry.get().isApplied(null);
+    return startupLogEntry.isInitialized() && startupLogEntry.get().isApplied();
   }
 
   void checkReady(LogEntryProto entry) {
-    Preconditions.assertTrue(startupLogEntry.isInitialized());
-    if (entry.getTerm() == getCurrentTerm() && startupLogEntry.get().isApplied(entry)) {
+    if (entry.getTerm() == server.getState().getCurrentTerm() && startupLogEntry.get().checkStartIndex(entry)) {
       server.getStateMachine().leaderEvent().notifyLeaderReady();
     }
   }
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/ReadIndexHeartbeats.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/ReadIndexHeartbeats.java
index 3f31a2530..d08a1ea40 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/ReadIndexHeartbeats.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/ReadIndexHeartbeats.java
@@ -22,6 +22,7 @@ import org.apache.ratis.protocol.RaftPeerId;
 import org.apache.ratis.server.leader.LogAppender;
 import org.apache.ratis.server.raftlog.RaftLog;
 import org.apache.ratis.server.raftlog.RaftLogIndex;
+import org.apache.ratis.util.JavaUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -99,7 +100,7 @@ class ReadIndexHeartbeats {
 
     boolean receive(LogAppender logAppender, AppendEntriesReplyProto proto,
                     Predicate<Predicate<RaftPeerId>> hasMajority) {
-      if (isCompletedNormally()) {
+      if (JavaUtils.isCompletedNormally(future)) {
         return true;
       }
 
@@ -112,16 +113,12 @@ class ReadIndexHeartbeats {
         }
       }
 
-      return isCompletedNormally();
+      return JavaUtils.isCompletedNormally(future);
     }
 
     boolean isAcknowledged(RaftPeerId id) {
       return Optional.ofNullable(replies.get(id)).filter(HeartbeatAck::isAcknowledged).isPresent();
     }
-
-    boolean isCompletedNormally() {
-      return future.isDone() && !future.isCancelled() && !future.isCompletedExceptionally();
-    }
   }
 
   class AppendEntriesListeners {
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/RetryCacheImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/RetryCacheImpl.java
index a8bac4e5e..50d238b07 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/RetryCacheImpl.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/RetryCacheImpl.java
@@ -67,7 +67,7 @@ class RetryCacheImpl implements RetryCache {
     }
 
     boolean isCompletedNormally() {
-      return !failed && replyFuture.isDone() && !replyFuture.isCompletedExceptionally() && !replyFuture.isCancelled();
+      return !failed && JavaUtils.isCompletedNormally(replyFuture);
     }
 
     void updateResult(RaftClientReply reply) {


(ratis) 08/12: RATIS-2048. Fix SimpleStateMachineStorage potential NPE exception (#1056)

Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

williamsong pushed a commit to branch snapshot-3
in repository https://gitbox.apache.org/repos/asf/ratis.git

commit ba6e1604dacd2040ba4256d9ad59f31059fa6ab1
Author: Potato <ta...@apache.org>
AuthorDate: Sat Mar 23 00:58:31 2024 +0800

    RATIS-2048. Fix SimpleStateMachineStorage potential NPE exception (#1056)
---
 .../org/apache/ratis/statemachine/impl/SimpleStateMachineStorage.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/SimpleStateMachineStorage.java b/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/SimpleStateMachineStorage.java
index 37e94a8a0..88cc57dab 100644
--- a/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/SimpleStateMachineStorage.java
+++ b/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/SimpleStateMachineStorage.java
@@ -116,7 +116,7 @@ public class SimpleStateMachineStorage implements StateMachineStorage {
 
     final List<SingleFileSnapshotInfo> allSnapshotFiles = getSingleFileSnapshotInfos(stateMachineDir.toPath());
 
-    if (allSnapshotFiles.size() > snapshotRetentionPolicy.getNumSnapshotsRetained()) {
+    if (allSnapshotFiles.size() > numSnapshotsRetained) {
       allSnapshotFiles.sort(Comparator.comparing(SingleFileSnapshotInfo::getIndex).reversed());
       allSnapshotFiles.subList(numSnapshotsRetained, allSnapshotFiles.size())
           .stream()


(ratis) 03/12: RATIS-2008. Follower should recognize candidate if the candidate is the same peer as the current valid leader (#1024)

Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

williamsong pushed a commit to branch snapshot-3
in repository https://gitbox.apache.org/repos/asf/ratis.git

commit f8f12d8908e1eabbee3f12f9e84c00c1132345b8
Author: Ivan Andika <36...@users.noreply.github.com>
AuthorDate: Thu Jan 25 01:14:49 2024 +0800

    RATIS-2008. Follower should recognize candidate if the candidate is the same peer as the current valid leader (#1024)
---
 .../src/main/java/org/apache/ratis/server/impl/VoteContext.java       | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/VoteContext.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/VoteContext.java
index 6375d652c..3ba0c0e01 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/VoteContext.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/VoteContext.java
@@ -102,8 +102,10 @@ class VoteContext {
     if (info.isFollower()) {
       final RaftPeerId leader = impl.getState().getLeaderId();
       if (leader != null
+          && !leader.equals(candidateId)
           && impl.getRole().getFollowerState().map(FollowerState::isCurrentLeaderValid).orElse(false)) {
-        return reject("this server is a follower and still has a valid leader " + leader);
+        return reject("this server is a follower and still has a valid leader " + leader
+            + " different than the candidate " + candidateId);
       }
     }
     return true;


(ratis) 07/12: RATIS-2047. Avoid unnecessary warn log when creating raft group (#1054)

Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

williamsong pushed a commit to branch snapshot-3
in repository https://gitbox.apache.org/repos/asf/ratis.git

commit 50a5f18a0f12bd0de08bfd1b29c8602d25e8050c
Author: Potato <ta...@apache.org>
AuthorDate: Sat Mar 23 00:53:30 2024 +0800

    RATIS-2047. Avoid unnecessary warn log when creating raft group (#1054)
---
 .../apache/ratis/server/storage/RaftStorageImpl.java    | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java
index fbb7bf7d4..ce809cad8 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java
@@ -19,7 +19,6 @@ package org.apache.ratis.server.storage;
 
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.nio.file.NoSuchFileException;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ratis.proto.RaftProtos.LogEntryProto;
 import org.apache.ratis.server.RaftConfiguration;
@@ -153,14 +152,16 @@ public class RaftStorageImpl implements RaftStorage {
 
   public RaftConfiguration readRaftConfiguration() {
     File confFile = storageDir.getMetaConfFile();
-    try (InputStream fio = FileUtils.newInputStream(confFile)) {
-      LogEntryProto confProto = LogEntryProto.newBuilder().mergeFrom(fio).build();
-      return LogProtoUtils.toRaftConfiguration(confProto);
-    } catch (FileNotFoundException | NoSuchFileException e) {
-      return null;
-    } catch (Exception e) {
-      LOG.error("Failed reading configuration from file:" + confFile, e);
+    if (!confFile.exists()) {
       return null;
+    } else {
+      try (InputStream fio = FileUtils.newInputStream(confFile)) {
+        LogEntryProto confProto = LogEntryProto.newBuilder().mergeFrom(fio).build();
+        return LogProtoUtils.toRaftConfiguration(confProto);
+      } catch (Exception e) {
+        LOG.error("Failed reading configuration from file:" + confFile, e);
+        return null;
+      }
     }
   }
 


(ratis) 04/12: RATIS-2017: checkAndEvictCache must be done in writeLock. (#1031)

Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

williamsong pushed a commit to branch snapshot-3
in repository https://gitbox.apache.org/repos/asf/ratis.git

commit 5baff19fffa768f2c2cabcd26842f06534262191
Author: Duong Nguyen <du...@gmail.com>
AuthorDate: Sat Jan 27 13:10:03 2024 -0800

    RATIS-2017: checkAndEvictCache must be done in writeLock. (#1031)
---
 .../ratis/server/raftlog/segmented/SegmentedRaftLog.java     | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLog.java b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLog.java
index a729f8e2e..4936525cb 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLog.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLog.java
@@ -324,11 +324,13 @@ public final class SegmentedRaftLog extends RaftLogBase {
 
   private void checkAndEvictCache() {
     if (cache.shouldEvict()) {
-      // TODO if the cache is hitting the maximum size and we cannot evict any
-      // segment's cache, should block the new entry appending or new segment
-      // allocation.
-      cache.evictCache(server.getFollowerNextIndices(), fileLogWorker.getSafeCacheEvictIndex(),
-          server.getLastAppliedIndex());
+      try (AutoCloseableLock ignored = writeLock()){
+        // TODO if the cache is hitting the maximum size and we cannot evict any
+        // segment's cache, should block the new entry appending or new segment
+        // allocation.
+        cache.evictCache(server.getFollowerNextIndices(), fileLogWorker.getSafeCacheEvictIndex(),
+            server.getLastAppliedIndex());
+      }
     }
   }
 


(ratis) 02/12: RATIS-2012. Client should not retry after close. (#1025)

Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

williamsong pushed a commit to branch snapshot-3
in repository https://gitbox.apache.org/repos/asf/ratis.git

commit 1691a3c0f700868e009f5c23393589c6e0d8dac1
Author: Tsz-Wo Nicholas Sze <sz...@apache.org>
AuthorDate: Tue Jan 23 10:01:01 2024 -0800

    RATIS-2012. Client should not retry after close. (#1025)
---
 .../org/apache/ratis/client/impl/BlockingImpl.java |  12 +-
 .../org/apache/ratis/client/impl/OrderedAsync.java | 101 ++++------
 .../apache/ratis/client/impl/RaftClientImpl.java   |  42 ++--
 .../apache/ratis/client/impl/UnorderedAsync.java   |  14 +-
 .../ratis/client/retry/ClientRetryEvent.java       |  24 +--
 .../java/org/apache/ratis/util/PeerProxyMap.java   |  17 +-
 .../test/java/org/apache/ratis/RaftAsyncTests.java |   2 +-
 .../test/java/org/apache/ratis/RaftBasicTests.java |  14 +-
 .../test/java/org/apache/ratis/RaftTestUtil.java   |  30 ++-
 .../org/apache/ratis/RaftLogTruncateTests.java     | 216 +++++++++++++++++++++
 .../ratis/grpc/TestRaftLogTruncateWithGrpc.java    |  24 +++
 .../org/apache/ratis/grpc/TestRaftWithGrpc.java    |   6 -
 .../ratis/retry/TestExceptionDependentRetry.java   |   2 +-
 .../org/apache/ratis/retry/TestRetryPolicy.java    |  20 +-
 .../apache/ratis/server/ServerRestartTests.java    |   2 +-
 15 files changed, 380 insertions(+), 146 deletions(-)

diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/BlockingImpl.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/BlockingImpl.java
index 4be9fa327..76987801b 100644
--- a/ratis-client/src/main/java/org/apache/ratis/client/impl/BlockingImpl.java
+++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/BlockingImpl.java
@@ -119,16 +119,18 @@ class BlockingImpl implements BlockingApi {
         ioe = e;
       }
 
-      pending.incrementExceptionCount(ioe);
-      ClientRetryEvent event = new ClientRetryEvent(request, ioe, pending);
+      if (client.isClosed()) {
+        throw new AlreadyClosedException(this + " is closed.");
+      }
+
+      final ClientRetryEvent event = pending.newClientRetryEvent(request, ioe);
       final RetryPolicy retryPolicy = client.getRetryPolicy();
       final RetryPolicy.Action action = retryPolicy.handleAttemptFailure(event);
-      TimeDuration sleepTime = client.getEffectiveSleepTime(ioe, action.getSleepTime());
-
       if (!action.shouldRetry()) {
-        throw (IOException)client.noMoreRetries(event);
+        throw client.noMoreRetries(event);
       }
 
+      final TimeDuration sleepTime = client.getEffectiveSleepTime(ioe, action.getSleepTime());
       try {
         sleepTime.sleep();
       } catch (InterruptedException e) {
diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/OrderedAsync.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/OrderedAsync.java
index a1aa58681..34dc3be11 100644
--- a/ratis-client/src/main/java/org/apache/ratis/client/impl/OrderedAsync.java
+++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/OrderedAsync.java
@@ -149,10 +149,6 @@ public final class OrderedAsync {
     getSlidingWindow(request).fail(request.getSlidingWindowEntry().getSeqNum(), t);
   }
 
-  private void handleAsyncRetryFailure(ClientRetryEvent event) {
-    failAllAsyncRequests(event.getRequest(), client.noMoreRetries(event));
-  }
-
   CompletableFuture<RaftClientReply> send(RaftClientRequest.Type type, Message message, RaftPeerId server) {
     if (!type.is(TypeCase.WATCH) && !type.is(TypeCase.MESSAGESTREAM)) {
       Objects.requireNonNull(message, "message == null");
@@ -187,85 +183,68 @@ public final class OrderedAsync {
     if (pending == null) {
       return;
     }
-
-    final CompletableFuture<RaftClientReply> f = pending.getReplyFuture();
-    if (f.isDone()) {
+    if (pending.getReplyFuture().isDone()) {
       return;
     }
 
-    final RaftClientRequest request = pending.newRequestImpl();
+    final RaftClientRequest request = pending.newRequest();
     if (request == null) { // already done
-      LOG.debug("{} newRequestImpl returns null", pending);
+      LOG.debug("{} newRequest returns null", pending);
       return;
     }
 
-    final RetryPolicy retryPolicy = client.getRetryPolicy();
-    sendRequest(pending).exceptionally(e -> {
-      if (e instanceof CompletionException) {
-        e = JavaUtils.unwrapCompletionException(e);
-        scheduleWithTimeout(pending, request, retryPolicy, e);
-        return null;
-      }
-      f.completeExceptionally(e);
-      return null;
-    });
-  }
-
-  private void scheduleWithTimeout(PendingOrderedRequest pending,
-      RaftClientRequest request, RetryPolicy retryPolicy, Throwable e) {
-    final int attempt = pending.getAttemptCount();
-    final ClientRetryEvent event = new ClientRetryEvent(request, e, pending);
-    final TimeDuration sleepTime = client.getEffectiveSleepTime(e,
-        retryPolicy.handleAttemptFailure(event).getSleepTime());
-    LOG.debug("schedule* attempt #{} with sleep {} and policy {} for {}", attempt, sleepTime, retryPolicy, request);
-    scheduleWithTimeout(pending, sleepTime, getSlidingWindow(request));
-  }
-
-  private void scheduleWithTimeout(PendingOrderedRequest pending, TimeDuration sleepTime,
-      SlidingWindow.Client<PendingOrderedRequest, RaftClientReply> slidingWindow) {
-    client.getScheduler().onTimeout(sleepTime,
-        () -> slidingWindow.retry(pending, this::sendRequestWithRetry),
-        LOG, () -> "Failed* to retry " + pending);
-  }
-
-  private CompletableFuture<RaftClientReply> sendRequest(PendingOrderedRequest pending) {
-    final RetryPolicy retryPolicy = client.getRetryPolicy();
-    final RaftClientRequest request;
     if (getSlidingWindow((RaftPeerId) null).isFirst(pending.getSeqNum())) {
       pending.setFirstRequest();
     }
-    request = pending.newRequest();
     LOG.debug("{}: send* {}", client.getId(), request);
-    return client.getClientRpc().sendRequestAsync(request).thenApply(reply -> {
+    client.getClientRpc().sendRequestAsync(request).thenAccept(reply -> {
       LOG.debug("{}: receive* {}", client.getId(), reply);
       Objects.requireNonNull(reply, "reply == null");
       client.handleReply(request, reply);
       getSlidingWindow(request).receiveReply(
           request.getSlidingWindowEntry().getSeqNum(), reply, this::sendRequestWithRetry);
-      return reply;
     }).exceptionally(e -> {
       LOG.error(client.getId() + ": Failed* " + request, e);
-      e = JavaUtils.unwrapCompletionException(e);
-      if (e instanceof IOException && !(e instanceof GroupMismatchException)) {
-        pending.incrementExceptionCount(e);
-        final ClientRetryEvent event = new ClientRetryEvent(request, e, pending);
-        if (!retryPolicy.handleAttemptFailure(event).shouldRetry()) {
-          handleAsyncRetryFailure(event);
-        } else {
-          if (e instanceof NotLeaderException) {
-            NotLeaderException nle = (NotLeaderException)e;
-            client.handleNotLeaderException(request, nle, this::resetSlidingWindow);
-          } else {
-            client.handleIOException(request, (IOException) e, null, this::resetSlidingWindow);
-          }
-        }
-        throw new CompletionException(e);
-      }
-      failAllAsyncRequests(request, e);
+      handleException(pending, request, e);
       return null;
     });
   }
 
+  private void handleException(PendingOrderedRequest pending, RaftClientRequest request, Throwable e) {
+    final RetryPolicy retryPolicy = client.getRetryPolicy();
+    if (client.isClosed()) {
+      failAllAsyncRequests(request, new AlreadyClosedException(client + " is closed."));
+      return;
+    }
+
+    e = JavaUtils.unwrapCompletionException(e);
+    if (!(e instanceof IOException) || e instanceof GroupMismatchException) {
+      // non-retryable exceptions
+      failAllAsyncRequests(request, e);
+      return;
+    }
+
+    final ClientRetryEvent event = pending.newClientRetryEvent(request, e);
+    final RetryPolicy.Action action = retryPolicy.handleAttemptFailure(event);
+    if (!action.shouldRetry()) {
+      failAllAsyncRequests(request, client.noMoreRetries(event));
+      return;
+    }
+
+    if (e instanceof NotLeaderException) {
+      client.handleNotLeaderException(request, (NotLeaderException) e, this::resetSlidingWindow);
+    } else {
+      client.handleIOException(request, (IOException) e, null, this::resetSlidingWindow);
+    }
+    final TimeDuration sleepTime = client.getEffectiveSleepTime(e, action.getSleepTime());
+    LOG.debug("schedule* retry with sleep {} for attempt #{} of {}, {}",
+        sleepTime, event.getAttemptCount(), request, retryPolicy);
+    final SlidingWindow.Client<PendingOrderedRequest, RaftClientReply> slidingWindow = getSlidingWindow(request);
+    client.getScheduler().onTimeout(sleepTime,
+        () -> slidingWindow.retry(pending, this::sendRequestWithRetry),
+        LOG, () -> "Failed* to retry " + pending);
+  }
+
   void assertRequestSemaphore(int expectedAvailablePermits, int expectedQueueLength) {
     Preconditions.assertSame(expectedAvailablePermits, requestSemaphore.availablePermits(), "availablePermits");
     Preconditions.assertSame(expectedQueueLength, requestSemaphore.getQueueLength(), "queueLength");
diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java
index f42391947..1b82709da 100644
--- a/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java
+++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java
@@ -44,11 +44,13 @@ import org.apache.ratis.retry.RetryPolicy;
 import org.apache.ratis.thirdparty.com.google.common.cache.Cache;
 import org.apache.ratis.thirdparty.com.google.common.cache.CacheBuilder;
 import org.apache.ratis.util.CollectionUtils;
+import org.apache.ratis.util.IOUtils;
 import org.apache.ratis.util.JavaUtils;
 import org.apache.ratis.util.MemoizedSupplier;
 import org.apache.ratis.util.Preconditions;
 import org.apache.ratis.util.TimeDuration;
 import org.apache.ratis.util.TimeoutExecutor;
+import org.apache.ratis.util.Timestamp;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -65,6 +67,7 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
@@ -79,10 +82,10 @@ public final class RaftClientImpl implements RaftClient {
       .build();
 
   public abstract static class PendingClientRequest {
-    private final long creationTimeInMs = System.currentTimeMillis();
+    private final Timestamp creationTime = Timestamp.currentTime();
     private final CompletableFuture<RaftClientReply> replyFuture = new CompletableFuture<>();
     private final AtomicInteger attemptCount = new AtomicInteger();
-    private final Map<Class<?>, Integer> exceptionCount = new ConcurrentHashMap<>();
+    private final Map<Class<?>, Integer> exceptionCounts = new ConcurrentHashMap<>();
 
     public abstract RaftClientRequest newRequestImpl();
 
@@ -101,19 +104,10 @@ public final class RaftClientImpl implements RaftClient {
       return attemptCount.get();
     }
 
-    int incrementExceptionCount(Throwable t) {
-      return t != null ? exceptionCount.compute(t.getClass(), (k, v) -> v != null ? v + 1 : 1) : 0;
-    }
-
-    public int getExceptionCount(Throwable t) {
-      return t != null ? Optional.ofNullable(exceptionCount.get(t.getClass())).orElse(0) : 0;
-    }
-
-    public boolean isRequestTimeout(TimeDuration timeout) {
-      if (timeout == null) {
-        return false;
-      }
-      return System.currentTimeMillis() - creationTimeInMs > timeout.toLong(TimeUnit.MILLISECONDS);
+    public ClientRetryEvent newClientRetryEvent(RaftClientRequest request, Throwable throwable) {
+      final int exceptionCount = throwable == null? 0
+          : exceptionCounts.compute(throwable.getClass(), (k, v) -> v == null? 1: v+1);
+      return new ClientRetryEvent(getAttemptCount(), request, exceptionCount, throwable, creationTime);
     }
   }
 
@@ -196,6 +190,8 @@ public final class RaftClientImpl implements RaftClient {
   private final ConcurrentMap<RaftPeerId, LeaderElectionManagementApi>
       leaderElectionManagement = new ConcurrentHashMap<>();
 
+  private final AtomicBoolean closed = new AtomicBoolean();
+
   @SuppressWarnings("checkstyle:ParameterNumber")
   RaftClientImpl(ClientId clientId, RaftGroup group, RaftPeerId leaderId, RaftPeer primaryDataStreamServer,
       RaftClientRpc clientRpc, RetryPolicy retryPolicy, RaftProperties properties, Parameters parameters) {
@@ -346,11 +342,11 @@ public final class RaftClientImpl implements RaftClient {
     return dataStreamApi.get();
   }
 
-  Throwable noMoreRetries(ClientRetryEvent event) {
+  IOException noMoreRetries(ClientRetryEvent event) {
     final int attemptCount = event.getAttemptCount();
     final Throwable throwable = event.getCause();
     if (attemptCount == 1 && throwable != null) {
-      return throwable;
+      return IOUtils.asIOException(throwable);
     }
     return new RaftRetryFailureException(event.getRequest(), attemptCount, retryPolicy, throwable);
   }
@@ -418,8 +414,7 @@ public final class RaftClientImpl implements RaftClient {
 
   void handleIOException(RaftClientRequest request, IOException ioe,
       RaftPeerId newLeader, Consumer<RaftClientRequest> handler) {
-    LOG.debug("{}: suggested new leader: {}. Failed {} with {}",
-        clientId, newLeader, request, ioe);
+    LOG.debug("{}: suggested new leader: {}. Failed {}", clientId, newLeader, request, ioe);
     if (LOG.isTraceEnabled()) {
       LOG.trace("Stack trace", new Throwable("TRACE"));
     }
@@ -456,8 +451,17 @@ public final class RaftClientImpl implements RaftClient {
     return clientRpc;
   }
 
+  boolean isClosed() {
+    return closed.get();
+  }
+
   @Override
   public void close() throws IOException {
+    if (!closed.compareAndSet(false, true)) {
+      return;
+    }
+
+    LOG.debug("close {}", getId());
     clientRpc.close();
     if (dataStreamApi.isInitialized()) {
       dataStreamApi.get().close();
diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/UnorderedAsync.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/UnorderedAsync.java
index 84b817b58..eccda4dbd 100644
--- a/ratis-client/src/main/java/org/apache/ratis/client/impl/UnorderedAsync.java
+++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/UnorderedAsync.java
@@ -22,6 +22,7 @@ import org.apache.ratis.client.impl.RaftClientImpl.PendingClientRequest;
 import org.apache.ratis.protocol.ClientId;
 import org.apache.ratis.protocol.Message;
 import org.apache.ratis.protocol.RaftPeerId;
+import org.apache.ratis.protocol.exceptions.AlreadyClosedException;
 import org.apache.ratis.protocol.exceptions.GroupMismatchException;
 import org.apache.ratis.protocol.exceptions.NotLeaderException;
 import org.apache.ratis.protocol.RaftClientReply;
@@ -89,11 +90,14 @@ public interface UnorderedAsync {
         }
 
         final Throwable cause = replyException != null ? replyException : e;
-        pending.incrementExceptionCount(cause);
-        final ClientRetryEvent event = new ClientRetryEvent(request, cause, pending);
+        if (client.isClosed()) {
+          f.completeExceptionally(new AlreadyClosedException(client + " is closed"));
+          return;
+        }
+
+        final ClientRetryEvent event = pending.newClientRetryEvent(request, cause);
         RetryPolicy retryPolicy = client.getRetryPolicy();
         final RetryPolicy.Action action = retryPolicy.handleAttemptFailure(event);
-        TimeDuration sleepTime = client.getEffectiveSleepTime(cause, action.getSleepTime());
         if (!action.shouldRetry()) {
           f.completeExceptionally(client.noMoreRetries(event));
           return;
@@ -124,7 +128,9 @@ public interface UnorderedAsync {
           }
         }
 
-        LOG.debug("schedule retry for attempt #{}, policy={}, request={}", attemptCount, retryPolicy, request);
+        final TimeDuration sleepTime = client.getEffectiveSleepTime(cause, action.getSleepTime());
+        LOG.debug("schedule~ attempt #{} with sleep {} and policy {} for {}",
+            attemptCount, sleepTime, retryPolicy, request);
         client.getScheduler().onTimeout(sleepTime,
             () -> sendRequestWithRetry(pending, client), LOG, () -> clientId + ": Failed~ to retry " + request);
       } catch (Exception ex) {
diff --git a/ratis-client/src/main/java/org/apache/ratis/client/retry/ClientRetryEvent.java b/ratis-client/src/main/java/org/apache/ratis/client/retry/ClientRetryEvent.java
index f0c38efb9..c6a8beb06 100644
--- a/ratis-client/src/main/java/org/apache/ratis/client/retry/ClientRetryEvent.java
+++ b/ratis-client/src/main/java/org/apache/ratis/client/retry/ClientRetryEvent.java
@@ -17,12 +17,11 @@
  */
 package org.apache.ratis.client.retry;
 
-import org.apache.ratis.client.impl.RaftClientImpl.PendingClientRequest;
 import org.apache.ratis.protocol.RaftClientRequest;
 import org.apache.ratis.retry.RetryPolicy;
-import org.apache.ratis.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.ratis.util.JavaUtils;
 import org.apache.ratis.util.TimeDuration;
+import org.apache.ratis.util.Timestamp;
 
 /** An {@link RetryPolicy.Event} specific to client request failure. */
 public class ClientRetryEvent implements RetryPolicy.Event {
@@ -30,23 +29,15 @@ public class ClientRetryEvent implements RetryPolicy.Event {
   private final int causeCount;
   private final RaftClientRequest request;
   private final Throwable cause;
-  private PendingClientRequest pending;
+  private final Timestamp pendingRequestCreationTime;
 
-  @VisibleForTesting
-  public ClientRetryEvent(int attemptCount, RaftClientRequest request, Throwable cause) {
-    this(attemptCount, request, attemptCount, cause);
-  }
-
-  public ClientRetryEvent(RaftClientRequest request, Throwable t, PendingClientRequest pending) {
-    this(pending.getAttemptCount(), request, pending.getExceptionCount(t), t);
-    this.pending = pending;
-  }
-
-  private ClientRetryEvent(int attemptCount, RaftClientRequest request, int causeCount, Throwable cause) {
+  public ClientRetryEvent(int attemptCount, RaftClientRequest request, int causeCount, Throwable cause,
+      Timestamp pendingRequestCreationTime) {
     this.attemptCount = attemptCount;
     this.causeCount = causeCount;
     this.request = request;
     this.cause = cause;
+    this.pendingRequestCreationTime = pendingRequestCreationTime;
   }
 
   @Override
@@ -69,7 +60,7 @@ public class ClientRetryEvent implements RetryPolicy.Event {
   }
 
   boolean isRequestTimeout(TimeDuration timeout) {
-    return pending != null && pending.isRequestTimeout(timeout);
+    return timeout != null && pendingRequestCreationTime.elapsedTime().compareTo(timeout) >= 0;
   }
 
   @Override
@@ -77,6 +68,7 @@ public class ClientRetryEvent implements RetryPolicy.Event {
     return JavaUtils.getClassSimpleName(getClass())
         + ":attempt=" + attemptCount
         + ",request=" + request
-        + ",cause=" + cause;
+        + ",cause=" + cause
+        + ",causeCount=" + causeCount;
   }
 }
diff --git a/ratis-common/src/main/java/org/apache/ratis/util/PeerProxyMap.java b/ratis-common/src/main/java/org/apache/ratis/util/PeerProxyMap.java
index 105ecbfb4..0ce0595fa 100644
--- a/ratis-common/src/main/java/org/apache/ratis/util/PeerProxyMap.java
+++ b/ratis-common/src/main/java/org/apache/ratis/util/PeerProxyMap.java
@@ -36,6 +36,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /** A map from peer id to peer and its proxy. */
 public class PeerProxyMap<PROXY extends Closeable> implements RaftPeer.Add, Closeable {
@@ -65,7 +66,7 @@ public class PeerProxyMap<PROXY extends Closeable> implements RaftPeer.Add, Clos
               throw new AlreadyClosedException(name + " is already " + current);
             }
             lifeCycle.startAndTransition(
-                () -> proxy = createProxy.apply(peer), IOException.class);
+                () -> proxy = createProxyImpl(peer), IOException.class);
           }
         }
       }
@@ -92,6 +93,7 @@ public class PeerProxyMap<PROXY extends Closeable> implements RaftPeer.Add, Clos
   private final Object resetLock = new Object();
 
   private final CheckedFunction<RaftPeer, PROXY, IOException> createProxy;
+  private final AtomicBoolean closed = new AtomicBoolean();
 
   public PeerProxyMap(String name, CheckedFunction<RaftPeer, PROXY, IOException> createProxy) {
     this.name = name;
@@ -102,6 +104,13 @@ public class PeerProxyMap<PROXY extends Closeable> implements RaftPeer.Add, Clos
     return name;
   }
 
+  private PROXY createProxyImpl(RaftPeer peer) throws IOException {
+    if (closed.get()) {
+      throw new AlreadyClosedException(name + ": Failed to create proxy for " + peer);
+    }
+    return createProxy.apply(peer);
+  }
+
   public PROXY getProxy(RaftPeerId id) throws IOException {
     Objects.requireNonNull(id, "id == null");
     PeerAndProxy p = peers.get(id);
@@ -161,6 +170,10 @@ public class PeerProxyMap<PROXY extends Closeable> implements RaftPeer.Add, Clos
 
   @Override
   public void close() {
+    if (!closed.compareAndSet(false, true)) {
+      return;
+    }
+
     final List<IOException> exceptions = Collections.synchronizedList(new ArrayList<>());
     ConcurrentUtils.parallelForEachAsync(peers.values(),
         pp -> pp.setNullProxyAndClose().map(proxy -> closeProxy(proxy, pp)).ifPresent(exceptions::add),
@@ -180,7 +193,7 @@ public class PeerProxyMap<PROXY extends Closeable> implements RaftPeer.Add, Clos
 
   private IOException closeProxy(PROXY proxy, PeerAndProxy pp) {
     try {
-      LOG.debug("{}: Closing proxy for peer {}", name, pp);
+      LOG.debug("{}: Closing proxy {} {} for peer {}", name, proxy.getClass().getSimpleName(), proxy, pp);
       proxy.close();
       return null;
     } catch (IOException e) {
diff --git a/ratis-server/src/test/java/org/apache/ratis/RaftAsyncTests.java b/ratis-server/src/test/java/org/apache/ratis/RaftAsyncTests.java
index 260f6013e..71c5c5ef0 100644
--- a/ratis-server/src/test/java/org/apache/ratis/RaftAsyncTests.java
+++ b/ratis-server/src/test/java/org/apache/ratis/RaftAsyncTests.java
@@ -372,7 +372,7 @@ public abstract class RaftAsyncTests<CLUSTER extends MiniRaftCluster> extends Ba
   @Test
   public void testStateMachineMetrics() throws Exception {
     runWithNewCluster(NUM_SERVERS, cluster ->
-        RaftBasicTests.testStateMachineMetrics(true, cluster, LOG));
+        RaftBasicTests.runTestStateMachineMetrics(true, cluster));
   }
 
   @Test
diff --git a/ratis-server/src/test/java/org/apache/ratis/RaftBasicTests.java b/ratis-server/src/test/java/org/apache/ratis/RaftBasicTests.java
index 47c9b0e08..4ff9681f0 100644
--- a/ratis-server/src/test/java/org/apache/ratis/RaftBasicTests.java
+++ b/ratis-server/src/test/java/org/apache/ratis/RaftBasicTests.java
@@ -168,12 +168,10 @@ public abstract class RaftBasicTests<CLUSTER extends MiniRaftCluster>
 
     final List<RaftServer.Division> divisions = cluster.getServerAliveStream().collect(Collectors.toList());
     for(RaftServer.Division impl: divisions) {
-        JavaUtils.attempt(() -> RaftTestUtil.assertLogEntries(impl, term, messages),
-            50, TimeDuration.valueOf(1, TimeUnit.SECONDS), impl.getId() + " assertLogEntries", LOG);
+      RaftTestUtil.assertLogEntries(impl, term, messages, 50, LOG);
     }
   }
 
-
   @Test
   public void testOldLeaderCommit() throws Exception {
     runWithNewCluster(NUM_SERVERS, this::runTestOldLeaderCommit);
@@ -218,7 +216,7 @@ public abstract class RaftBasicTests<CLUSTER extends MiniRaftCluster>
 
     cluster.getServerAliveStream()
         .map(RaftServer.Division::getRaftLog)
-        .forEach(log -> RaftTestUtil.assertLogEntries(log, term, messages));
+        .forEach(log -> RaftTestUtil.assertLogEntries(log, term, messages, System.out::println));
   }
 
   @Test
@@ -453,8 +451,12 @@ public abstract class RaftBasicTests<CLUSTER extends MiniRaftCluster>
     }
   }
 
-  public static void testStateMachineMetrics(boolean async,
-      MiniRaftCluster cluster, Logger LOG) throws Exception {
+  @Test
+  public void testStateMachineMetrics() throws Exception {
+    runWithNewCluster(NUM_SERVERS, cluster -> runTestStateMachineMetrics(false, cluster));
+  }
+
+  static void runTestStateMachineMetrics(boolean async, MiniRaftCluster cluster) throws Exception {
     RaftServer.Division leader = waitForLeader(cluster);
     try (final RaftClient client = cluster.createClient()) {
       Gauge appliedIndexGauge = getStatemachineGaugeWithName(leader,
diff --git a/ratis-server/src/test/java/org/apache/ratis/RaftTestUtil.java b/ratis-server/src/test/java/org/apache/ratis/RaftTestUtil.java
index 1d45c6821..21af91776 100644
--- a/ratis-server/src/test/java/org/apache/ratis/RaftTestUtil.java
+++ b/ratis-server/src/test/java/org/apache/ratis/RaftTestUtil.java
@@ -248,19 +248,16 @@ public interface RaftTestUtil {
     }
   }
 
-  static void assertLogEntries(RaftServer.Division server, long expectedTerm, SimpleMessage... expectedMessages) {
-    LOG.info("checking raft log for {}", server.getMemberId());
-    final RaftLog log = server.getRaftLog();
-    try {
-      RaftTestUtil.assertLogEntries(log, expectedTerm, expectedMessages);
-    } catch (AssertionError e) {
-      LOG.error("Unexpected raft log in {}", server.getMemberId(), e);
-      throw e;
-    }
+  static void assertLogEntries(RaftServer.Division server, long expectedTerm, SimpleMessage[] expectedMessages,
+      int numAttempts, Logger log) throws Exception {
+    final String name = server.getId() + " assertLogEntries";
+    final Function<Integer, Consumer<String>> print = i -> i < numAttempts? s -> {}: System.out::println;
+    JavaUtils.attempt(i -> assertLogEntries(server.getRaftLog(), expectedTerm, expectedMessages, print.apply(i)),
+        numAttempts, TimeDuration.ONE_SECOND, () -> name, log);
   }
 
   static Iterable<LogEntryProto> getLogEntryProtos(RaftLog log) {
-    return CollectionUtils.as(log.getEntries(0, Long.MAX_VALUE), ti -> {
+    return CollectionUtils.as(log.getEntries(0, log.getLastEntryTermIndex().getIndex() + 1), ti -> {
       try {
         return log.get(ti.getIndex());
       } catch (IOException exception) {
@@ -269,17 +266,17 @@ public interface RaftTestUtil {
     });
   }
 
-  static List<LogEntryProto> getStateMachineLogEntries(RaftLog log) {
+  static List<LogEntryProto> getStateMachineLogEntries(RaftLog log, Consumer<String> print) {
     final List<LogEntryProto> entries = new ArrayList<>();
     for (LogEntryProto e : getLogEntryProtos(log)) {
       final String s = LogProtoUtils.toLogEntryString(e);
       if (e.hasStateMachineLogEntry()) {
-        LOG.info(s + ", " + e.getStateMachineLogEntry().toString().trim().replace("\n", ", "));
+        print.accept(entries.size() + ") " + s);
         entries.add(e);
       } else if (e.hasConfigurationEntry()) {
-        LOG.info("Found {}, ignoring it.", s);
+        print.accept("Ignoring " + s);
       } else if (e.hasMetadataEntry()) {
-        LOG.info("Found {}, ignoring it.", s);
+        print.accept("Ignoring " + s);
       } else {
         throw new AssertionError("Unexpected LogEntryBodyCase " + e.getLogEntryBodyCase() + " at " + s);
       }
@@ -287,13 +284,14 @@ public interface RaftTestUtil {
     return entries;
   }
 
-  static void assertLogEntries(RaftLog log, long expectedTerm, SimpleMessage... expectedMessages) {
-    final List<LogEntryProto> entries = getStateMachineLogEntries(log);
+  static Void assertLogEntries(RaftLog log, long expectedTerm, SimpleMessage[] expectedMessages, Consumer<String> print) {
+    final List<LogEntryProto> entries = getStateMachineLogEntries(log, print);
     try {
       assertLogEntries(entries, expectedTerm, expectedMessages);
     } catch(Exception t) {
       throw new AssertionError("entries: " + entries, t);
     }
+    return null;
   }
 
   static void assertLogEntries(List<LogEntryProto> entries, long expectedTerm, SimpleMessage... expectedMessages) {
diff --git a/ratis-test/src/test/java/org/apache/ratis/RaftLogTruncateTests.java b/ratis-test/src/test/java/org/apache/ratis/RaftLogTruncateTests.java
new file mode 100644
index 000000000..80c57741c
--- /dev/null
+++ b/ratis-test/src/test/java/org/apache/ratis/RaftLogTruncateTests.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ratis;
+
+import org.apache.ratis.RaftTestUtil.SimpleMessage;
+import org.apache.ratis.client.RaftClient;
+import org.apache.ratis.client.RaftClientConfigKeys;
+import org.apache.ratis.client.impl.OrderedAsync;
+import org.apache.ratis.conf.RaftProperties;
+import org.apache.ratis.protocol.RaftClientReply;
+import org.apache.ratis.protocol.RaftPeerId;
+import org.apache.ratis.server.RaftServer;
+import org.apache.ratis.server.RaftServerConfigKeys;
+import org.apache.ratis.server.impl.MiniRaftCluster;
+import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLog;
+import org.apache.ratis.statemachine.StateMachine;
+import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing;
+import org.apache.ratis.util.Slf4jUtils;
+import org.apache.ratis.util.TimeDuration;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.event.Level;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.apache.ratis.RaftTestUtil.waitForLeader;
+
+public abstract class RaftLogTruncateTests<CLUSTER extends MiniRaftCluster> extends BaseTest
+    implements MiniRaftCluster.Factory.Get<CLUSTER> {
+  public static final int NUM_SERVERS = 5;
+  final TimeDuration MIN_TIMEOUT = TimeDuration.valueOf(3, TimeUnit.SECONDS);
+
+  static SimpleMessage[] arraycopy(SimpleMessage[] src1, SimpleMessage[] src2) {
+    final SimpleMessage[] dst = new SimpleMessage[src1.length + src2.length];
+    System.arraycopy(src1, 0, dst, 0, src1.length);
+    System.arraycopy(src2, 0, dst, src1.length, src2.length);
+    return dst;
+  }
+
+  {
+    Slf4jUtils.setLogLevel(OrderedAsync.LOG, Level.ERROR);
+    Slf4jUtils.setLogLevel(RaftServerConfigKeys.LOG, Level.ERROR);
+    Slf4jUtils.setLogLevel(RaftClientConfigKeys.LOG, Level.ERROR);
+
+    final RaftProperties p = getProperties();
+    p.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY, SimpleStateMachine4Testing.class, StateMachine.class);
+
+    // set a long rpc timeout so, when the leader does not have the majority, it won't step down fast.
+    RaftServerConfigKeys.Rpc.setTimeoutMin(p, MIN_TIMEOUT);
+    RaftServerConfigKeys.Rpc.setTimeoutMax(p, MIN_TIMEOUT.multiply(2));
+    RaftServerConfigKeys.Rpc.setFirstElectionTimeoutMin(p, TimeDuration.ONE_SECOND);
+    RaftServerConfigKeys.Rpc.setFirstElectionTimeoutMax(p, TimeDuration.ONE_SECOND.multiply(2));
+  }
+
+  @Override
+  public int getGlobalTimeoutSeconds() {
+    return 200;
+  }
+
+  @Test
+  public void testLogTruncate() throws Exception {
+    runWithNewCluster(NUM_SERVERS, this::runTestLogTruncate);
+  }
+
+  void runTestLogTruncate(MiniRaftCluster cluster) throws Exception {
+    final RaftServer.Division oldLeader = waitForLeader(cluster);
+    final List<RaftServer.Division> oldFollowers = cluster.getFollowers();
+    final List<RaftPeerId> killedPeers = new ArrayList<>();
+    final List<RaftPeerId> remainingPeers = new ArrayList<>();
+
+    final int majorityIndex = NUM_SERVERS / 2 + 1;
+    Assert.assertEquals(NUM_SERVERS - 1, oldFollowers.size());
+    Assert.assertTrue(majorityIndex < oldFollowers.size());
+
+    for (int i = 0; i < majorityIndex; i++) {
+      killedPeers.add(oldFollowers.get(i).getId());
+    }
+    remainingPeers.add(oldLeader.getId());
+    for (int i = majorityIndex; i < oldFollowers.size(); i++) {
+      remainingPeers.add(oldFollowers.get(i).getId());
+    }
+
+    try {
+      runTestLogTruncate(cluster, oldLeader, killedPeers, remainingPeers);
+    } catch (Throwable e) {
+      LOG.info("killedPeers   : {}", killedPeers);
+      LOG.info("remainingPeers: {}", remainingPeers);
+      throw e;
+    }
+  }
+
+  void runTestLogTruncate(MiniRaftCluster cluster, RaftServer.Division oldLeader,
+      List<RaftPeerId> killedPeers, List<RaftPeerId> remainingPeers) throws Exception {
+    final List<Throwable> exceptions = Collections.synchronizedList(new ArrayList<>());
+    final long oldLeaderTerm = oldLeader.getInfo().getCurrentTerm();
+    LOG.info("oldLeader: {}, term={}", oldLeader.getId(), oldLeaderTerm);
+
+    final SimpleMessage[] firstBatch = SimpleMessage.create(5, "first");
+    final SimpleMessage[] secondBatch = SimpleMessage.create(4, "second");
+
+    try (final RaftClient client = cluster.createClient(oldLeader.getId())) {
+      // send some messages
+      for (SimpleMessage batch : firstBatch) {
+        final RaftClientReply reply = client.io().send(batch);
+        Assert.assertTrue(reply.isSuccess());
+      }
+      for (RaftServer.Division f : cluster.getFollowers()) {
+        assertLogEntries(f, oldLeaderTerm, firstBatch);
+      }
+
+      // kill a majority of followers
+      LOG.info("Before killServer {}: {}", killedPeers, cluster.printServers());
+      for (RaftPeerId f : killedPeers) {
+        cluster.killServer(f);
+      }
+      LOG.info("After killServer {}: {}", killedPeers, cluster.printServers());
+
+      // send more messages, but they won't be committed due to not enough followers
+      final SimpleMessage[] messagesToBeTruncated = SimpleMessage.create(3, "messagesToBeTruncated");
+      final AtomicBoolean done = new AtomicBoolean();
+      for (SimpleMessage message : messagesToBeTruncated) {
+        client.async().send(message).whenComplete((r, e) -> {
+          if (!done.get()) {
+            exceptions.add(new IllegalStateException(message + " is completed: reply=" + r, e));
+          }
+        });
+      }
+
+      // check log messages
+      final SimpleMessage[] expectedMessages = arraycopy(firstBatch, messagesToBeTruncated);
+      for (RaftPeerId f : remainingPeers) {
+        assertLogEntries(cluster.getDivision(f), oldLeaderTerm, expectedMessages);
+      }
+      done.set(true);
+      LOG.info("done");
+    }
+
+    // kill the remaining servers
+    LOG.info("Before killServer {}: {}", remainingPeers, cluster.printServers());
+    for (RaftPeerId f : remainingPeers) {
+      cluster.killServer(f);
+    }
+    LOG.info("After killServer {}: {}", remainingPeers, cluster.printServers());
+
+    // restart the earlier followers
+    for (RaftPeerId f : killedPeers) {
+      cluster.restartServer(f, false);
+    }
+
+    // The new leader should be one of the earlier followers
+    final RaftServer.Division newLeader = waitForLeader(cluster);
+    LOG.info("After restartServer {}: {}", killedPeers, cluster.printServers());
+    final long newLeaderTerm = newLeader.getInfo().getCurrentTerm();
+
+    final SegmentedRaftLog newLeaderLog = (SegmentedRaftLog) newLeader.getRaftLog();
+    LOG.info("newLeader: {}, term {}, last={}", newLeader.getId(), newLeaderTerm,
+        newLeaderLog.getLastEntryTermIndex());
+    Assert.assertTrue(killedPeers.contains(newLeader.getId()));
+
+    // restart the remaining servers
+    for (RaftPeerId f : remainingPeers) {
+      cluster.restartServer(f, false);
+    }
+
+    // check RaftLog truncate
+    for (RaftPeerId f : remainingPeers) {
+      assertLogEntries(cluster.getDivision(f), oldLeaderTerm, firstBatch);
+    }
+
+    try (final RaftClient client = cluster.createClient(newLeader.getId())) {
+      // send more messages
+      for (SimpleMessage batch : secondBatch) {
+        final RaftClientReply reply = client.io().send(batch);
+        Assert.assertTrue(reply.isSuccess());
+      }
+    }
+
+    // check log messages -- it should be truncated and then append the new messages
+    final SimpleMessage[] expectedMessages = arraycopy(firstBatch, secondBatch);
+    for (RaftPeerId f : killedPeers) {
+      assertLogEntries(cluster.getDivision(f), oldLeaderTerm, expectedMessages);
+    }
+
+    if (!exceptions.isEmpty()) {
+      LOG.info("{} exceptions", exceptions.size());
+      for(int i = 0 ; i < exceptions.size(); i++) {
+        LOG.info("exception {})", i, exceptions.get(i));
+      }
+      Assert.fail();
+    }
+  }
+
+  private void assertLogEntries(RaftServer.Division server, long term, SimpleMessage[] expectedMessages)
+      throws Exception {
+    RaftTestUtil.assertLogEntries(server, term, expectedMessages, 30, LOG);
+  }
+}
diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftLogTruncateWithGrpc.java b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftLogTruncateWithGrpc.java
new file mode 100644
index 000000000..dc2846374
--- /dev/null
+++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftLogTruncateWithGrpc.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ratis.grpc;
+
+import org.apache.ratis.RaftLogTruncateTests;
+
+public class TestRaftLogTruncateWithGrpc extends RaftLogTruncateTests<MiniRaftClusterWithGrpc>
+    implements MiniRaftClusterWithGrpc.FactoryGet {
+}
diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java
index bc0061f5f..046453d58 100644
--- a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java
+++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java
@@ -73,12 +73,6 @@ public class TestRaftWithGrpc
     runWithNewCluster(NUM_SERVERS, cluster -> testRequestTimeout(false, cluster, LOG));
   }
 
-  @Test
-  public void testStateMachineMetrics() throws Exception {
-    runWithNewCluster(NUM_SERVERS, cluster ->
-        testStateMachineMetrics(false, cluster, LOG));
-  }
-
   @Test
   public void testUpdateViaHeartbeat() throws Exception {
     runWithNewCluster(NUM_SERVERS, this::runTestUpdateViaHeartbeat);
diff --git a/ratis-test/src/test/java/org/apache/ratis/retry/TestExceptionDependentRetry.java b/ratis-test/src/test/java/org/apache/ratis/retry/TestExceptionDependentRetry.java
index 264db8946..36e6dfbcc 100644
--- a/ratis-test/src/test/java/org/apache/ratis/retry/TestExceptionDependentRetry.java
+++ b/ratis-test/src/test/java/org/apache/ratis/retry/TestExceptionDependentRetry.java
@@ -154,7 +154,7 @@ public class TestExceptionDependentRetry extends BaseTest implements MiniRaftClu
       long sleepTime) {
     for (int i = 0; i < retries + 1; i++) {
       RetryPolicy.Action action = exceptionDependentRetry
-          .handleAttemptFailure(new ClientRetryEvent(i, null, exception));
+          .handleAttemptFailure(TestRetryPolicy.newClientRetryEvent(i, null, exception));
 
       final boolean expected = i < retries && i < maxAttempts;
       Assert.assertEquals(expected, action.shouldRetry());
diff --git a/ratis-test/src/test/java/org/apache/ratis/retry/TestRetryPolicy.java b/ratis-test/src/test/java/org/apache/ratis/retry/TestRetryPolicy.java
index d69cd1a2e..1b9536b4b 100644
--- a/ratis-test/src/test/java/org/apache/ratis/retry/TestRetryPolicy.java
+++ b/ratis-test/src/test/java/org/apache/ratis/retry/TestRetryPolicy.java
@@ -33,6 +33,7 @@ import org.apache.ratis.protocol.RaftPeerId;
 import org.apache.ratis.protocol.exceptions.TimeoutIOException;
 import org.apache.ratis.protocol.exceptions.ResourceUnavailableException;
 import org.apache.ratis.util.TimeDuration;
+import org.apache.ratis.util.Timestamp;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -70,6 +71,10 @@ public class TestRetryPolicy extends BaseTest {
     }
   }
 
+  static ClientRetryEvent newClientRetryEvent(int attemptCount, RaftClientRequest request, Throwable cause) {
+    return new ClientRetryEvent(attemptCount, request, attemptCount, cause, Timestamp.currentTime());
+  }
+
   @Test
   public void testRequestTypeDependentRetry() {
     final RequestTypeDependentRetryPolicy.Builder b = RequestTypeDependentRetryPolicy.newBuilder();
@@ -88,7 +93,7 @@ public class TestRetryPolicy extends BaseTest {
         RaftClientRequest.watchRequestType(1, ReplicationLevel.MAJORITY));
     for(int i = 1; i < 2*n; i++) {
       { //write
-        final ClientRetryEvent event = new ClientRetryEvent(i, writeRequest, null);
+        final ClientRetryEvent event = newClientRetryEvent(i, writeRequest, null);
         final RetryPolicy.Action action = policy.handleAttemptFailure(event);
 
         final boolean expected = i < n;
@@ -101,21 +106,21 @@ public class TestRetryPolicy extends BaseTest {
       }
 
       { //read and stale read are using default
-        final ClientRetryEvent event = new ClientRetryEvent(i, readRequest, null);
+        final ClientRetryEvent event = newClientRetryEvent(i, readRequest, null);
         final RetryPolicy.Action action = policy.handleAttemptFailure(event);
         Assert.assertTrue(action.shouldRetry());
         Assert.assertEquals(0L, action.getSleepTime().getDuration());
       }
 
       {
-        final ClientRetryEvent event = new ClientRetryEvent(i, staleReadRequest, null);
+        final ClientRetryEvent event = newClientRetryEvent(i, staleReadRequest, null);
         final RetryPolicy.Action action = policy.handleAttemptFailure(event);
         Assert.assertTrue(action.shouldRetry());
         Assert.assertEquals(0L, action.getSleepTime().getDuration());
       }
 
       { //watch has no retry
-        final ClientRetryEvent event = new ClientRetryEvent(i, watchRequest, null);
+        final ClientRetryEvent event = newClientRetryEvent(i, watchRequest, null);
         final RetryPolicy.Action action = policy.handleAttemptFailure(event);
         Assert.assertFalse(action.shouldRetry());
         Assert.assertEquals(0L, action.getSleepTime().getDuration());
@@ -148,7 +153,7 @@ public class TestRetryPolicy extends BaseTest {
     };
 
     for (RaftClientRequest request : requests) {
-      final ClientRetryEvent event = new ClientRetryEvent(request, new Exception(), pending);
+      final ClientRetryEvent event = pending.newClientRetryEvent(request, new Exception());
       final RetryPolicy.Action action = policy.handleAttemptFailure(event);
       Assert.assertTrue(action.shouldRetry());
       Assert.assertEquals(0L, action.getSleepTime().getDuration());
@@ -156,7 +161,7 @@ public class TestRetryPolicy extends BaseTest {
 
     timeout.sleep();
     for (RaftClientRequest request : requests) {
-      final ClientRetryEvent event = new ClientRetryEvent(request, new Exception(), pending);
+      final ClientRetryEvent event = pending.newClientRetryEvent(request, new Exception());
       final RetryPolicy.Action action = policy.handleAttemptFailure(event);
       Assert.assertFalse(action.shouldRetry());
     }
@@ -218,8 +223,7 @@ public class TestRetryPolicy extends BaseTest {
    */
   private void checkEvent(int exceptionAttemptCount, RetryPolicy retryPolicy, RaftClientRequest raftClientRequest,
       Throwable exception, Pair exceptionPolicyPair) {
-    final ClientRetryEvent event =
-        new ClientRetryEvent(exceptionAttemptCount, raftClientRequest, exception);
+    final ClientRetryEvent event = newClientRetryEvent(exceptionAttemptCount, raftClientRequest, exception);
     final RetryPolicy.Action action = retryPolicy.handleAttemptFailure(event);
 
     final boolean expected = exceptionAttemptCount < exceptionPolicyPair.retries;
diff --git a/ratis-test/src/test/java/org/apache/ratis/server/ServerRestartTests.java b/ratis-test/src/test/java/org/apache/ratis/server/ServerRestartTests.java
index 73ff1eb53..2f3edf781 100644
--- a/ratis-test/src/test/java/org/apache/ratis/server/ServerRestartTests.java
+++ b/ratis-test/src/test/java/org/apache/ratis/server/ServerRestartTests.java
@@ -261,7 +261,7 @@ public abstract class ServerRestartTests<CLUSTER extends MiniRaftCluster>
     final RaftPeerId leaderId = leader.getId();
     ids.add(leaderId);
 
-    RaftTestUtil.getStateMachineLogEntries(leaderLog);
+    RaftTestUtil.getStateMachineLogEntries(leaderLog, LOG::info);
 
     // check that the last metadata entry is written to the log
     JavaUtils.attempt(() -> assertLastLogEntry(leader), 20, HUNDRED_MILLIS, "leader last metadata entry", LOG);


(ratis) 01/12: RATIS-2013. OrderedAsync retry results an IllegalStateException in GrpcClientProtocolService. (#1026)

Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

williamsong pushed a commit to branch snapshot-3
in repository https://gitbox.apache.org/repos/asf/ratis.git

commit cc038aea981d53c6ece1673b137ee16d5fe63f7f
Author: Tsz-Wo Nicholas Sze <sz...@apache.org>
AuthorDate: Mon Jan 22 14:30:30 2024 -0800

    RATIS-2013. OrderedAsync retry results an IllegalStateException in GrpcClientProtocolService. (#1026)
---
 .../org/apache/ratis/client/impl/RaftClientImpl.java   |  6 ++++--
 .../main/java/org/apache/ratis/util/SlidingWindow.java | 18 +++++++++++++++---
 .../ratis/retry/TestExceptionDependentRetry.java       |  9 ++++++---
 3 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java
index ec16763c2..f42391947 100644
--- a/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java
+++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java
@@ -87,8 +87,10 @@ public final class RaftClientImpl implements RaftClient {
     public abstract RaftClientRequest newRequestImpl();
 
     final RaftClientRequest newRequest() {
-      attemptCount.incrementAndGet();
-      return newRequestImpl();
+      final int attempt = attemptCount.incrementAndGet();
+      final RaftClientRequest request = newRequestImpl();
+      LOG.debug("attempt #{}, newRequest {}", attempt, request);
+      return request;
     }
 
     CompletableFuture<RaftClientReply> getReplyFuture() {
diff --git a/ratis-common/src/main/java/org/apache/ratis/util/SlidingWindow.java b/ratis-common/src/main/java/org/apache/ratis/util/SlidingWindow.java
index 316604db0..732e3d890 100644
--- a/ratis-common/src/main/java/org/apache/ratis/util/SlidingWindow.java
+++ b/ratis-common/src/main/java/org/apache/ratis/util/SlidingWindow.java
@@ -122,6 +122,13 @@ public interface SlidingWindow {
       return requests.values().iterator();
     }
 
+    /** @return true iff the request already exists. */
+    boolean putIfAbsent(REQUEST request) {
+      final long seqNum = request.getSeqNum();
+      final REQUEST previous = requests.putIfAbsent(seqNum, request);
+      return previous != null;
+    }
+
     void putNewRequest(REQUEST request) {
       final long seqNum = request.getSeqNum();
       CollectionUtils.putNew(seqNum, request, requests, () -> getName() + ":requests");
@@ -443,11 +450,16 @@ public interface SlidingWindow {
       final long seqNum = request.getSeqNum();
       if (nextToProcess == -1 && (request.isFirstRequest() || seqNum == 0)) {
         nextToProcess = seqNum;
-        LOG.debug("{}: got seq={} (first request), set nextToProcess in {}", requests.getName(), seqNum, this);
+        requests.putNewRequest(request);
+        LOG.debug("Received seq={} (first request), {}", seqNum, this);
       } else {
-        LOG.debug("{}: got seq={} in {}", requests.getName(), seqNum, this);
+        final boolean isRetry = requests.putIfAbsent(request);
+        LOG.debug("Received seq={}, isRetry? {}, {}", seqNum, isRetry, this);
+        if (isRetry) {
+          return;
+        }
       }
-      requests.putNewRequest(request);
+
       processRequestsFromHead(processingMethod);
     }
 
diff --git a/ratis-test/src/test/java/org/apache/ratis/retry/TestExceptionDependentRetry.java b/ratis-test/src/test/java/org/apache/ratis/retry/TestExceptionDependentRetry.java
index 7eba6a82b..264db8946 100644
--- a/ratis-test/src/test/java/org/apache/ratis/retry/TestExceptionDependentRetry.java
+++ b/ratis-test/src/test/java/org/apache/ratis/retry/TestExceptionDependentRetry.java
@@ -178,12 +178,14 @@ public class TestExceptionDependentRetry extends BaseTest implements MiniRaftClu
   }
 
   void runTestExceptionRetryAttempts(MiniRaftClusterWithGrpc cluster) throws Exception {
-    final RaftServer.Division leader = RaftTestUtil.waitForLeader(cluster);
+    final int retryCount = 5;
+    final RetryPolicy timeoutPolicy = MultipleLinearRandomRetry.parseCommaSeparated("1ms, " + retryCount);
     final ExceptionDependentRetry policy = ExceptionDependentRetry.newBuilder()
-        .setExceptionToPolicy(TimeoutIOException.class, MultipleLinearRandomRetry.parseCommaSeparated("1ms, 5"))
+        .setExceptionToPolicy(TimeoutIOException.class, timeoutPolicy)
         .setDefaultPolicy(RetryPolicies.retryForeverNoSleep())
         .build();
 
+    final RaftServer.Division leader = RaftTestUtil.waitForLeader(cluster);
     // create a client with the exception dependent policy
     try (final RaftClient client = cluster.createClient(policy)) {
       client.async().send(new RaftTestUtil.SimpleMessage("1")).get();
@@ -196,7 +198,8 @@ public class TestExceptionDependentRetry extends BaseTest implements MiniRaftClu
       Assert.fail("Test should have failed.");
     } catch (ExecutionException e) {
       RaftRetryFailureException rrfe = (RaftRetryFailureException) e.getCause();
-      Assert.assertEquals(16, rrfe.getAttemptCount());
+      final int expectedCount = 1 + retryCount; // new request attempt + retry attempts
+      Assert.assertEquals(expectedCount, rrfe.getAttemptCount());
     } finally {
       SimpleStateMachine4Testing.get(leader).unblockWriteStateMachineData();
       cluster.shutdown();


(ratis) 11/12: RATIS-2050. Add creationGap param to snapshot management API (#1058)

Posted by wi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

williamsong pushed a commit to branch snapshot-3
in repository https://gitbox.apache.org/repos/asf/ratis.git

commit 52e8db6975501017bfa21deac9644b2c7834b4c2
Author: szywilliam <sz...@163.com>
AuthorDate: Wed Mar 27 14:09:48 2024 +0800

    RATIS-2050. Add creationGap param to snapshot management API (#1058)
---
 .../ratis/client/api/SnapshotManagementApi.java    | 22 ++++++++++++++++++++--
 .../apache/ratis/client/impl/ClientProtoUtils.java |  5 +++--
 .../ratis/client/impl/SnapshotManagementImpl.java  |  5 +++--
 .../ratis/protocol/SnapshotManagementRequest.java  | 18 ++++++++++++++++--
 ratis-proto/src/main/proto/Raft.proto              |  2 +-
 .../apache/ratis/server/impl/RaftServerImpl.java   |  5 +++--
 6 files changed, 46 insertions(+), 11 deletions(-)

diff --git a/ratis-client/src/main/java/org/apache/ratis/client/api/SnapshotManagementApi.java b/ratis-client/src/main/java/org/apache/ratis/client/api/SnapshotManagementApi.java
index edd047544..f83d97604 100644
--- a/ratis-client/src/main/java/org/apache/ratis/client/api/SnapshotManagementApi.java
+++ b/ratis-client/src/main/java/org/apache/ratis/client/api/SnapshotManagementApi.java
@@ -27,6 +27,24 @@ import java.io.IOException;
  */
 public interface SnapshotManagementApi {
 
-  /** trigger create snapshot file. */
-  RaftClientReply create(long timeoutMs) throws IOException;
+  /** The same as create(0, timeoutMs). */
+  default RaftClientReply create(long timeoutMs) throws IOException {
+    return create(0, timeoutMs);
+  }
+
+    /** The same as create(force? 1 : 0, timeoutMs). */
+  default RaftClientReply create(boolean force, long timeoutMs) throws IOException {
+    return create(force? 1 : 0, timeoutMs);
+  }
+
+    /**
+   * Trigger to create a snapshot.
+   *
+   * @param creationGap When (creationGap > 0) and (astAppliedIndex - lastSnapshotIndex < creationGap),
+   *                    return lastSnapshotIndex; otherwise, take a new snapshot and then return its index.
+   *                    When creationGap == 0, use the server configured value as the creationGap.
+   * @return a reply.  When {@link RaftClientReply#isSuccess()} is true,
+   *         {@link RaftClientReply#getLogIndex()} is the snapshot index fulfilling the operation.
+   */
+  RaftClientReply create(long creationGap, long timeoutMs) throws IOException;
 }
diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/ClientProtoUtils.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/ClientProtoUtils.java
index db1983195..cfa849047 100644
--- a/ratis-client/src/main/java/org/apache/ratis/client/impl/ClientProtoUtils.java
+++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/ClientProtoUtils.java
@@ -657,7 +657,8 @@ public interface ClientProtoUtils {
     switch(p.getOpCase()) {
       case CREATE:
         return SnapshotManagementRequest.newCreate(clientId, serverId,
-            ProtoUtils.toRaftGroupId(m.getRaftGroupId()), m.getCallId(), m.getTimeoutMs());
+            ProtoUtils.toRaftGroupId(m.getRaftGroupId()), m.getCallId(), m.getTimeoutMs(),
+            p.getCreate().getCreationGap());
       default:
         throw new IllegalArgumentException("Unexpected op " + p.getOpCase() + " in " + p);
     }
@@ -669,7 +670,7 @@ public interface ClientProtoUtils {
         .setRpcRequest(toRaftRpcRequestProtoBuilder(request));
     final SnapshotManagementRequest.Create create = request.getCreate();
     if (create != null) {
-      b.setCreate(SnapshotCreateRequestProto.newBuilder().build());
+      b.setCreate(SnapshotCreateRequestProto.newBuilder().setCreationGap(create.getCreationGap()).build());
     }
     return b.build();
   }
diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/SnapshotManagementImpl.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/SnapshotManagementImpl.java
index 1762dc0e4..65c54d0f2 100644
--- a/ratis-client/src/main/java/org/apache/ratis/client/impl/SnapshotManagementImpl.java
+++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/SnapshotManagementImpl.java
@@ -37,9 +37,10 @@ class SnapshotManagementImpl implements SnapshotManagementApi {
   }
 
   @Override
-  public RaftClientReply create(long timeoutMs) throws IOException {
+  public RaftClientReply create(long creationGap, long timeoutMs) throws IOException {
     final long callId = CallId.getAndIncrement();
     return client.io().sendRequestWithRetry(() -> SnapshotManagementRequest.newCreate(client.getId(),
-        Optional.ofNullable(server).orElseGet(client::getLeaderId), client.getGroupId(), callId, timeoutMs));
+        Optional.ofNullable(server).orElseGet(client::getLeaderId),
+        client.getGroupId(), callId, timeoutMs, creationGap));
   }
 }
diff --git a/ratis-common/src/main/java/org/apache/ratis/protocol/SnapshotManagementRequest.java b/ratis-common/src/main/java/org/apache/ratis/protocol/SnapshotManagementRequest.java
index 2ea2059b5..269fdfc59 100644
--- a/ratis-common/src/main/java/org/apache/ratis/protocol/SnapshotManagementRequest.java
+++ b/ratis-common/src/main/java/org/apache/ratis/protocol/SnapshotManagementRequest.java
@@ -24,7 +24,16 @@ public final class SnapshotManagementRequest extends RaftClientRequest {
   public abstract static class Op {
 
   }
-  public static class Create extends Op {
+
+  public static final class Create extends Op {
+    private final long creationGap;
+    private Create(long creationGap) {
+      this.creationGap = creationGap;
+    }
+
+    public long getCreationGap() {
+      return creationGap;
+    }
 
     @Override
     public String toString() {
@@ -35,8 +44,13 @@ public final class SnapshotManagementRequest extends RaftClientRequest {
 
   public static SnapshotManagementRequest newCreate(ClientId clientId,
       RaftPeerId serverId, RaftGroupId groupId, long callId, long timeoutMs) {
+    return newCreate(clientId, serverId, groupId, callId, timeoutMs, 0);
+  }
+
+  public static SnapshotManagementRequest newCreate(ClientId clientId,
+      RaftPeerId serverId, RaftGroupId groupId, long callId, long timeoutMs, long creationGap) {
     return new SnapshotManagementRequest(clientId,
-        serverId, groupId, callId, timeoutMs,new SnapshotManagementRequest.Create());
+        serverId, groupId, callId, timeoutMs, new SnapshotManagementRequest.Create(creationGap));
   }
 
   private final Op op;
diff --git a/ratis-proto/src/main/proto/Raft.proto b/ratis-proto/src/main/proto/Raft.proto
index d8a1b626a..10139d448 100644
--- a/ratis-proto/src/main/proto/Raft.proto
+++ b/ratis-proto/src/main/proto/Raft.proto
@@ -469,7 +469,7 @@ message SnapshotManagementRequestProto {
 }
 
 message SnapshotCreateRequestProto {
-
+  uint64 creationGap = 1;
 }
 
 message StartLeaderElectionRequestProto {
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
index 4ae60acaa..a6dba8ab6 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java
@@ -1209,9 +1209,10 @@ class RaftServerImpl implements RaftServer.Division,
     LOG.info("{}: takeSnapshotAsync {}", getMemberId(), request);
     assertLifeCycleState(LifeCycle.States.RUNNING);
     assertGroup(request.getRequestorId(), request.getRaftGroupId());
+    Preconditions.assertNotNull(request.getCreate(), "create");
 
-    //TODO(liuyaolong): get the gap value from shell command
-    long minGapValue = RaftServerConfigKeys.Snapshot.creationGap(proxy.getProperties());
+    final long creationGap = request.getCreate().getCreationGap();
+    long minGapValue = creationGap > 0? creationGap : RaftServerConfigKeys.Snapshot.creationGap(proxy.getProperties());
     final long lastSnapshotIndex = Optional.ofNullable(stateMachine.getLatestSnapshot())
         .map(SnapshotInfo::getIndex)
         .orElse(0L);