You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by as...@apache.org on 2020/12/29 12:04:20 UTC
[ignite-3] branch ignite-13885 updated: IGNITE-13885 wip tests.
This is an automated email from the ASF dual-hosted git repository.
ascherbakov pushed a commit to branch ignite-13885
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/ignite-13885 by this push:
new c4c7e91 IGNITE-13885 wip tests.
c4c7e91 is described below
commit c4c7e9137d63bbc39763d17c19c471f7a335181a
Author: Alexey Scherbakov <al...@gmail.com>
AuthorDate: Tue Dec 29 15:04:06 2020 +0300
IGNITE-13885 wip tests.
---
modules/raft/pom.xml | 18 +-
.../sofa/jraft/entity/LocalFileMetaOutter.java | 9 +-
.../com/alipay/sofa/jraft/rpc/CliRequests.java | 14 +-
.../java/com/alipay/sofa/jraft/rpc/Message.java | 7 +-
.../sofa/jraft/rpc/MessageBuilderFactory.java | 5 +-
.../com/alipay/sofa/jraft/rpc/RpcRequests.java | 2 +
.../sofa/jraft/rpc/message/AddPeerRequestImpl.java | 43 +
.../rpc/message/DefaultMessageBuilderFactory.java | 43 +-
.../sofa/jraft/rpc/message/LocalFileMetaImpl.java | 61 +
.../alipay/sofa/jraft/storage/io/ProtoBufFile.java | 20 -
.../com/alipay/sofa/jraft/util/ByteString.java | 15 +
.../sofa/jraft/util/RecyclableByteBufferList.java | 14 +
.../java/com/alipay/sofa/jraft/RouteTableTest.java | 169 +
.../java/com/alipay/sofa/jraft/StatusTest.java | 93 +
.../sofa/jraft/closure/ClosureQueueTest.java | 116 +
.../jraft/closure/SynchronizedClosureTest.java | 75 +
.../sofa/jraft/conf/ConfigurationEntryTest.java | 89 +
.../sofa/jraft/conf/ConfigurationManagerTest.java | 110 +
.../alipay/sofa/jraft/conf/ConfigurationTest.java | 172 +
.../com/alipay/sofa/jraft/core/BallotBoxTest.java | 155 +
.../com/alipay/sofa/jraft/core/CliServiceTest.java | 492 +++
.../com/alipay/sofa/jraft/core/ExpectClosure.java | 66 +
.../com/alipay/sofa/jraft/core/FSMCallerTest.java | 287 ++
.../alipay/sofa/jraft/core/IteratorImplTest.java | 137 +
.../com/alipay/sofa/jraft/core/IteratorTest.java | 114 +
.../com/alipay/sofa/jraft/core/MockClosure.java | 30 +
.../alipay/sofa/jraft/core/MockStateMachine.java | 226 ++
.../java/com/alipay/sofa/jraft/core/NodeTest.java | 3418 ++++++++++++++++++++
.../sofa/jraft/core/ReadOnlyServiceTest.java | 267 ++
.../sofa/jraft/core/ReplicatorGroupTest.java | 299 ++
.../com/alipay/sofa/jraft/core/ReplicatorTest.java | 809 +++++
.../com/alipay/sofa/jraft/core/TestCluster.java | 494 +++
.../sofa/jraft/core/TestJRaftServiceFactory.java | 38 +
.../sofa/jraft/core/V1JRaftServiceFactory.java | 29 +
.../com/alipay/sofa/jraft/entity/BallotTest.java | 51 +
.../com/alipay/sofa/jraft/entity/LogEntryTest.java | 126 +
.../com/alipay/sofa/jraft/entity/LogIdTest.java | 51 +
.../com/alipay/sofa/jraft/entity/PeerIdTest.java | 144 +
.../entity/codec/BaseLogEntryCodecFactoryTest.java | 118 +
.../jraft/entity/codec/LogEntryCodecPerfTest.java | 127 +
.../codec/v1/LogEntryV1CodecFactoryTest.java | 29 +
.../sofa/jraft/rpc/AbstractClientServiceTest.java | 283 ++
.../sofa/jraft/rpc/AppendEntriesBenchmark.java | 258 ++
.../sofa/jraft/rpc/RpcResponseFactoryTest.java | 67 +
.../com/alipay/sofa/jraft/rpc/impl/FutureTest.java | 136 +
.../jraft/rpc/impl/PingRequestProcessorTest.java | 37 +
.../impl/cli/AbstractCliRequestProcessorTest.java | 94 +
.../impl/cli/AddLearnersRequestProcessorTest.java | 68 +
.../rpc/impl/cli/AddPeerRequestProcessorTest.java | 62 +
.../rpc/impl/cli/BaseCliRequestProcessorTest.java | 202 ++
.../impl/cli/ChangePeersRequestProcessorTest.java | 64 +
.../rpc/impl/cli/GetPeersRequestProcessorTest.java | 54 +
.../cli/RemoveLearnersRequestProcessorTest.java | 65 +
.../impl/cli/RemovePeerRequestProcessorTest.java | 62 +
.../cli/ResetLearnersRequestProcessorTest.java | 68 +
.../impl/cli/ResetPeersRequestProcessorTest.java | 53 +
.../rpc/impl/cli/SnapshotRequestProcessorTest.java | 50 +
.../TransferLeadershipRequestProcessorTest.java | 51 +
.../core/AppendEntriesRequestProcessorTest.java | 206 ++
.../impl/core/BaseNodeRequestProcessorTest.java | 78 +
.../impl/core/DefaultRaftClientServiceTest.java | 54 +
.../core/InstallSnapshotRequestProcessorTest.java | 60 +
.../rpc/impl/core/NodeRequestProcessorTest.java | 125 +
.../rpc/impl/core/PreVoteRequestProcessorTest.java | 54 +
.../impl/core/ReadIndexRequestProcessorTest.java | 51 +
.../impl/core/RequestVoteRequestProcessorTest.java | 54 +
.../impl/core/TimeoutNowRequestProcessorTest.java | 52 +
.../alipay/sofa/jraft/storage/BaseStorageTest.java | 47 +
.../alipay/sofa/jraft/storage/FileServiceTest.java | 153 +
.../sofa/jraft/storage/SnapshotExecutorTest.java | 300 ++
.../jraft/storage/impl/BaseLogStorageTest.java | 240 ++
.../storage/impl/LocalRaftMetaStorageTest.java | 101 +
.../sofa/jraft/storage/impl/LogManagerTest.java | 410 +++
.../jraft/storage/impl/LogStorageBenchmark.java | 142 +
.../sofa/jraft/storage/io/LocalFileReaderTest.java | 106 +
.../sofa/jraft/storage/io/ProtobufFileTest.java | 53 +
.../snapshot/ThroughputSnapshotThrottleTest.java | 43 +
.../snapshot/local/LocalSnapshotCopierTest.java | 207 ++
.../snapshot/local/LocalSnapshotMetaTableTest.java | 111 +
.../snapshot/local/LocalSnapshotReaderTest.java | 86 +
.../snapshot/local/LocalSnapshotStorageTest.java | 95 +
.../snapshot/local/LocalSnapshotWriterTest.java | 91 +
.../snapshot/local/SnapshotFileReaderTest.java | 91 +
.../storage/snapshot/remote/CopySessionTest.java | 177 +
.../snapshot/remote/RemoteFileCopierTest.java | 67 +
.../alipay/sofa/jraft/test/MockAsyncContext.java | 61 +
.../java/com/alipay/sofa/jraft/test/TestUtils.java | 161 +
.../sofa/jraft/util/AdaptiveBufAllocatorTest.java | 75 +
.../com/alipay/sofa/jraft/util/ArrayDequeTest.java | 71 +
.../sofa/jraft/util/AsciiCodecBenchmark.java | 93 +
.../sofa/jraft/util/AsciiStringUtilTest.java | 38 +
.../java/com/alipay/sofa/jraft/util/BitsTest.java | 39 +
.../sofa/jraft/util/ByteBufferCollectorTest.java | 69 +
.../com/alipay/sofa/jraft/util/BytesUtilTest.java | 128 +
.../alipay/sofa/jraft/util/CountDownEventTest.java | 77 +
.../com/alipay/sofa/jraft/util/CrcUtilTest.java | 43 +
.../com/alipay/sofa/jraft/util/EndpointTest.java | 47 +
.../jraft/util/FileOutputSignalHandlerTest.java | 63 +
.../sofa/jraft/util/JRaftServiceLoaderTest.java | 89 +
.../jraft/util/RecyclableByteBufferListTest.java | 53 +
.../com/alipay/sofa/jraft/util/RecyclersTest.java | 146 +
.../alipay/sofa/jraft/util/RepeatedTimerTest.java | 135 +
.../alipay/sofa/jraft/util/SegmentListTest.java | 295 ++
.../com/alipay/sofa/jraft/util/ThreadIdTest.java | 108 +
.../alipay/sofa/jraft/util/Utf8CodecBenchmark.java | 99 +
.../java/com/alipay/sofa/jraft/util/UtilsTest.java | 202 ++
.../util/concurrent/AdjustableSemaphoreTest.java | 69 +
.../LongHeldDetectingReadWriteLockTest.java | 134 +
.../concurrent/MpscSingleThreadExecutorTest.java | 161 +
.../concurrent/SingleThreadExecutorBenchmark.java | 168 +
110 files changed, 16061 insertions(+), 73 deletions(-)
diff --git a/modules/raft/pom.xml b/modules/raft/pom.xml
index a01aefb..a0b6971 100644
--- a/modules/raft/pom.xml
+++ b/modules/raft/pom.xml
@@ -62,25 +62,21 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.13.2</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.13.2</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>2.13.2</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-jcl</artifactId>
<version>2.13.2</version>
- <scope>test</scope>
</dependency>
<!-- commons -->
<dependency>
@@ -103,6 +99,20 @@
<artifactId>metrics-core</artifactId>
<version>4.0.2</version>
</dependency>
+
+ <!-- benchmark -->
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-core</artifactId>
+ <version>1.20</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.openjdk.jmh</groupId>
+ <artifactId>jmh-generator-annprocess</artifactId>
+ <version>1.20</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/modules/raft/src/main/java/com/alipay/sofa/jraft/entity/LocalFileMetaOutter.java b/modules/raft/src/main/java/com/alipay/sofa/jraft/entity/LocalFileMetaOutter.java
index f808749..3c1a368 100644
--- a/modules/raft/src/main/java/com/alipay/sofa/jraft/entity/LocalFileMetaOutter.java
+++ b/modules/raft/src/main/java/com/alipay/sofa/jraft/entity/LocalFileMetaOutter.java
@@ -20,6 +20,7 @@
package com.alipay.sofa.jraft.entity;
import com.alipay.sofa.jraft.rpc.Message;
+import com.alipay.sofa.jraft.rpc.MessageBuilderFactory;
import com.alipay.sofa.jraft.util.ByteString;
public final class LocalFileMetaOutter {
@@ -70,7 +71,7 @@ public final class LocalFileMetaOutter {
public interface LocalFileMeta extends Message {
static Builder newBuilder() {
- return null;
+ return MessageBuilderFactory.DEFAULT.createLocalFileMeta();
}
ByteString getUserMeta();
@@ -81,12 +82,18 @@ public final class LocalFileMetaOutter {
boolean hasChecksum();
+ boolean hasUserMeta();
+
interface Builder {
LocalFileMeta build();
Builder setUserMeta(ByteString data);
void mergeFrom(Message fileMeta);
+
+ Builder setChecksum(String sum);
+
+ Builder setSource(FileSource source);
}
}
}
diff --git a/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/CliRequests.java b/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/CliRequests.java
index 4f1f46d..4c8776d 100644
--- a/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/CliRequests.java
+++ b/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/CliRequests.java
@@ -38,7 +38,7 @@ public final class CliRequests {
}
public static Builder newBuilder() {
- return MessageBuilderFactory.DEFAULT.create();
+ return MessageBuilderFactory.DEFAULT.createAddPeer();
}
}
@@ -177,7 +177,7 @@ public final class CliRequests {
Builder setLeaderId(String leaderId);
- void addNewPeers(String peerId);
+ Builder addNewPeers(String peerId);
ChangePeersRequest build();
}
@@ -281,7 +281,7 @@ public final class CliRequests {
Builder setPeerId(String peerId);
- void addNewPeers(String peerId);
+ Builder addNewPeers(String peerId);
ResetPeerRequest build();
}
@@ -305,7 +305,7 @@ public final class CliRequests {
Builder setLeaderId(String leaderId);
- void setPeerId(String peerId);
+ Builder setPeerId(String peerId);
TransferLeaderRequest build();
}
@@ -427,7 +427,7 @@ public final class CliRequests {
Builder setLeaderId(String leaderId);
- void addLearners(String learnerId);
+ Builder addLearners(String learnerId);
AddLearnersRequest build();
}
@@ -453,7 +453,7 @@ public final class CliRequests {
Builder setLeaderId(String leaderId);
- void addLearners(String leaderId);
+ Builder addLearners(String leaderId);
RemoveLearnersRequest build();
}
@@ -485,7 +485,7 @@ public final class CliRequests {
Builder setLeaderId(String leaderId);
- void addLearners(String learnerId);
+ Builder addLearners(String learnerId);
ResetLearnersRequest build();
}
diff --git a/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/Message.java b/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/Message.java
index fe191cf..26fa6b0 100644
--- a/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/Message.java
+++ b/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/Message.java
@@ -1,4 +1,9 @@
package com.alipay.sofa.jraft.rpc;
-public interface Message {
+import java.io.Serializable;
+
+/**
+ * Base message. Extends Serializable for compatibility with JDK serialization.
+ */
+public interface Message extends Serializable {
}
diff --git a/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/MessageBuilderFactory.java b/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/MessageBuilderFactory.java
index 9d1a965..2785a8e 100644
--- a/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/MessageBuilderFactory.java
+++ b/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/MessageBuilderFactory.java
@@ -1,9 +1,12 @@
package com.alipay.sofa.jraft.rpc;
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter;
import com.alipay.sofa.jraft.rpc.message.DefaultMessageBuilderFactory;
public interface MessageBuilderFactory {
public static MessageBuilderFactory DEFAULT = new DefaultMessageBuilderFactory();
- CliRequests.AddPeerRequest.Builder create();
+ CliRequests.AddPeerRequest.Builder createAddPeer();
+
+ LocalFileMetaOutter.LocalFileMeta.Builder createLocalFileMeta();
}
diff --git a/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/RpcRequests.java b/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/RpcRequests.java
index 22f2ff9..74568b7 100644
--- a/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/RpcRequests.java
+++ b/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/RpcRequests.java
@@ -322,6 +322,8 @@ public final class RpcRequests {
boolean hasData();
+ byte[] toByteArray();
+
interface Builder {
AppendEntriesRequest build();
diff --git a/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/message/AddPeerRequestImpl.java b/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/message/AddPeerRequestImpl.java
new file mode 100644
index 0000000..3548fc2
--- /dev/null
+++ b/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/message/AddPeerRequestImpl.java
@@ -0,0 +1,43 @@
+package com.alipay.sofa.jraft.rpc.message;
+
+import com.alipay.sofa.jraft.rpc.CliRequests;
+
+class AddPeerRequestImpl implements CliRequests.AddPeerRequest, CliRequests.AddPeerRequest.Builder {
+ private String groupId;
+ private String leaderId;
+ private String peerId;
+
+ @Override public String getGroupId() {
+ return groupId;
+ }
+
+ @Override public String getLeaderId() {
+ return leaderId;
+ }
+
+ @Override public String getPeerId() {
+ return peerId;
+ }
+
+ @Override public Builder setGroupId(String groupId) {
+ this.groupId = groupId;
+
+ return this;
+ }
+
+ @Override public Builder setLeaderId(String leaderId) {
+ this.leaderId = leaderId;
+
+ return this;
+ }
+
+ @Override public Builder setPeerId(String peerId) {
+ this.peerId = peerId;
+
+ return this;
+ }
+
+ @Override public CliRequests.AddPeerRequest build() {
+ return this;
+ }
+}
diff --git a/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/message/DefaultMessageBuilderFactory.java b/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/message/DefaultMessageBuilderFactory.java
index ce11a1b..14a3fb4 100644
--- a/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/message/DefaultMessageBuilderFactory.java
+++ b/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/message/DefaultMessageBuilderFactory.java
@@ -1,50 +1,15 @@
package com.alipay.sofa.jraft.rpc.message;
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter;
import com.alipay.sofa.jraft.rpc.CliRequests;
import com.alipay.sofa.jraft.rpc.MessageBuilderFactory;
public class DefaultMessageBuilderFactory implements MessageBuilderFactory {
- @Override public CliRequests.AddPeerRequest.Builder create() {
+ @Override public CliRequests.AddPeerRequest.Builder createAddPeer() {
return new AddPeerRequestImpl();
}
- private static class AddPeerRequestImpl implements CliRequests.AddPeerRequest, CliRequests.AddPeerRequest.Builder {
- private String groupId;
- private String leaderId;
- private String peerId;
-
- @Override public String getGroupId() {
- return groupId;
- }
-
- @Override public String getLeaderId() {
- return leaderId;
- }
-
- @Override public String getPeerId() {
- return peerId;
- }
-
- @Override public Builder setGroupId(String groupId) {
- this.groupId = groupId;
-
- return this;
- }
-
- @Override public Builder setLeaderId(String leaderId) {
- this.leaderId = leaderId;
-
- return this;
- }
-
- @Override public Builder setPeerId(String peerId) {
- this.peerId = peerId;
-
- return this;
- }
-
- @Override public CliRequests.AddPeerRequest build() {
- return this;
- }
+ @Override public LocalFileMetaOutter.LocalFileMeta.Builder createLocalFileMeta() {
+ return new LocalFileMetaImpl();
}
}
diff --git a/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/message/LocalFileMetaImpl.java b/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/message/LocalFileMetaImpl.java
new file mode 100644
index 0000000..3d34b40
--- /dev/null
+++ b/modules/raft/src/main/java/com/alipay/sofa/jraft/rpc/message/LocalFileMetaImpl.java
@@ -0,0 +1,61 @@
+package com.alipay.sofa.jraft.rpc.message;
+
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter;
+import com.alipay.sofa.jraft.rpc.Message;
+import com.alipay.sofa.jraft.util.ByteString;
+
+public class LocalFileMetaImpl implements LocalFileMetaOutter.LocalFileMeta, LocalFileMetaOutter.LocalFileMeta.Builder {
+ private ByteString userMeta; // TODO asch not used currently.
+ private LocalFileMetaOutter.FileSource fileSource;
+ private String checksum;
+
+ @Override public ByteString getUserMeta() {
+ return userMeta;
+ }
+
+ @Override public LocalFileMetaOutter.FileSource getSource() {
+ return fileSource;
+ }
+
+ @Override public String getChecksum() {
+ return checksum;
+ }
+
+ @Override public boolean hasChecksum() {
+ return checksum != null;
+ }
+
+ @Override public boolean hasUserMeta() {
+ return userMeta != null;
+ }
+
+ @Override public LocalFileMetaOutter.LocalFileMeta build() {
+ return this;
+ }
+
+ @Override public Builder setUserMeta(ByteString data) {
+ this.userMeta = data;
+
+ return this;
+ }
+
+ @Override public void mergeFrom(Message fileMeta) {
+ LocalFileMetaOutter.LocalFileMeta tmp = (LocalFileMetaOutter.LocalFileMeta) fileMeta;
+
+ this.userMeta = tmp.getUserMeta();
+ this.fileSource = tmp.getSource();
+ this.checksum = tmp.getChecksum();
+ }
+
+ @Override public Builder setChecksum(String checksum) {
+ this.checksum = checksum;
+
+ return this;
+ }
+
+ @Override public Builder setSource(LocalFileMetaOutter.FileSource source) {
+ this.fileSource = source;
+
+ return this;
+ }
+}
diff --git a/modules/raft/src/main/java/com/alipay/sofa/jraft/storage/io/ProtoBufFile.java b/modules/raft/src/main/java/com/alipay/sofa/jraft/storage/io/ProtoBufFile.java
index 4ce83f0..16913f0 100644
--- a/modules/raft/src/main/java/com/alipay/sofa/jraft/storage/io/ProtoBufFile.java
+++ b/modules/raft/src/main/java/com/alipay/sofa/jraft/storage/io/ProtoBufFile.java
@@ -118,24 +118,4 @@ public class ProtoBufFile {
return Utils.atomicMoveFile(file, new File(this.path), sync);
}
-
- public static void main(String[] args) throws IOException {
- File file = File.createTempFile("store", "tmp");
-
- ProtoBufFile tmp = new ProtoBufFile(file.getAbsolutePath());
-
- CliRequests.AddPeerRequest.Builder b = CliRequests.AddPeerRequest.newBuilder();
- b.setGroupId("grp1");
- b.setLeaderId("zzz");
- b.setPeerId("tmp");
-
- CliRequests.AddPeerRequest req0 = b.build();
- boolean saved = tmp.save(req0, false);
-
- CliRequests.AddPeerRequest req1 = tmp.load();
-
- System.out.println();
-
- file.delete();
- }
}
diff --git a/modules/raft/src/main/java/com/alipay/sofa/jraft/util/ByteString.java b/modules/raft/src/main/java/com/alipay/sofa/jraft/util/ByteString.java
index b4d92a5..c66b09a 100644
--- a/modules/raft/src/main/java/com/alipay/sofa/jraft/util/ByteString.java
+++ b/modules/raft/src/main/java/com/alipay/sofa/jraft/util/ByteString.java
@@ -1,5 +1,6 @@
package com.alipay.sofa.jraft.util;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
@@ -36,4 +37,18 @@ public class ByteString {
channel.write(buf);
}
+
+ public byte[] toByteArray() {
+ if (buf.hasArray())
+ return buf.array();
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ WritableByteChannel channel = Channels.newChannel(bos);
+ try {
+ channel.write(buf);
+ } catch (IOException e) {
+ throw new Error(e);
+ }
+ return bos.toByteArray();
+ }
}
diff --git a/modules/raft/src/main/java/com/alipay/sofa/jraft/util/RecyclableByteBufferList.java b/modules/raft/src/main/java/com/alipay/sofa/jraft/util/RecyclableByteBufferList.java
index 928dffd..c48372c 100644
--- a/modules/raft/src/main/java/com/alipay/sofa/jraft/util/RecyclableByteBufferList.java
+++ b/modules/raft/src/main/java/com/alipay/sofa/jraft/util/RecyclableByteBufferList.java
@@ -16,9 +16,12 @@
*/
package com.alipay.sofa.jraft.util;
+import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
+import java.util.List;
/**
* A simple {@link java.nio.ByteBuffer} list which is recyclable.
@@ -48,6 +51,17 @@ public final class RecyclableByteBufferList extends ArrayList<ByteBuffer> implem
return ret;
}
+ /**
+ * TODO asch slow concatenation by copying, should use RopeByteBuffer.
+ *
+ * @param buffers Buffers.
+ */
+ public static ByteString concatenate(List<ByteBuffer> buffers) {
+ final ByteBuffer combined = ByteBuffer.allocate(buffers.stream().mapToInt(Buffer::remaining).sum());
+ buffers.stream().forEach(b -> combined.put(b.duplicate()));
+ return new ByteString(combined);
+ }
+
public int getCapacity() {
return this.capacity;
}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/RouteTableTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/RouteTableTest.java
new file mode 100644
index 0000000..1995d3b
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/RouteTableTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.core.NodeImpl;
+import com.alipay.sofa.jraft.core.TestCluster;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.CliOptions;
+import com.alipay.sofa.jraft.rpc.impl.cli.CliClientServiceImpl;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class RouteTableTest {
+
+ static final Logger LOG = LoggerFactory.getLogger(RouteTableTest.class);
+
+ private String dataPath;
+
+ private TestCluster cluster;
+ private final String groupId = "RouteTableTest";
+
+ CliClientServiceImpl cliClientService;
+
+ @Before
+ public void setup() throws Exception {
+ cliClientService = new CliClientServiceImpl();
+ cliClientService.init(new CliOptions());
+ this.dataPath = TestUtils.mkTempDir();
+ FileUtils.forceMkdir(new File(this.dataPath));
+ assertEquals(NodeImpl.GLOBAL_NUM_NODES.get(), 0);
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ cluster = new TestCluster(groupId, dataPath, peers);
+ for (final PeerId peer : peers) {
+ cluster.start(peer.getEndpoint());
+ }
+ cluster.waitLeader();
+ }
+
+ @After
+ public void teardown() throws Exception {
+ cliClientService.shutdown();
+ cluster.stopAll();
+ if (NodeImpl.GLOBAL_NUM_NODES.get() > 0) {
+ Thread.sleep(1000);
+ assertEquals(NodeImpl.GLOBAL_NUM_NODES.get(), 0);
+ }
+ FileUtils.deleteDirectory(new File(this.dataPath));
+ NodeManager.getInstance().clear();
+ RouteTable.getInstance().reset();
+ }
+
+ @Test
+ public void testUpdateConfSelectLeader() throws Exception {
+ final RouteTable rt = RouteTable.getInstance();
+ assertNull(rt.getConfiguration(groupId));
+ rt.updateConfiguration(groupId, new Configuration(cluster.getPeers()));
+ assertEquals(rt.getConfiguration(groupId), new Configuration(cluster.getPeers()));
+ assertNull(rt.selectLeader(groupId));
+ assertTrue(rt.refreshLeader(cliClientService, groupId, 10000).isOk());
+
+ final PeerId leader = rt.selectLeader(groupId);
+ assertEquals(leader, cluster.getLeader().getNodeId().getPeerId());
+ }
+
+ @Test
+ public void testUpdateLeaderNull() throws Exception {
+ this.testUpdateConfSelectLeader();
+ final RouteTable rt = RouteTable.getInstance();
+ rt.updateLeader(groupId, (PeerId) null);
+ assertNull(rt.selectLeader(groupId));
+ assertTrue(rt.refreshLeader(cliClientService, groupId, 10000).isOk());
+
+ final PeerId leader = rt.selectLeader(groupId);
+ assertEquals(leader, cluster.getLeader().getNodeId().getPeerId());
+ }
+
+ @Test
+ public void testRefreshLeaderWhenLeaderStops() throws Exception {
+ final RouteTable rt = RouteTable.getInstance();
+ testUpdateConfSelectLeader();
+ PeerId leader = rt.selectLeader(groupId);
+ this.cluster.stop(leader.getEndpoint());
+ this.cluster.waitLeader();
+ final PeerId oldLeader = leader.copy();
+
+ assertTrue(rt.refreshLeader(cliClientService, groupId, 10000).isOk());
+ leader = rt.selectLeader(groupId);
+ assertNotEquals(leader, oldLeader);
+ assertEquals(leader, cluster.getLeader().getNodeId().getPeerId());
+ }
+
+ @Test
+ public void testRefreshLeaderWhenFirstPeerDown() throws Exception {
+ final RouteTable rt = RouteTable.getInstance();
+ rt.updateConfiguration(groupId, new Configuration(cluster.getPeers()));
+ assertTrue(rt.refreshLeader(cliClientService, groupId, 10000).isOk());
+ cluster.stop(cluster.getPeers().get(0).getEndpoint());
+ Thread.sleep(1000);
+ this.cluster.waitLeader();
+ assertTrue(rt.refreshLeader(cliClientService, groupId, 10000).isOk());
+ }
+
+ @Test
+ public void testRefreshFail() throws Exception {
+ cluster.stopAll();
+ final RouteTable rt = RouteTable.getInstance();
+ rt.updateConfiguration(groupId, new Configuration(cluster.getPeers()));
+ final Status status = rt.refreshLeader(cliClientService, groupId, 5000);
+ assertFalse(status.isOk());
+ assertTrue(status.getErrorMsg().contains("Fail to init channel"));
+ }
+
+ @Test
+ public void testRefreshConfiguration() throws Exception {
+ final RouteTable rt = RouteTable.getInstance();
+ final List<PeerId> partConf = new ArrayList<>();
+ partConf.add(cluster.getLeader().getLeaderId());
+ // part of peers conf, only contains leader peer
+ rt.updateConfiguration(groupId, new Configuration(partConf));
+ // fetch all conf
+ final Status st = rt.refreshConfiguration(cliClientService, groupId, 10000);
+ assertTrue(st.isOk());
+ final Configuration newCnf = rt.getConfiguration(groupId);
+ assertArrayEquals(new HashSet<>(cluster.getPeers()).toArray(), new HashSet<>(newCnf.getPeerSet()).toArray());
+ }
+
+ @Test
+ public void testRefreshConfigurationFail() throws Exception {
+ cluster.stopAll();
+ final RouteTable rt = RouteTable.getInstance();
+ rt.updateConfiguration(groupId, new Configuration(cluster.getPeers()));
+ final Status st = rt.refreshConfiguration(cliClientService, groupId, 10000);
+ assertFalse(st.isOk());
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/StatusTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/StatusTest.java
new file mode 100644
index 0000000..2c3a1e9
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/StatusTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft;
+
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.error.RaftError;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class StatusTest {
+
+ @Test
+ public void testOKStatus() {
+ Status s = new Status();
+ assertTrue(s.isOk());
+ assertEquals(0, s.getCode());
+ assertNull(s.getErrorMsg());
+ }
+
+ @Test
+ public void testStatusOK() {
+ Status s = Status.OK();
+ assertTrue(s.isOk());
+ assertEquals(0, s.getCode());
+ assertNull(s.getErrorMsg());
+ assertNotSame(Status.OK(), s);
+ }
+
+ @Test
+ public void testNewStatus() {
+ Status s = new Status(-2, "test");
+ assertEquals(-2, s.getCode());
+ assertEquals("test", s.getErrorMsg());
+ assertFalse(s.isOk());
+ }
+
+ @Test
+ public void testNewStatusVaridicArgs() {
+ Status s = new Status(-2, "test %s %d", "world", 100);
+ assertEquals(-2, s.getCode());
+ assertEquals("test world 100", s.getErrorMsg());
+ assertFalse(s.isOk());
+ }
+
+ @Test
+ public void testNewStatusRaftError() {
+ Status s = new Status(RaftError.EACCES, "test %s %d", "world", 100);
+ assertEquals(RaftError.EACCES.getNumber(), s.getCode());
+ assertEquals(RaftError.EACCES, s.getRaftError());
+ assertEquals("test world 100", s.getErrorMsg());
+ assertFalse(s.isOk());
+ }
+
+ @Test
+ public void testSetErrorRaftError() {
+ Status s = new Status();
+ s.setError(RaftError.EACCES, "test %s %d", "world", 100);
+ assertEquals(RaftError.EACCES.getNumber(), s.getCode());
+ assertEquals(RaftError.EACCES, s.getRaftError());
+ assertEquals("test world 100", s.getErrorMsg());
+ assertFalse(s.isOk());
+ }
+
+ @Test
+ public void testSetError() {
+ Status s = new Status();
+ s.setError(RaftError.EACCES.getNumber(), "test %s %d", "world", 100);
+ assertEquals(RaftError.EACCES.getNumber(), s.getCode());
+ assertEquals(RaftError.EACCES, s.getRaftError());
+ assertEquals("test world 100", s.getErrorMsg());
+ assertFalse(s.isOk());
+ }
+
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/closure/ClosureQueueTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/closure/ClosureQueueTest.java
new file mode 100644
index 0000000..2512e6a
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/closure/ClosureQueueTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.closure;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.Closure;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ClosureQueueTest {
+ private ClosureQueueImpl queue;
+
+ @Before
+ public void setup() {
+ this.queue = new ClosureQueueImpl();
+ }
+
+ @SuppressWarnings("SameParameterValue")
+ private Closure mockClosure(final CountDownLatch latch) {
+ return status -> {
+ if (latch != null) {
+ latch.countDown();
+ }
+ };
+ }
+
+ @Test
+ public void testAppendPop() {
+ for (int i = 0; i < 10; i++) {
+ this.queue.appendPendingClosure(mockClosure(null));
+ }
+ assertEquals(0, this.queue.getFirstIndex());
+ List<Closure> closures = new ArrayList<>();
+ assertEquals(0, this.queue.popClosureUntil(4, closures));
+ assertEquals(5, closures.size());
+
+ assertEquals(5, this.queue.getFirstIndex());
+
+ closures.clear();
+ assertEquals(5, this.queue.popClosureUntil(4, closures));
+ assertTrue(closures.isEmpty());
+ assertEquals(4, this.queue.popClosureUntil(3, closures));
+ assertTrue(closures.isEmpty());
+
+ assertEquals(-1, this.queue.popClosureUntil(10, closures));
+ assertTrue(closures.isEmpty());
+
+ //pop remaining 5 elements
+ assertEquals(5, this.queue.popClosureUntil(9, closures));
+ assertEquals(5, closures.size());
+ assertEquals(10, this.queue.getFirstIndex());
+ closures.clear();
+ assertEquals(2, this.queue.popClosureUntil(1, closures));
+ assertTrue(closures.isEmpty());
+ assertEquals(4, this.queue.popClosureUntil(3, closures));
+ assertTrue(closures.isEmpty());
+
+ for (int i = 0; i < 10; i++) {
+ this.queue.appendPendingClosure(mockClosure(null));
+ }
+
+ assertEquals(10, this.queue.popClosureUntil(15, closures));
+ assertEquals(6, closures.size());
+ assertEquals(16, this.queue.getFirstIndex());
+
+ assertEquals(-1, this.queue.popClosureUntil(20, closures));
+ assertTrue(closures.isEmpty());
+ assertEquals(16, this.queue.popClosureUntil(19, closures));
+ assertEquals(4, closures.size());
+ assertEquals(20, this.queue.getFirstIndex());
+ }
+
+ @Test
+ public void testResetFirstIndex() {
+ assertEquals(0, this.queue.getFirstIndex());
+ this.queue.resetFirstIndex(10);
+ assertEquals(10, this.queue.getFirstIndex());
+ for (int i = 0; i < 10; i++) {
+ this.queue.appendPendingClosure(mockClosure(null));
+ }
+
+ List<Closure> closures = new ArrayList<>();
+ assertEquals(5, this.queue.popClosureUntil(4, closures));
+ assertTrue(closures.isEmpty());
+ assertEquals(4, this.queue.popClosureUntil(3, closures));
+ assertTrue(closures.isEmpty());
+
+ assertEquals(10, this.queue.popClosureUntil(19, closures));
+ assertEquals(20, this.queue.getFirstIndex());
+ assertEquals(10, closures.size());
+ // empty ,return index+1
+ assertEquals(21, this.queue.popClosureUntil(20, closures));
+ assertTrue(closures.isEmpty());
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/closure/SynchronizedClosureTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/closure/SynchronizedClosureTest.java
new file mode 100644
index 0000000..94744bf
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/closure/SynchronizedClosureTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.closure;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.Status;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class SynchronizedClosureTest {
+ private SynchronizedClosure done;
+
+ @Before
+ public void setup() {
+ this.done = new SynchronizedClosure(1);
+ }
+
+ @Test
+ public void testAwaitRun() throws Exception {
+ CountDownLatch latch = new CountDownLatch(1);
+ AtomicLong cost = new AtomicLong(0);
+ new Thread() {
+ @Override
+ public void run() {
+ try {
+ long start = System.currentTimeMillis();
+ done.await();
+ cost.set(System.currentTimeMillis() - start);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ latch.countDown();
+ }
+ }.start();
+
+ int n = 1000;
+ Thread.sleep(n);
+ this.done.run(Status.OK());
+ latch.await();
+ assertEquals(n, cost.get(), 50);
+ assertTrue(this.done.getStatus().isOk());
+ }
+
+ @Test
+ public void testReset() throws Exception {
+ testAwaitRun();
+ this.done.await();
+ assertTrue(true);
+ this.done.reset();
+ assertNull(this.done.getStatus());
+ testAwaitRun();
+ assertTrue(this.done.getStatus().isOk());
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationEntryTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationEntryTest.java
new file mode 100644
index 0000000..009933c
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationEntryTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.conf;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.HashSet;
+
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+public class ConfigurationEntryTest {
+ @Test
+ public void testStuffMethods() {
+ ConfigurationEntry entry = TestUtils.getConfEntry("localhost:8081,localhost:8082,localhost:8083", null);
+ assertTrue(entry.isStable());
+ assertFalse(entry.isEmpty());
+ assertTrue(entry.contains(new PeerId("localhost", 8081)));
+ assertTrue(entry.contains(new PeerId("localhost", 8082)));
+ assertTrue(entry.contains(new PeerId("localhost", 8083)));
+ assertEquals(
+ entry.listPeers(),
+ new HashSet<>(Arrays.asList(new PeerId("localhost", 8081), new PeerId("localhost", 8082), new PeerId(
+ "localhost", 8083))));
+
+ }
+
+ @Test
+ public void testStuffMethodsWithPriority() {
+ ConfigurationEntry entry = TestUtils.getConfEntry(
+ "localhost:8081::100,localhost:8082::100,localhost:8083::100", null);
+ assertTrue(entry.isStable());
+ assertFalse(entry.isEmpty());
+ assertTrue(entry.contains(new PeerId("localhost", 8081, 0, 100)));
+ assertTrue(entry.contains(new PeerId("localhost", 8082, 0, 100)));
+ assertTrue(entry.contains(new PeerId("localhost", 8083, 0, 100)));
+ assertEquals(
+ entry.listPeers(),
+ new HashSet<>(Arrays.asList(new PeerId("localhost", 8081, 0, 100), new PeerId("localhost", 8082, 0, 100),
+ new PeerId("localhost", 8083, 0, 100))));
+
+ }
+
+ @Test
+ public void testIsValid() {
+ ConfigurationEntry entry = TestUtils.getConfEntry("localhost:8081,localhost:8082,localhost:8083", null);
+ assertTrue(entry.isValid());
+
+ entry = TestUtils.getConfEntry("localhost:8081,localhost:8082,localhost:8083",
+ "localhost:8081,localhost:8082,localhost:8084");
+ assertTrue(entry.isValid());
+
+ entry.getConf().addLearner(new PeerId("localhost", 8084));
+ assertFalse(entry.isValid());
+ entry.getConf().addLearner(new PeerId("localhost", 8081));
+ assertFalse(entry.isValid());
+ }
+
+ @Test
+ public void testIsStable() {
+ ConfigurationEntry entry = TestUtils.getConfEntry("localhost:8081,localhost:8082,localhost:8083",
+ "localhost:8080,localhost:8081,localhost:8082");
+ assertFalse(entry.isStable());
+ assertEquals(4, entry.listPeers().size());
+ assertTrue(entry.contains(new PeerId("localhost", 8080)));
+ assertTrue(entry.contains(new PeerId("localhost", 8081)));
+ assertTrue(entry.contains(new PeerId("localhost", 8082)));
+ assertTrue(entry.contains(new PeerId("localhost", 8083)));
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationManagerTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationManagerTest.java
new file mode 100644
index 0000000..d3771c6
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationManagerTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.conf;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.entity.LogId;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+public class ConfigurationManagerTest {
+
+ private ConfigurationManager confManager;
+
+ @Before
+ public void setup() {
+ this.confManager = new ConfigurationManager();
+ }
+
+ @Test
+ public void testGetStuff() {
+ ConfigurationEntry lastConf = this.confManager.getLastConfiguration();
+ ConfigurationEntry snapshot = this.confManager.getSnapshot();
+ assertSame(snapshot, lastConf);
+ assertSame(snapshot, this.confManager.get(0));
+
+ ConfigurationEntry confEntry1 = TestUtils.getConfEntry("localhost:8080", null);
+ confEntry1.setId(new LogId(0, 0));
+ assertTrue(this.confManager.add(confEntry1));
+ lastConf = this.confManager.getLastConfiguration();
+ assertNotSame(snapshot, lastConf);
+ assertSame(confEntry1, lastConf);
+
+ assertSame(confEntry1, this.confManager.get(0));
+ assertSame(confEntry1, this.confManager.get(1));
+ assertSame(confEntry1, this.confManager.get(2));
+
+ ConfigurationEntry confEntry2 = TestUtils.getConfEntry("localhost:8080,localhost:8081", "localhost:8080");
+ confEntry2.setId(new LogId(1, 1));
+ assertTrue(this.confManager.add(confEntry2));
+
+ lastConf = this.confManager.getLastConfiguration();
+ assertNotSame(snapshot, lastConf);
+ assertSame(confEntry2, lastConf);
+
+ assertSame(confEntry1, this.confManager.get(0));
+ assertSame(confEntry2, this.confManager.get(1));
+ assertSame(confEntry2, this.confManager.get(2));
+
+ ConfigurationEntry confEntry3 = TestUtils.getConfEntry("localhost:8080,localhost:8081,localhost:8082",
+ "localhost:8080,localhost:8081");
+ confEntry3.setId(new LogId(2, 1));
+ assertTrue(this.confManager.add(confEntry3));
+
+ lastConf = this.confManager.getLastConfiguration();
+ assertNotSame(snapshot, lastConf);
+ assertSame(confEntry3, lastConf);
+
+ assertSame(confEntry1, this.confManager.get(0));
+ assertSame(confEntry2, this.confManager.get(1));
+ assertSame(confEntry3, this.confManager.get(2));
+ }
+
+ private ConfigurationEntry createEnry(int index) {
+ ConfigurationEntry configurationEntry = new ConfigurationEntry();
+ configurationEntry.getId().setIndex(index);
+ return configurationEntry;
+ }
+
+ @Test
+ public void testTruncate() {
+ for (int i = 0; i < 10; i++) {
+ assertTrue(this.confManager.add(createEnry(i)));
+ }
+
+ assertEquals(9, this.confManager.getLastConfiguration().getId().getIndex());
+ assertEquals(5, this.confManager.get(5).getId().getIndex());
+ assertEquals(6, this.confManager.get(6).getId().getIndex());
+ this.confManager.truncatePrefix(6);
+ //truncated, so is snapshot index
+ assertEquals(0, this.confManager.get(5).getId().getIndex());
+ assertEquals(6, this.confManager.get(6).getId().getIndex());
+ assertEquals(9, this.confManager.getLastConfiguration().getId().getIndex());
+
+ this.confManager.truncateSuffix(7);
+ assertEquals(0, this.confManager.get(5).getId().getIndex());
+ assertEquals(6, this.confManager.get(6).getId().getIndex());
+ assertEquals(7, this.confManager.getLastConfiguration().getId().getIndex());
+ assertEquals(7, this.confManager.get(9).getId().getIndex());
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationTest.java
new file mode 100644
index 0000000..cb48fce
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.conf;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.entity.PeerId;
+
+public class ConfigurationTest {
+
+ @Test
+ public void testToStringParseStuff() {
+ final String confStr = "localhost:8081,localhost:8082,localhost:8083";
+ final Configuration conf = JRaftUtils.getConfiguration(confStr);
+ assertEquals(3, conf.size());
+ for (final PeerId peer : conf) {
+ assertTrue(peer.toString().startsWith("localhost:80"));
+ }
+ assertFalse(conf.isEmpty());
+ assertEquals(confStr, conf.toString());
+ final Configuration newConf = new Configuration();
+ assertTrue(newConf.parse(conf.toString()));
+ assertEquals(3, newConf.getPeerSet().size());
+ assertTrue(newConf.contains(new PeerId("localhost", 8081)));
+ assertTrue(newConf.contains(new PeerId("localhost", 8082)));
+ assertTrue(newConf.contains(new PeerId("localhost", 8083)));
+ assertEquals(confStr, newConf.toString());
+ assertEquals(conf.hashCode(), newConf.hashCode());
+ assertEquals(conf, newConf);
+ }
+
+ @Test
+ public void testToStringParseStuffWithPriority() {
+ final String confStr = "localhost:8081:1:100,localhost:8082:1:100,localhost:8083:1:100";
+ final Configuration conf = JRaftUtils.getConfiguration(confStr);
+ assertEquals(3, conf.size());
+ for (final PeerId peer : conf) {
+ assertTrue(peer.toString().startsWith("localhost:80"));
+ assertEquals(100, peer.getPriority());
+ assertEquals(1, peer.getIdx());
+ }
+ assertFalse(conf.isEmpty());
+ assertEquals(confStr, conf.toString());
+ final Configuration newConf = new Configuration();
+ assertTrue(newConf.parse(conf.toString()));
+ assertEquals(3, newConf.getPeerSet().size());
+ assertTrue(newConf.contains(new PeerId("localhost", 8081, 1, 100)));
+ assertTrue(newConf.contains(new PeerId("localhost", 8082, 1, 100)));
+ assertTrue(newConf.contains(new PeerId("localhost", 8083, 1, 100)));
+ assertEquals(confStr, newConf.toString());
+ assertEquals(conf.hashCode(), newConf.hashCode());
+ assertEquals(conf, newConf);
+ }
+
+ @Test
+ public void testToStringParseStuffWithPriorityAndNone() {
+ final String confStr = "localhost:8081,localhost:8082,localhost:8083:1:100";
+ final Configuration conf = JRaftUtils.getConfiguration(confStr);
+ assertEquals(3, conf.size());
+ for (final PeerId peer : conf) {
+ assertTrue(peer.toString().startsWith("localhost:80"));
+
+ if (peer.getIp().equals("localhost:8083")) {
+ assertEquals(100, peer.getPriority());
+ assertEquals(1, peer.getIdx());
+ }
+ }
+ assertFalse(conf.isEmpty());
+ assertEquals(confStr, conf.toString());
+ final Configuration newConf = new Configuration();
+ assertTrue(newConf.parse(conf.toString()));
+ assertEquals(3, newConf.getPeerSet().size());
+ assertTrue(newConf.contains(new PeerId("localhost", 8081)));
+ assertTrue(newConf.contains(new PeerId("localhost", 8082)));
+ assertTrue(newConf.contains(new PeerId("localhost", 8083, 1, 100)));
+ assertEquals(confStr, newConf.toString());
+ assertEquals(conf.hashCode(), newConf.hashCode());
+ assertEquals(conf, newConf);
+ }
+
+ @Test
+ public void testLearnerStuff() {
+ final String confStr = "localhost:8081,localhost:8082,localhost:8083";
+ final Configuration conf = JRaftUtils.getConfiguration(confStr);
+ assertEquals(3, conf.size());
+ assertEquals(confStr, conf.toString());
+ assertTrue(conf.isValid());
+
+ PeerId learner1 = new PeerId("192.168.1.1", 8081);
+ assertTrue(conf.addLearner(learner1));
+ assertFalse(conf.addLearner(learner1));
+ PeerId learner2 = new PeerId("192.168.1.2", 8081);
+ assertTrue(conf.addLearner(learner2));
+
+ assertEquals(2, conf.getLearners().size());
+ assertTrue(conf.getLearners().contains(learner1));
+ assertTrue(conf.getLearners().contains(learner2));
+
+ String newConfStr = "localhost:8081,localhost:8082,localhost:8083,192.168.1.1:8081/learner,192.168.1.2:8081/learner";
+ assertEquals(newConfStr, conf.toString());
+ assertTrue(conf.isValid());
+
+ final Configuration newConf = JRaftUtils.getConfiguration(newConfStr);
+ assertEquals(newConf, conf);
+ assertEquals(2, newConf.getLearners().size());
+ assertEquals(newConfStr, newConf.toString());
+ assertTrue(newConf.isValid());
+
+ // Also adds localhost:8081 as learner
+ assertTrue(conf.addLearner(new PeerId("localhost", 8081)));
+ // The conf is invalid, because the peers and learns have intersection.
+ assertFalse(conf.isValid());
+ }
+
+ @Test
+ public void testCopy() {
+ final Configuration conf = JRaftUtils.getConfiguration("localhost:8081,localhost:8082,localhost:8083");
+ final Configuration copied = conf.copy();
+ assertEquals(conf, copied);
+ assertNotSame(conf, copied);
+ assertEquals(copied.size(), 3);
+ assertEquals("localhost:8081,localhost:8082,localhost:8083", copied.toString());
+
+ final PeerId newPeer = new PeerId("localhost", 8084);
+ conf.addPeer(newPeer);
+ assertEquals(copied.size(), 3);
+ assertEquals(conf.size(), 4);
+ assertTrue(conf.contains(newPeer));
+ assertFalse(copied.contains(newPeer));
+ }
+
+ @Test
+ public void testReset() {
+ final Configuration conf = JRaftUtils.getConfiguration("localhost:8081,localhost:8082,localhost:8083");
+ assertFalse(conf.isEmpty());
+ conf.reset();
+ assertTrue(conf.isEmpty());
+ assertTrue(conf.getPeerSet().isEmpty());
+ }
+
+ @Test
+ public void testDiff() {
+ final Configuration conf1 = JRaftUtils.getConfiguration("localhost:8081,localhost:8082,localhost:8083");
+ final Configuration conf2 = JRaftUtils
+ .getConfiguration("localhost:8081,localhost:8083,localhost:8085,localhost:8086");
+ final Configuration included = new Configuration();
+ final Configuration excluded = new Configuration();
+ conf1.diff(conf2, included, excluded);
+ assertEquals("localhost:8082", included.toString());
+ assertEquals("localhost:8085,localhost:8086", excluded.toString());
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/core/BallotBoxTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/BallotBoxTest.java
new file mode 100644
index 0000000..a310e56
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/BallotBoxTest.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.FSMCaller;
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.closure.ClosureQueueImpl;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.BallotBoxOptions;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class BallotBoxTest {
+ private BallotBox box;
+ @Mock
+ private FSMCaller waiter;
+ private ClosureQueueImpl closureQueue;
+
+ @Before
+ public void setup() {
+ BallotBoxOptions opts = new BallotBoxOptions();
+ this.closureQueue = new ClosureQueueImpl();
+ opts.setClosureQueue(this.closureQueue);
+ opts.setWaiter(this.waiter);
+ box = new BallotBox();
+ assertTrue(box.init(opts));
+ }
+
+ @After
+ public void teardown() {
+ box.shutdown();
+ }
+
+ @Test
+ public void testResetPendingIndex() {
+ assertEquals(0, closureQueue.getFirstIndex());
+ assertEquals(0, box.getPendingIndex());
+ assertTrue(box.resetPendingIndex(1));
+ assertEquals(1, closureQueue.getFirstIndex());
+ assertEquals(1, box.getPendingIndex());
+ }
+
+ @Test
+ public void testAppendPendingTask() {
+ assertTrue(this.box.getPendingMetaQueue().isEmpty());
+ assertTrue(this.closureQueue.getQueue().isEmpty());
+ assertFalse(this.box.appendPendingTask(
+ JRaftUtils.getConfiguration("localhost:8081,localhost:8082,localhost:8083"),
+ JRaftUtils.getConfiguration("localhost:8081"), new Closure() {
+
+ @Override
+ public void run(Status status) {
+
+ }
+ }));
+ assertTrue(box.resetPendingIndex(1));
+ assertTrue(this.box.appendPendingTask(
+ JRaftUtils.getConfiguration("localhost:8081,localhost:8082,localhost:8083"),
+ JRaftUtils.getConfiguration("localhost:8081"), new Closure() {
+
+ @Override
+ public void run(Status status) {
+
+ }
+ }));
+
+ assertEquals(1, this.box.getPendingMetaQueue().size());
+ assertEquals(1, this.closureQueue.getQueue().size());
+ }
+
+ @Test
+ public void testClearPendingTasks() {
+ testAppendPendingTask();
+ this.box.clearPendingTasks();
+ assertTrue(this.box.getPendingMetaQueue().isEmpty());
+ assertTrue(this.closureQueue.getQueue().isEmpty());
+ assertEquals(0, closureQueue.getFirstIndex());
+ }
+
+ @Test
+ public void testCommitAt() {
+ assertFalse(this.box.commitAt(1, 3, new PeerId("localhost", 8081)));
+ assertTrue(box.resetPendingIndex(1));
+ assertTrue(this.box.appendPendingTask(
+ JRaftUtils.getConfiguration("localhost:8081,localhost:8082,localhost:8083"),
+ JRaftUtils.getConfiguration("localhost:8081"), new Closure() {
+
+ @Override
+ public void run(Status status) {
+
+ }
+ }));
+ assertEquals(0, this.box.getLastCommittedIndex());
+ try {
+ this.box.commitAt(1, 3, new PeerId("localhost", 8081));
+ fail();
+ } catch (ArrayIndexOutOfBoundsException e) {
+
+ }
+ assertTrue(this.box.commitAt(1, 1, new PeerId("localhost", 8081)));
+ assertEquals(0, this.box.getLastCommittedIndex());
+ assertEquals(1, this.box.getPendingIndex());
+ assertTrue(this.box.commitAt(1, 1, new PeerId("localhost", 8082)));
+ assertEquals(1, this.box.getLastCommittedIndex());
+ assertEquals(2, this.box.getPendingIndex());
+ Mockito.verify(this.waiter, Mockito.only()).onCommitted(1);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testSetLastCommittedIndexHasPending() {
+ assertTrue(box.resetPendingIndex(1));
+ assertFalse(this.box.setLastCommittedIndex(1));
+ }
+
+ @Test
+ public void testSetLastCommittedIndexLessThan() {
+ assertFalse(this.box.setLastCommittedIndex(-1));
+ }
+
+ @Test
+ public void testSetLastCommittedIndex() {
+ assertEquals(0, this.box.getLastCommittedIndex());
+ assertTrue(this.box.setLastCommittedIndex(1));
+ assertEquals(1, this.box.getLastCommittedIndex());
+ Mockito.verify(this.waiter, Mockito.only()).onCommitted(1);
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/core/CliServiceTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/CliServiceTest.java
new file mode 100644
index 0000000..d51258c
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/CliServiceTest.java
@@ -0,0 +1,492 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+import com.alipay.sofa.jraft.CliService;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.NodeManager;
+import com.alipay.sofa.jraft.RouteTable;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.entity.Task;
+import com.alipay.sofa.jraft.option.CliOptions;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class CliServiceTest {
+
+ private String dataPath;
+
+ private TestCluster cluster;
+ private final String groupId = "CliServiceTest";
+
+ private CliService cliService;
+
+ private Configuration conf;
+
+ @Rule
+ public TestName testName = new TestName();
+
+ private static final int LEARNER_PORT_STEP = 100;
+
+ @Before
+ public void setup() throws Exception {
+ System.out.println(">>>>>>>>>>>>>>> Start test method: " + this.testName.getMethodName());
+ this.dataPath = TestUtils.mkTempDir();
+ FileUtils.forceMkdir(new File(this.dataPath));
+ assertEquals(NodeImpl.GLOBAL_NUM_NODES.get(), 0);
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final LinkedHashSet<PeerId> learners = new LinkedHashSet<>();
+ //2 learners
+ for (int i = 0; i < 2; i++) {
+ learners.add(new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + LEARNER_PORT_STEP + i));
+ }
+
+ this.cluster = new TestCluster(this.groupId, this.dataPath, peers, learners, 300);
+ for (final PeerId peer : peers) {
+ this.cluster.start(peer.getEndpoint());
+ }
+
+ for (final PeerId peer : learners) {
+ this.cluster.startLearner(peer);
+ }
+
+ this.cluster.waitLeader();
+
+ this.cliService = new CliServiceImpl();
+ this.conf = new Configuration(peers, learners);
+ assertTrue(this.cliService.init(new CliOptions()));
+ }
+
+ @After
+ public void teardown() throws Exception {
+ this.cliService.shutdown();
+ this.cluster.stopAll();
+ if (NodeImpl.GLOBAL_NUM_NODES.get() > 0) {
+ Thread.sleep(1000);
+ assertEquals(NodeImpl.GLOBAL_NUM_NODES.get(), 0);
+ }
+ FileUtils.deleteDirectory(new File(this.dataPath));
+ NodeManager.getInstance().clear();
+ RouteTable.getInstance().reset();
+ System.out.println(">>>>>>>>>>>>>>> End test method: " + this.testName.getMethodName());
+ }
+
+ @Test
+ public void testTransferLeader() throws Exception {
+ final PeerId leader = this.cluster.getLeader().getNodeId().getPeerId().copy();
+ assertNotNull(leader);
+
+ final Set<PeerId> peers = this.conf.getPeerSet();
+ PeerId targetPeer = null;
+ for (final PeerId peer : peers) {
+ if (!peer.equals(leader)) {
+ targetPeer = peer;
+ break;
+ }
+ }
+ assertNotNull(targetPeer);
+ assertTrue(this.cliService.transferLeader(this.groupId, this.conf, targetPeer).isOk());
+ this.cluster.waitLeader();
+ assertEquals(targetPeer, this.cluster.getLeader().getNodeId().getPeerId());
+ }
+
+ @SuppressWarnings("SameParameterValue")
+ private void sendTestTaskAndWait(final Node node, final int code) throws InterruptedException {
+ final CountDownLatch latch = new CountDownLatch(10);
+ for (int i = 0; i < 10; i++) {
+ final ByteBuffer data = ByteBuffer.wrap(("hello" + i).getBytes());
+ final Task task = new Task(data, new ExpectClosure(code, null, latch));
+ node.apply(task);
+ }
+ assertTrue(latch.await(10, TimeUnit.SECONDS));
+ }
+
+ @Test
+ public void testLearnerServices() throws Exception {
+ final PeerId learner3 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + LEARNER_PORT_STEP + 3);
+ assertTrue(this.cluster.startLearner(learner3));
+ sendTestTaskAndWait(this.cluster.getLeader(), 0);
+ Thread.sleep(500);
+ for (final MockStateMachine fsm : this.cluster.getFsms()) {
+ if (!fsm.getAddress().equals(learner3.getEndpoint())) {
+ assertEquals(10, fsm.getLogs().size());
+ }
+ }
+ assertEquals(0, this.cluster.getFsmByPeer(learner3).getLogs().size());
+ List<PeerId> oldLearners = new ArrayList<PeerId>(this.conf.getLearners());
+ assertEquals(oldLearners, this.cliService.getLearners(this.groupId, this.conf));
+ assertEquals(oldLearners, this.cliService.getAliveLearners(this.groupId, this.conf));
+
+ // Add learner3
+ this.cliService.addLearners(this.groupId, this.conf, Arrays.asList(learner3));
+ Thread.sleep(1000);
+ assertEquals(10, this.cluster.getFsmByPeer(learner3).getLogs().size());
+
+ sendTestTaskAndWait(this.cluster.getLeader(), 0);
+ Thread.sleep(1000);
+ for (final MockStateMachine fsm : this.cluster.getFsms()) {
+ assertEquals(20, fsm.getLogs().size());
+
+ }
+ List<PeerId> newLearners = new ArrayList<>(oldLearners);
+ newLearners.add(learner3);
+ assertEquals(newLearners, this.cliService.getLearners(this.groupId, this.conf));
+ assertEquals(newLearners, this.cliService.getAliveLearners(this.groupId, this.conf));
+
+ // Remove 3
+ this.cliService.removeLearners(this.groupId, this.conf, Arrays.asList(learner3));
+ sendTestTaskAndWait(this.cluster.getLeader(), 0);
+ Thread.sleep(1000);
+ for (final MockStateMachine fsm : this.cluster.getFsms()) {
+ if (!fsm.getAddress().equals(learner3.getEndpoint())) {
+ assertEquals(30, fsm.getLogs().size());
+ }
+ }
+ // Latest 10 logs are not replicated to learner3, because it's removed.
+ assertEquals(20, this.cluster.getFsmByPeer(learner3).getLogs().size());
+ assertEquals(oldLearners, this.cliService.getLearners(this.groupId, this.conf));
+ assertEquals(oldLearners, this.cliService.getAliveLearners(this.groupId, this.conf));
+
+ // Set learners into [learner3]
+ this.cliService.resetLearners(this.groupId, this.conf, Arrays.asList(learner3));
+ Thread.sleep(100);
+ assertEquals(30, this.cluster.getFsmByPeer(learner3).getLogs().size());
+
+ sendTestTaskAndWait(this.cluster.getLeader(), 0);
+ Thread.sleep(1000);
+ // Latest 10 logs are not replicated to learner1 and learner2, because they were removed by resetting learners set.
+ for (final MockStateMachine fsm : this.cluster.getFsms()) {
+ if (!oldLearners.contains(new PeerId(fsm.getAddress(), 0))) {
+ assertEquals(40, fsm.getLogs().size());
+ } else {
+ assertEquals(30, fsm.getLogs().size());
+ }
+ }
+ assertEquals(Arrays.asList(learner3), this.cliService.getLearners(this.groupId, this.conf));
+ assertEquals(Arrays.asList(learner3), this.cliService.getAliveLearners(this.groupId, this.conf));
+
+ // Stop learner3
+ this.cluster.stop(learner3.getEndpoint());
+ Thread.sleep(1000);
+ assertEquals(Arrays.asList(learner3), this.cliService.getLearners(this.groupId, this.conf));
+ assertTrue(this.cliService.getAliveLearners(this.groupId, this.conf).isEmpty());
+ }
+
+ @Test
+ public void testAddPeerRemovePeer() throws Exception {
+ final PeerId peer3 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 3);
+ assertTrue(this.cluster.start(peer3.getEndpoint()));
+ sendTestTaskAndWait(this.cluster.getLeader(), 0);
+ Thread.sleep(100);
+ assertEquals(0, this.cluster.getFsmByPeer(peer3).getLogs().size());
+
+ assertTrue(this.cliService.addPeer(this.groupId, this.conf, peer3).isOk());
+ Thread.sleep(100);
+ assertEquals(10, this.cluster.getFsmByPeer(peer3).getLogs().size());
+ sendTestTaskAndWait(this.cluster.getLeader(), 0);
+ Thread.sleep(100);
+ assertEquals(6, this.cluster.getFsms().size());
+ for (final MockStateMachine fsm : this.cluster.getFsms()) {
+ assertEquals(20, fsm.getLogs().size());
+ }
+
+ //remove peer3
+ assertTrue(this.cliService.removePeer(this.groupId, this.conf, peer3).isOk());
+ Thread.sleep(200);
+ sendTestTaskAndWait(this.cluster.getLeader(), 0);
+ Thread.sleep(1000);
+ assertEquals(6, this.cluster.getFsms().size());
+ for (final MockStateMachine fsm : this.cluster.getFsms()) {
+ if (fsm.getAddress().equals(peer3.getEndpoint())) {
+ assertEquals(20, fsm.getLogs().size());
+ } else {
+ assertEquals(30, fsm.getLogs().size());
+ }
+ }
+ }
+
+ @Test
+ public void testChangePeers() throws Exception {
+ final List<PeerId> newPeers = TestUtils.generatePeers(10);
+ newPeers.removeAll(this.conf.getPeerSet());
+ for (final PeerId peer : newPeers) {
+ assertTrue(this.cluster.start(peer.getEndpoint()));
+ }
+ this.cluster.waitLeader();
+ final Node oldLeaderNode = this.cluster.getLeader();
+ assertNotNull(oldLeaderNode);
+ final PeerId oldLeader = oldLeaderNode.getNodeId().getPeerId();
+ assertNotNull(oldLeader);
+ assertTrue(this.cliService.changePeers(this.groupId, this.conf, new Configuration(newPeers)).isOk());
+ this.cluster.waitLeader();
+ final PeerId newLeader = this.cluster.getLeader().getNodeId().getPeerId();
+ assertNotEquals(oldLeader, newLeader);
+ assertTrue(newPeers.contains(newLeader));
+ }
+
+ @Test
+ public void testSnapshot() throws Exception {
+ sendTestTaskAndWait(this.cluster.getLeader(), 0);
+ assertEquals(5, this.cluster.getFsms().size());
+ for (final MockStateMachine fsm : this.cluster.getFsms()) {
+ assertEquals(0, fsm.getSaveSnapshotTimes());
+ }
+
+ for (final PeerId peer : this.conf) {
+ assertTrue(this.cliService.snapshot(this.groupId, peer).isOk());
+ }
+ for (final PeerId peer : this.conf.getLearners()) {
+ assertTrue(this.cliService.snapshot(this.groupId, peer).isOk());
+ }
+ Thread.sleep(1000);
+ for (final MockStateMachine fsm : this.cluster.getFsms()) {
+ assertEquals(1, fsm.getSaveSnapshotTimes());
+ }
+ }
+
+ @Test
+ public void testGetPeers() throws Exception {
+ PeerId leader = this.cluster.getLeader().getNodeId().getPeerId();
+ assertNotNull(leader);
+ assertArrayEquals(this.conf.getPeerSet().toArray(),
+ new HashSet<>(this.cliService.getPeers(this.groupId, this.conf)).toArray());
+
+ // stop one peer
+ final List<PeerId> peers = this.conf.getPeers();
+ this.cluster.stop(peers.get(0).getEndpoint());
+
+ this.cluster.waitLeader();
+
+ leader = this.cluster.getLeader().getNodeId().getPeerId();
+ assertNotNull(leader);
+ assertArrayEquals(this.conf.getPeerSet().toArray(),
+ new HashSet<>(this.cliService.getPeers(this.groupId, this.conf)).toArray());
+
+ this.cluster.stopAll();
+
+ try {
+ this.cliService.getPeers(this.groupId, this.conf);
+ fail();
+ } catch (final IllegalStateException e) {
+ assertEquals("Fail to get leader of group " + this.groupId, e.getMessage());
+ }
+ }
+
+ @Test
+ public void testGetAlivePeers() throws Exception {
+ PeerId leader = this.cluster.getLeader().getNodeId().getPeerId();
+ assertNotNull(leader);
+ assertArrayEquals(this.conf.getPeerSet().toArray(),
+ new HashSet<>(this.cliService.getAlivePeers(this.groupId, this.conf)).toArray());
+
+ // stop one peer
+ final List<PeerId> peers = this.conf.getPeers();
+ this.cluster.stop(peers.get(0).getEndpoint());
+ peers.remove(0);
+
+ this.cluster.waitLeader();
+
+ Thread.sleep(1000);
+
+ leader = this.cluster.getLeader().getNodeId().getPeerId();
+ assertNotNull(leader);
+ assertArrayEquals(new HashSet<>(peers).toArray(),
+ new HashSet<>(this.cliService.getAlivePeers(this.groupId, this.conf)).toArray());
+
+ this.cluster.stopAll();
+
+ try {
+ this.cliService.getAlivePeers(this.groupId, this.conf);
+ fail();
+ } catch (final IllegalStateException e) {
+ assertEquals("Fail to get leader of group " + this.groupId, e.getMessage());
+ }
+ }
+
+ @Test
+ public void testRebalance() {
+ final Set<String> groupIds = new TreeSet<>();
+ groupIds.add("group_1");
+ groupIds.add("group_2");
+ groupIds.add("group_3");
+ groupIds.add("group_4");
+ groupIds.add("group_5");
+ groupIds.add("group_6");
+ groupIds.add("group_7");
+ groupIds.add("group_8");
+ final Configuration conf = new Configuration();
+ conf.addPeer(new PeerId("host_1", 8080));
+ conf.addPeer(new PeerId("host_2", 8080));
+ conf.addPeer(new PeerId("host_3", 8080));
+
+ final Map<String, PeerId> rebalancedLeaderIds = new HashMap<>();
+
+ final CliService cliService = new MockCliService(rebalancedLeaderIds, new PeerId("host_1", 8080));
+
+ assertTrue(cliService.rebalance(groupIds, conf, rebalancedLeaderIds).isOk());
+ assertEquals(groupIds.size(), rebalancedLeaderIds.size());
+
+ final Map<PeerId, Integer> ret = new HashMap<>();
+ for (Map.Entry<String, PeerId> entry : rebalancedLeaderIds.entrySet()) {
+ ret.compute(entry.getValue(), (ignored, num) -> num == null ? 1 : num + 1);
+ }
+ final int expectedAvgLeaderNum = (int) Math.ceil((double) groupIds.size() / conf.size());
+ for (Map.Entry<PeerId, Integer> entry : ret.entrySet()) {
+ System.out.println(entry);
+ assertTrue(entry.getValue() <= expectedAvgLeaderNum);
+ }
+ }
+
+ @Test
+ public void testRebalanceOnLeaderFail() {
+ final Set<String> groupIds = new TreeSet<>();
+ groupIds.add("group_1");
+ groupIds.add("group_2");
+ groupIds.add("group_3");
+ groupIds.add("group_4");
+ final Configuration conf = new Configuration();
+ conf.addPeer(new PeerId("host_1", 8080));
+ conf.addPeer(new PeerId("host_2", 8080));
+ conf.addPeer(new PeerId("host_3", 8080));
+
+ final Map<String, PeerId> rebalancedLeaderIds = new HashMap<>();
+
+ final CliService cliService = new MockLeaderFailCliService();
+
+ assertEquals("Fail to get leader", cliService.rebalance(groupIds, conf, rebalancedLeaderIds).getErrorMsg());
+ }
+
+ @Test
+ public void testRelalanceOnTransferLeaderFail() {
+ final Set<String> groupIds = new TreeSet<>();
+ groupIds.add("group_1");
+ groupIds.add("group_2");
+ groupIds.add("group_3");
+ groupIds.add("group_4");
+ groupIds.add("group_5");
+ groupIds.add("group_6");
+ groupIds.add("group_7");
+ final Configuration conf = new Configuration();
+ conf.addPeer(new PeerId("host_1", 8080));
+ conf.addPeer(new PeerId("host_2", 8080));
+ conf.addPeer(new PeerId("host_3", 8080));
+
+ final Map<String, PeerId> rebalancedLeaderIds = new HashMap<>();
+
+ final CliService cliService = new MockTransferLeaderFailCliService(rebalancedLeaderIds,
+ new PeerId("host_1", 8080));
+
+ assertEquals("Fail to transfer leader",
+ cliService.rebalance(groupIds, conf, rebalancedLeaderIds).getErrorMsg());
+ assertTrue(groupIds.size() >= rebalancedLeaderIds.size());
+
+ final Map<PeerId, Integer> ret = new HashMap<>();
+ for (Map.Entry<String, PeerId> entry : rebalancedLeaderIds.entrySet()) {
+ ret.compute(entry.getValue(), (ignored, num) -> num == null ? 1 : num + 1);
+ }
+ for (Map.Entry<PeerId, Integer> entry : ret.entrySet()) {
+ System.out.println(entry);
+ assertEquals(new PeerId("host_1", 8080), entry.getKey());
+ }
+ }
+
+ class MockCliService extends CliServiceImpl {
+
+ private final Map<String, PeerId> rebalancedLeaderIds;
+ private final PeerId initialLeaderId;
+
+ MockCliService(final Map<String, PeerId> rebalancedLeaderIds, final PeerId initialLeaderId) {
+ this.rebalancedLeaderIds = rebalancedLeaderIds;
+ this.initialLeaderId = initialLeaderId;
+ }
+
+ @Override
+ public Status getLeader(final String groupId, final Configuration conf, final PeerId leaderId) {
+ final PeerId ret = this.rebalancedLeaderIds.get(groupId);
+ if (ret != null) {
+ leaderId.parse(ret.toString());
+ } else {
+ leaderId.parse(this.initialLeaderId.toString());
+ }
+ return Status.OK();
+ }
+
+ @Override
+ public List<PeerId> getAlivePeers(final String groupId, final Configuration conf) {
+ return conf.getPeers();
+ }
+
+ @Override
+ public Status transferLeader(final String groupId, final Configuration conf, final PeerId peer) {
+ return Status.OK();
+ }
+ }
+
+ class MockLeaderFailCliService extends MockCliService {
+
+ MockLeaderFailCliService() {
+ super(null, null);
+ }
+
+ @Override
+ public Status getLeader(final String groupId, final Configuration conf, final PeerId leaderId) {
+ return new Status(-1, "Fail to get leader");
+ }
+ }
+
+ class MockTransferLeaderFailCliService extends MockCliService {
+
+ MockTransferLeaderFailCliService(final Map<String, PeerId> rebalancedLeaderIds, final PeerId initialLeaderId) {
+ super(rebalancedLeaderIds, initialLeaderId);
+ }
+
+ @Override
+ public Status transferLeader(final String groupId, final Configuration conf, final PeerId peer) {
+ return new Status(-1, "Fail to transfer leader");
+ }
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/core/ExpectClosure.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/ExpectClosure.java
new file mode 100644
index 0000000..111151b
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/ExpectClosure.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.util.concurrent.CountDownLatch;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.error.RaftError;
+
+import static org.junit.Assert.assertEquals;
+
+public class ExpectClosure implements Closure {
+ private int expectedErrCode;
+ private String expectErrMsg;
+ private CountDownLatch latch;
+
+ public ExpectClosure(CountDownLatch latch) {
+ this(RaftError.SUCCESS, latch);
+ }
+
+ public ExpectClosure(RaftError expectedErrCode, CountDownLatch latch) {
+ this(expectedErrCode, null, latch);
+
+ }
+
+ public ExpectClosure(RaftError expectedErrCode, String expectErrMsg, CountDownLatch latch) {
+ super();
+ this.expectedErrCode = expectedErrCode.getNumber();
+ this.expectErrMsg = expectErrMsg;
+ this.latch = latch;
+ }
+
+ public ExpectClosure(int code, String expectErrMsg, CountDownLatch latch) {
+ super();
+ this.expectedErrCode = code;
+ this.expectErrMsg = expectErrMsg;
+ this.latch = latch;
+ }
+
+ @Override
+ public void run(Status status) {
+ if (this.expectedErrCode >= 0) {
+ assertEquals(this.expectedErrCode, status.getCode());
+ }
+ if (this.expectErrMsg != null) {
+ assertEquals(this.expectErrMsg, status.getErrorMsg());
+ }
+ latch.countDown();
+ }
+
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/core/FSMCallerTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/FSMCallerTest.java
new file mode 100644
index 0000000..da58426
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/FSMCallerTest.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.concurrent.CountDownLatch;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Iterator;
+import com.alipay.sofa.jraft.StateMachine;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.closure.ClosureQueueImpl;
+import com.alipay.sofa.jraft.closure.LoadSnapshotClosure;
+import com.alipay.sofa.jraft.closure.SaveSnapshotClosure;
+import com.alipay.sofa.jraft.entity.EnumOutter.EntryType;
+import com.alipay.sofa.jraft.entity.EnumOutter.ErrorType;
+import com.alipay.sofa.jraft.entity.LeaderChangeContext;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.entity.LogId;
+import com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.error.RaftException;
+import com.alipay.sofa.jraft.option.FSMCallerOptions;
+import com.alipay.sofa.jraft.storage.LogManager;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class FSMCallerTest {
+ private FSMCallerImpl fsmCaller;
+ @Mock
+ private NodeImpl node;
+ @Mock
+ private StateMachine fsm;
+ @Mock
+ private LogManager logManager;
+ private ClosureQueueImpl closureQueue;
+
+ @Before
+ public void setup() {
+ this.fsmCaller = new FSMCallerImpl();
+ this.closureQueue = new ClosureQueueImpl();
+ final FSMCallerOptions opts = new FSMCallerOptions();
+ Mockito.when(this.node.getNodeMetrics()).thenReturn(new NodeMetrics(false));
+ opts.setNode(this.node);
+ opts.setFsm(this.fsm);
+ opts.setLogManager(this.logManager);
+ opts.setBootstrapId(new LogId(10, 1));
+ opts.setClosureQueue(this.closureQueue);
+ assertTrue(this.fsmCaller.init(opts));
+ }
+
+ @After
+ public void teardown() throws Exception {
+ if (this.fsmCaller != null) {
+ this.fsmCaller.shutdown();
+ this.fsmCaller.join();
+ }
+ }
+
+ @Test
+ public void testShutdownJoin() throws Exception {
+ this.fsmCaller.shutdown();
+ this.fsmCaller.join();
+ this.fsmCaller = null;
+ }
+
+ @Test
+ public void testOnCommittedError() throws Exception {
+ Mockito.when(this.logManager.getTerm(10)).thenReturn(1L);
+ Mockito.when(this.logManager.getEntry(11)).thenReturn(null);
+
+ assertTrue(this.fsmCaller.onCommitted(11));
+
+ this.fsmCaller.flush();
+ assertEquals(this.fsmCaller.getLastAppliedIndex(), 10);
+ Mockito.verify(this.logManager).setAppliedId(new LogId(10, 1));
+ assertFalse(this.fsmCaller.getError().getStatus().isOk());
+ assertEquals("Fail to get entry at index=11 while committed_index=11", this.fsmCaller.getError().getStatus()
+ .getErrorMsg());
+ }
+
+ @Test
+ public void testOnCommitted() throws Exception {
+ final LogEntry log = new LogEntry(EntryType.ENTRY_TYPE_DATA);
+ log.getId().setIndex(11);
+ log.getId().setTerm(1);
+ Mockito.when(this.logManager.getTerm(11)).thenReturn(1L);
+ Mockito.when(this.logManager.getEntry(11)).thenReturn(log);
+ final ArgumentCaptor<Iterator> itArg = ArgumentCaptor.forClass(Iterator.class);
+
+ assertTrue(this.fsmCaller.onCommitted(11));
+
+ this.fsmCaller.flush();
+ assertEquals(this.fsmCaller.getLastAppliedIndex(), 11);
+ Mockito.verify(this.fsm).onApply(itArg.capture());
+ final Iterator it = itArg.getValue();
+ assertFalse(it.hasNext());
+ assertEquals(it.getIndex(), 12);
+ Mockito.verify(this.logManager).setAppliedId(new LogId(11, 1));
+ assertTrue(this.fsmCaller.getError().getStatus().isOk());
+ }
+
+ @Test
+ public void testOnSnapshotLoad() throws Exception {
+ final SnapshotReader reader = Mockito.mock(SnapshotReader.class);
+
+ final SnapshotMeta meta = SnapshotMeta.newBuilder().setLastIncludedIndex(12).setLastIncludedTerm(1).build();
+ Mockito.when(reader.load()).thenReturn(meta);
+ Mockito.when(this.fsm.onSnapshotLoad(reader)).thenReturn(true);
+ final CountDownLatch latch = new CountDownLatch(1);
+ this.fsmCaller.onSnapshotLoad(new LoadSnapshotClosure() {
+
+ @Override
+ public void run(final Status status) {
+ assertTrue(status.isOk());
+ latch.countDown();
+ }
+
+ @Override
+ public SnapshotReader start() {
+ return reader;
+ }
+ });
+ latch.await();
+ assertEquals(this.fsmCaller.getLastAppliedIndex(), 12);
+ Mockito.verify(this.fsm).onConfigurationCommitted(Mockito.any());
+ }
+
+ @Test
+ public void testOnSnapshotLoadFSMError() throws Exception {
+ final SnapshotReader reader = Mockito.mock(SnapshotReader.class);
+
+ final SnapshotMeta meta = SnapshotMeta.newBuilder().setLastIncludedIndex(12).setLastIncludedTerm(1).build();
+ Mockito.when(reader.load()).thenReturn(meta);
+ Mockito.when(this.fsm.onSnapshotLoad(reader)).thenReturn(false);
+ final CountDownLatch latch = new CountDownLatch(1);
+ this.fsmCaller.onSnapshotLoad(new LoadSnapshotClosure() {
+
+ @Override
+ public void run(final Status status) {
+ assertFalse(status.isOk());
+ assertEquals(-1, status.getCode());
+ assertEquals("StateMachine onSnapshotLoad failed", status.getErrorMsg());
+ latch.countDown();
+ }
+
+ @Override
+ public SnapshotReader start() {
+ return reader;
+ }
+ });
+ latch.await();
+ assertEquals(this.fsmCaller.getLastAppliedIndex(), 10);
+ }
+
+ @Test
+ public void testOnSnapshotSaveEmptyConf() throws Exception {
+ final CountDownLatch latch = new CountDownLatch(1);
+ this.fsmCaller.onSnapshotSave(new SaveSnapshotClosure() {
+
+ @Override
+ public void run(final Status status) {
+ assertFalse(status.isOk());
+ assertEquals("Empty conf entry for lastAppliedIndex=10", status.getErrorMsg());
+ latch.countDown();
+ }
+
+ @Override
+ public SnapshotWriter start(final SnapshotMeta meta) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+ });
+ latch.await();
+ }
+
+ @Test
+ public void testOnSnapshotSave() throws Exception {
+ final SnapshotWriter writer = Mockito.mock(SnapshotWriter.class);
+ Mockito.when(this.logManager.getConfiguration(10)).thenReturn(
+ TestUtils.getConfEntry("localhost:8081,localhost:8082,localhost:8083", "localhost:8081"));
+ final SaveSnapshotClosure done = new SaveSnapshotClosure() {
+
+ @Override
+ public void run(final Status status) {
+
+ }
+
+ @Override
+ public SnapshotWriter start(final SnapshotMeta meta) {
+ assertEquals(10, meta.getLastIncludedIndex());
+ return writer;
+ }
+ };
+ this.fsmCaller.onSnapshotSave(done);
+ this.fsmCaller.flush();
+ Mockito.verify(this.fsm).onSnapshotSave(writer, done);
+ }
+
+ @Test
+ public void testOnLeaderStartStop() throws Exception {
+ this.fsmCaller.onLeaderStart(11);
+ this.fsmCaller.flush();
+ Mockito.verify(this.fsm).onLeaderStart(11);
+
+ final Status status = new Status(-1, "test");
+ this.fsmCaller.onLeaderStop(status);
+ this.fsmCaller.flush();
+ Mockito.verify(this.fsm).onLeaderStop(status);
+ }
+
+ @Test
+ public void testOnStartStopFollowing() throws Exception {
+ final LeaderChangeContext ctx = new LeaderChangeContext(null, 11, Status.OK());
+ this.fsmCaller.onStartFollowing(ctx);
+ this.fsmCaller.flush();
+ Mockito.verify(this.fsm).onStartFollowing(ctx);
+
+ this.fsmCaller.onStopFollowing(ctx);
+ this.fsmCaller.flush();
+ Mockito.verify(this.fsm).onStopFollowing(ctx);
+ }
+
+ @Test
+ public void testOnError() throws Exception {
+ this.fsmCaller.onError(new RaftException(ErrorType.ERROR_TYPE_LOG, new Status(-1, "test")));
+ this.fsmCaller.flush();
+ assertFalse(this.fsmCaller.getError().getStatus().isOk());
+ assertEquals(ErrorType.ERROR_TYPE_LOG, this.fsmCaller.getError().getType());
+ Mockito.verify(this.node).onError(Mockito.any());
+ Mockito.verify(this.fsm).onError(Mockito.any());
+ }
+
+ @Test
+ public void testOnSnapshotLoadStale() throws Exception {
+ final SnapshotReader reader = Mockito.mock(SnapshotReader.class);
+
+ final SnapshotMeta meta = SnapshotMeta.newBuilder().setLastIncludedIndex(5).setLastIncludedTerm(1).build();
+ Mockito.when(reader.load()).thenReturn(meta);
+
+ final CountDownLatch latch = new CountDownLatch(1);
+ this.fsmCaller.onSnapshotLoad(new LoadSnapshotClosure() {
+
+ @Override
+ public void run(final Status status) {
+ assertFalse(status.isOk());
+ assertEquals(RaftError.ESTALE, status.getRaftError());
+ latch.countDown();
+ }
+
+ @Override
+ public SnapshotReader start() {
+ return reader;
+ }
+ });
+ latch.await();
+ assertEquals(this.fsmCaller.getLastAppliedIndex(), 10);
+ }
+
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/core/IteratorImplTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/IteratorImplTest.java
new file mode 100644
index 0000000..2de7e32
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/IteratorImplTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.StateMachine;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.EnumOutter;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.storage.LogManager;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class IteratorImplTest {
+
+ private IteratorImpl iter;
+ @Mock
+ private StateMachine fsm;
+ @Mock
+ private LogManager logManager;
+ private List<Closure> closures;
+ private AtomicLong applyingIndex;
+
+ @Before
+ public void setup() {
+ this.applyingIndex = new AtomicLong(0);
+ this.closures = new ArrayList<>();
+ for (int i = 0; i < 11; i++) {
+ this.closures.add(new MockClosure());
+ final LogEntry log = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+ log.getId().setIndex(i);
+ log.getId().setTerm(1);
+ Mockito.when(this.logManager.getEntry(i)).thenReturn(log);
+ }
+ this.iter = new IteratorImpl(fsm, logManager, closures, 0L, 0L, 10L, applyingIndex);
+ }
+
+ @Test
+ public void testPredicates() {
+ assertTrue(this.iter.isGood());
+ assertFalse(this.iter.hasError());
+ }
+
+ @Test
+ public void testNext() {
+ int i = 1;
+ while (iter.isGood()) {
+ assertEquals(i, iter.getIndex());
+ assertNotNull(iter.done());
+ final LogEntry log = iter.entry();
+ assertEquals(i, log.getId().getIndex());
+ assertEquals(1, log.getId().getTerm());
+ iter.next();
+ i++;
+ }
+ assertEquals(i, 11);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testSetErrorAndRollbackInvalid() {
+ this.iter.setErrorAndRollback(-1, null);
+ }
+
+ @Test
+ public void testRunTheRestClosureWithError() throws Exception {
+ testSetErrorAndRollback();
+ for (final Closure closure : this.closures) {
+ final MockClosure mc = (MockClosure) closure;
+ assertNull(mc.s);
+ }
+
+ this.iter.runTheRestClosureWithError();
+ Thread.sleep(500);
+ int i = 0;
+ for (final Closure closure : this.closures) {
+ i++;
+ final MockClosure mc = (MockClosure) closure;
+ if (i < 7) {
+ assertNull(mc.s);
+ } else {
+ final Status s = mc.s;
+ Assert.assertEquals(RaftError.ESTATEMACHINE.getNumber(), s.getCode());
+ assertEquals(
+ "StateMachine meet critical error when applying one or more tasks since index=6, Status[UNKNOWN<-1>: test]",
+ s.getErrorMsg());
+ }
+ }
+ }
+
+ @Test
+ public void testSetErrorAndRollback() {
+ testNext();
+ assertFalse(iter.hasError());
+ this.iter.setErrorAndRollback(5, new Status(-1, "test"));
+ assertTrue(iter.hasError());
+ Assert.assertEquals(EnumOutter.ErrorType.ERROR_TYPE_STATE_MACHINE, iter.getError().getType());
+ Assert.assertEquals(RaftError.ESTATEMACHINE.getNumber(), iter.getError().getStatus().getCode());
+ Assert
+ .assertEquals(
+ "StateMachine meet critical error when applying one or more tasks since index=6, Status[UNKNOWN<-1>: test]",
+ iter.getError().getStatus().getErrorMsg());
+ assertEquals(6, iter.getIndex());
+ }
+
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/core/IteratorTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/IteratorTest.java
new file mode 100644
index 0000000..a722193
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/IteratorTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Iterator;
+import com.alipay.sofa.jraft.StateMachine;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.EnumOutter;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.storage.LogManager;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class IteratorTest {
+
+ private IteratorImpl iterImpl;
+ private Iterator iter;
+
+ @Mock
+ private StateMachine fsm;
+ @Mock
+ private LogManager logManager;
+ private List<Closure> closures;
+ private AtomicLong applyingIndex;
+
+ @Before
+ public void setup() {
+ this.applyingIndex = new AtomicLong(0);
+ this.closures = new ArrayList<>();
+ for (int i = 0; i < 11; i++) {
+ this.closures.add(new MockClosure());
+ final LogEntry log = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_DATA);
+ log.getId().setIndex(i);
+ log.getId().setTerm(1);
+ log.setData(ByteBuffer.allocate(i));
+ Mockito.when(this.logManager.getEntry(i)).thenReturn(log);
+ }
+ this.iterImpl = new IteratorImpl(fsm, logManager, closures, 0L, 0L, 10L, applyingIndex);
+ this.iter = new IteratorWrapper(iterImpl);
+ }
+
+ @Test
+ public void testPredicates() {
+ assertTrue(this.iter.hasNext());
+ }
+
+ @Test
+ public void testNext() {
+ int i = 1;
+ while (iter.hasNext()) {
+ assertEquals(i, iter.getIndex());
+ assertNotNull(iter.done());
+ assertEquals(i, iter.getIndex());
+ assertEquals(1, iter.getTerm());
+ assertEquals(i, iter.getData().remaining());
+ iter.next();
+ i++;
+ }
+ assertEquals(i, 11);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testSetErrorAndRollbackInvalid() {
+ this.iter.setErrorAndRollback(-1, null);
+ }
+
+ @Test
+ public void testSetErrorAndRollback() {
+ testNext();
+ assertFalse(iterImpl.hasError());
+ this.iter.setErrorAndRollback(5, new Status(-1, "test"));
+ assertTrue(iterImpl.hasError());
+ Assert.assertEquals(EnumOutter.ErrorType.ERROR_TYPE_STATE_MACHINE, iterImpl.getError().getType());
+ Assert.assertEquals(RaftError.ESTATEMACHINE.getNumber(), iterImpl.getError().getStatus().getCode());
+ Assert
+ .assertEquals(
+ "StateMachine meet critical error when applying one or more tasks since index=6, Status[UNKNOWN<-1>: test]",
+ iterImpl.getError().getStatus().getErrorMsg());
+ assertEquals(6, iter.getIndex());
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/core/MockClosure.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/MockClosure.java
new file mode 100644
index 0000000..2b67748
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/MockClosure.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Status;
+
+class MockClosure implements Closure {
+ Status s;
+
+ @Override
+ public void run(Status status) {
+ this.s = status;
+
+ }
+}
\ No newline at end of file
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/core/MockStateMachine.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/MockStateMachine.java
new file mode 100644
index 0000000..0191396
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/MockStateMachine.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Iterator;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.LeaderChangeContext;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
+import com.alipay.sofa.jraft.util.Bits;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+public class MockStateMachine extends StateMachineAdapter {
+
+ private final Lock lock = new ReentrantLock();
+ private volatile int onStartFollowingTimes = 0;
+ private volatile int onStopFollowingTimes = 0;
+ private volatile long leaderTerm = -1;
+ private volatile long appliedIndex = -1;
+ private volatile long snapshotIndex = -1L;
+ private final List<ByteBuffer> logs = new ArrayList<>();
+ private final Endpoint address;
+ private volatile int saveSnapshotTimes;
+ private volatile int loadSnapshotTimes;
+
+ public Endpoint getAddress() {
+ return this.address;
+ }
+
+ public MockStateMachine(final Endpoint address) {
+ super();
+ this.address = address;
+ }
+
+ public int getSaveSnapshotTimes() {
+ return this.saveSnapshotTimes;
+ }
+
+ public int getLoadSnapshotTimes() {
+ return this.loadSnapshotTimes;
+ }
+
+ public int getOnStartFollowingTimes() {
+ return this.onStartFollowingTimes;
+ }
+
+ public int getOnStopFollowingTimes() {
+ return this.onStopFollowingTimes;
+ }
+
+ public long getLeaderTerm() {
+ return this.leaderTerm;
+ }
+
+ public long getAppliedIndex() {
+ return this.appliedIndex;
+ }
+
+ public long getSnapshotIndex() {
+ return this.snapshotIndex;
+ }
+
+ public void lock() {
+ this.lock.lock();
+ }
+
+ public void unlock() {
+ this.lock.unlock();
+ }
+
+ public List<ByteBuffer> getLogs() {
+ this.lock.lock();
+ try {
+ return this.logs;
+ } finally {
+ this.lock.unlock();
+ }
+ }
+
+ private final AtomicLong lastAppliedIndex = new AtomicLong(-1);
+
+ @Override
+ public void onApply(final Iterator iter) {
+ while (iter.hasNext()) {
+ this.lock.lock();
+ try {
+ if (iter.getIndex() <= this.lastAppliedIndex.get()) {
+ //prevent duplication
+ continue;
+ }
+ this.lastAppliedIndex.set(iter.getIndex());
+ this.logs.add(iter.getData().slice());
+ if (iter.done() != null) {
+ iter.done().run(Status.OK());
+ }
+ } finally {
+ this.lock.unlock();
+ }
+ this.appliedIndex = iter.getIndex();
+ iter.next();
+ }
+ }
+
+ public boolean isLeader() {
+ return this.leaderTerm > 0;
+ }
+
+ @Override
+ public void onSnapshotSave(final SnapshotWriter writer, final Closure done) {
+ this.saveSnapshotTimes++;
+ final String path = writer.getPath() + File.separator + "data";
+ final File file = new File(path);
+ try (FileOutputStream fout = new FileOutputStream(file);
+ BufferedOutputStream out = new BufferedOutputStream(fout)) {
+ this.lock.lock();
+ try {
+ for (final ByteBuffer buf : this.logs) {
+ final byte[] bs = new byte[4];
+ Bits.putInt(bs, 0, buf.remaining());
+ out.write(bs);
+ out.write(buf.array());
+ }
+ this.snapshotIndex = this.appliedIndex;
+ } finally {
+ this.lock.unlock();
+ }
+ System.out.println("Node<" + this.address + "> saved snapshot into " + file);
+ writer.addFile("data");
+ done.run(Status.OK());
+ } catch (final IOException e) {
+ e.printStackTrace();
+ done.run(new Status(RaftError.EIO, "Fail to save snapshot"));
+ }
+ }
+
+ @Override
+ public boolean onSnapshotLoad(final SnapshotReader reader) {
+ this.lastAppliedIndex.set(0);
+ this.loadSnapshotTimes++;
+ final String path = reader.getPath() + File.separator + "data";
+ final File file = new File(path);
+ if (!file.exists()) {
+ return false;
+ }
+ try (FileInputStream fin = new FileInputStream(file); BufferedInputStream in = new BufferedInputStream(fin)) {
+ this.lock.lock();
+ this.logs.clear();
+ try {
+ while (true) {
+ final byte[] bs = new byte[4];
+ if (in.read(bs) == 4) {
+ final int len = Bits.getInt(bs, 0);
+ final byte[] buf = new byte[len];
+ if (in.read(buf) != len) {
+ break;
+ }
+ this.logs.add(ByteBuffer.wrap(buf));
+ } else {
+ break;
+ }
+ }
+ } finally {
+ this.lock.unlock();
+ }
+ System.out.println("Node<" + this.address + "> loaded snapshot from " + path);
+ return true;
+ } catch (final IOException e) {
+ e.printStackTrace();
+ return false;
+ }
+ }
+
+ @Override
+ public void onLeaderStart(final long term) {
+ super.onLeaderStart(term);
+ this.leaderTerm = term;
+ }
+
+ @Override
+ public void onLeaderStop(final Status status) {
+ super.onLeaderStop(status);
+ this.leaderTerm = -1;
+ }
+
+ @Override
+ public void onStopFollowing(final LeaderChangeContext ctx) {
+ super.onStopFollowing(ctx);
+ this.onStopFollowingTimes++;
+ }
+
+ @Override
+ public void onStartFollowing(final LeaderChangeContext ctx) {
+ super.onStartFollowing(ctx);
+ this.onStartFollowingTimes++;
+ }
+
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/core/NodeTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/NodeTest.java
new file mode 100644
index 0000000..ffbc283
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/NodeTest.java
@@ -0,0 +1,3418 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.Vector;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+//import org.rocksdb.util.SizeUnit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Iterator;
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.NodeManager;
+import com.alipay.sofa.jraft.RaftGroupService;
+import com.alipay.sofa.jraft.StateMachine;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.closure.JoinableClosure;
+import com.alipay.sofa.jraft.closure.ReadIndexClosure;
+import com.alipay.sofa.jraft.closure.SynchronizedClosure;
+import com.alipay.sofa.jraft.closure.TaskClosure;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.EnumOutter;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.entity.Task;
+import com.alipay.sofa.jraft.entity.UserLog;
+import com.alipay.sofa.jraft.error.LogIndexOutOfBoundsException;
+import com.alipay.sofa.jraft.error.LogNotFoundException;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.error.RaftException;
+import com.alipay.sofa.jraft.option.BootstrapOptions;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import com.alipay.sofa.jraft.storage.SnapshotThrottle;
+//import com.alipay.sofa.jraft.storage.impl.RocksDBLogStorage;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.ThroughputSnapshotThrottle;
+import com.alipay.sofa.jraft.test.TestUtils;
+import com.alipay.sofa.jraft.util.Bits;
+import com.alipay.sofa.jraft.util.Endpoint;
+//import com.alipay.sofa.jraft.util.StorageOptionsFactory;
+import com.alipay.sofa.jraft.util.Utils;
+import com.codahale.metrics.ConsoleReporter;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class NodeTest {
+
+ static final Logger LOG = LoggerFactory.getLogger(NodeTest.class);
+
+ private String dataPath;
+
+ private final AtomicInteger startedCounter = new AtomicInteger(0);
+ private final AtomicInteger stoppedCounter = new AtomicInteger(0);
+
+ @Rule
+ public TestName testName = new TestName();
+
+ private long testStartMs;
+
+ private static DumpThread dumpThread;
+
+ static class DumpThread extends Thread {
+ private static long DUMP_TIMEOUT_MS = 5 * 60 * 1000;
+ private volatile boolean stopped = false;
+
+ @Override
+ public void run() {
+ while (!this.stopped) {
+ try {
+ Thread.sleep(DUMP_TIMEOUT_MS);
+ System.out.println("Test hang too long, dump threads");
+ TestUtils.dumpThreads();
+ } catch (InterruptedException e) {
+ // reset request, continue
+ continue;
+ }
+ }
+ }
+ }
+
+ @BeforeClass
+ public static void setupNodeTest() {
+ dumpThread = new DumpThread();
+ dumpThread.setName("NodeTest-DumpThread");
+ dumpThread.setDaemon(true);
+ dumpThread.start();
+ }
+
+ @AfterClass
+ public static void tearNodeTest() throws Exception {
+ dumpThread.stopped = true;
+ dumpThread.interrupt();
+ dumpThread.join(100);
+ }
+
+ @Before
+ public void setup() throws Exception {
+ System.out.println(">>>>>>>>>>>>>>> Start test method: " + this.testName.getMethodName());
+ this.dataPath = TestUtils.mkTempDir();
+ FileUtils.forceMkdir(new File(this.dataPath));
+ assertEquals(NodeImpl.GLOBAL_NUM_NODES.get(), 0);
+ this.testStartMs = Utils.monotonicMs();
+ dumpThread.interrupt(); // reset dump timeout
+ }
+
+ @After
+ public void teardown() throws Exception {
+ if (!TestCluster.CLUSTERS.isEmpty()) {
+ for (final TestCluster c : TestCluster.CLUSTERS.removeAll()) {
+ c.stopAll();
+ }
+ }
+ if (NodeImpl.GLOBAL_NUM_NODES.get() > 0) {
+ Thread.sleep(5000);
+ assertEquals(0, NodeImpl.GLOBAL_NUM_NODES.get());
+ }
+ FileUtils.deleteDirectory(new File(this.dataPath));
+ NodeManager.getInstance().clear();
+ this.startedCounter.set(0);
+ this.stoppedCounter.set(0);
+ System.out.println(">>>>>>>>>>>>>>> End test method: " + this.testName.getMethodName() + ", cost:"
+ + (Utils.monotonicMs() - this.testStartMs) + " ms.");
+ }
+
+ @Test
+ public void testInitShutdown() throws Exception {
+ final Endpoint addr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+ NodeManager.getInstance().addAddress(addr);
+ final NodeOptions nodeOptions = new NodeOptions();
+ nodeOptions.setFsm(new MockStateMachine(addr));
+ nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+ nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+ nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+
+ final Node node = new NodeImpl("unittest", new PeerId(addr, 0));
+ assertTrue(node.init(nodeOptions));
+
+ node.shutdown();
+ node.join();
+ }
+
+ @Test
+ public void testNodeTaskOverload() throws Exception {
+ final Endpoint addr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+ final PeerId peer = new PeerId(addr, 0);
+
+ NodeManager.getInstance().addAddress(addr);
+ final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+ final RaftOptions raftOptions = new RaftOptions();
+ raftOptions.setDisruptorBufferSize(2);
+ nodeOptions.setRaftOptions(raftOptions);
+ final MockStateMachine fsm = new MockStateMachine(addr);
+ nodeOptions.setFsm(fsm);
+ nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+ nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+ nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+ nodeOptions.setInitialConf(new Configuration(Collections.singletonList(peer)));
+ final Node node = new NodeImpl("unittest", peer);
+ assertTrue(node.init(nodeOptions));
+
+ assertEquals(1, node.listPeers().size());
+ assertTrue(node.listPeers().contains(peer));
+
+ while (!node.isLeader()) {
+ ;
+ }
+
+ final List<Task> tasks = new ArrayList<>();
+ final AtomicInteger c = new AtomicInteger(0);
+ for (int i = 0; i < 10; i++) {
+ final ByteBuffer data = ByteBuffer.wrap(("hello" + i).getBytes());
+ final Task task = new Task(data, new JoinableClosure(status -> {
+ System.out.println(status);
+ if (!status.isOk()) {
+ assertTrue(
+ status.getRaftError() == RaftError.EBUSY || status.getRaftError() == RaftError.EPERM);
+ }
+ c.incrementAndGet();
+ }));
+ node.apply(task);
+ tasks.add(task);
+ }
+ try {
+ Task.joinAll(tasks, TimeUnit.SECONDS.toMillis(30));
+ assertEquals(10, c.get());
+ } finally {
+ node.shutdown();
+ node.join();
+ }
+ }
+
+ /**
+ * Test rollback stateMachine with readIndex for issue 317:
+ * https://github.com/sofastack/sofa-jraft/issues/317
+ */
+ @Test
+ public void testRollbackStateMachineWithReadIndex_Issue317() throws Exception {
+ final Endpoint addr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+ final PeerId peer = new PeerId(addr, 0);
+
+ NodeManager.getInstance().addAddress(addr);
+ final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+ final CountDownLatch applyCompleteLatch = new CountDownLatch(1);
+ final CountDownLatch applyLatch = new CountDownLatch(1);
+ final CountDownLatch readIndexLatch = new CountDownLatch(1);
+ final AtomicInteger currentValue = new AtomicInteger(-1);
+ final String errorMsg = this.testName.getMethodName();
+ final StateMachine fsm = new StateMachineAdapter() {
+
+ @Override
+ public void onApply(final Iterator iter) {
+ // Notify that the #onApply is preparing to go.
+ readIndexLatch.countDown();
+ // Wait for submitting a read-index request
+ try {
+ applyLatch.await();
+ } catch (InterruptedException e) {
+ fail();
+ }
+ int i = 0;
+ while (iter.hasNext()) {
+ byte[] data = iter.next().array();
+ int v = Bits.getInt(data, 0);
+ assertEquals(i++, v);
+ currentValue.set(v);
+ }
+ if (i > 0) {
+ // rollback
+ currentValue.set(i - 1);
+ iter.setErrorAndRollback(1, new Status(-1, errorMsg));
+ applyCompleteLatch.countDown();
+ }
+ }
+ };
+ nodeOptions.setFsm(fsm);
+ nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+ nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+ nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+ nodeOptions.setInitialConf(new Configuration(Collections.singletonList(peer)));
+ final Node node = new NodeImpl("unittest", peer);
+ assertTrue(node.init(nodeOptions));
+
+ assertEquals(1, node.listPeers().size());
+ assertTrue(node.listPeers().contains(peer));
+
+ while (!node.isLeader()) {
+ ;
+ }
+
+ int n = 5;
+ {
+ // apply tasks
+ for (int i = 0; i < n; i++) {
+ byte[] b = new byte[4];
+ Bits.putInt(b, 0, i);
+ node.apply(new Task(ByteBuffer.wrap(b), null));
+ }
+ }
+
+ final AtomicInteger readIndexSuccesses = new AtomicInteger(0);
+ {
+ // Submit a read-index, wait for #onApply
+ readIndexLatch.await();
+ final CountDownLatch latch = new CountDownLatch(1);
+ node.readIndex(null, new ReadIndexClosure() {
+
+ @Override
+ public void run(final Status status, final long index, final byte[] reqCtx) {
+ try {
+ if (status.isOk()) {
+ readIndexSuccesses.incrementAndGet();
+ } else {
+ assertTrue("Unexpected status: " + status,
+ status.getErrorMsg().contains(errorMsg) || status.getRaftError() == RaftError.ETIMEDOUT
+ || status.getErrorMsg().contains("Invalid state for readIndex: STATE_ERROR"));
+ }
+ } finally {
+ latch.countDown();
+ }
+ }
+ });
+ // We have already submit a read-index request,
+ // notify #onApply can go right now
+ applyLatch.countDown();
+
+ // The state machine is in error state, the node should step down.
+ while (node.isLeader()) {
+ Thread.sleep(10);
+ }
+ latch.await();
+ applyCompleteLatch.await();
+ }
+ // No read-index request succeed.
+ assertEquals(0, readIndexSuccesses.get());
+ assertTrue(n - 1 >= currentValue.get());
+
+ node.shutdown();
+ node.join();
+ }
+
+ @Test
+ public void testSingleNode() throws Exception {
+ final Endpoint addr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+ final PeerId peer = new PeerId(addr, 0);
+
+ NodeManager.getInstance().addAddress(addr);
+ final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+ final MockStateMachine fsm = new MockStateMachine(addr);
+ nodeOptions.setFsm(fsm);
+ nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+ nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+ nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+ nodeOptions.setInitialConf(new Configuration(Collections.singletonList(peer)));
+ final Node node = new NodeImpl("unittest", peer);
+ assertTrue(node.init(nodeOptions));
+
+ assertEquals(1, node.listPeers().size());
+ assertTrue(node.listPeers().contains(peer));
+
+ while (!node.isLeader()) {
+ ;
+ }
+
+ sendTestTaskAndWait(node);
+ assertEquals(10, fsm.getLogs().size());
+ int i = 0;
+ for (final ByteBuffer data : fsm.getLogs()) {
+ assertEquals("hello" + i++, new String(data.array()));
+ }
+ node.shutdown();
+ node.join();
+ }
+
+ @Test
+ public void testNoLeader() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+
+ assertTrue(cluster.start(peers.get(0).getEndpoint()));
+
+ final List<Node> followers = cluster.getFollowers();
+ assertEquals(1, followers.size());
+
+ final Node follower = followers.get(0);
+ sendTestTaskAndWait(follower, 0, RaftError.EPERM);
+
+ // adds a peer3
+ final PeerId peer3 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 3);
+ CountDownLatch latch = new CountDownLatch(1);
+ follower.addPeer(peer3, new ExpectClosure(RaftError.EPERM, latch));
+ waitLatch(latch);
+
+ // remove the peer0
+ final PeerId peer0 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+ latch = new CountDownLatch(1);
+ follower.removePeer(peer0, new ExpectClosure(RaftError.EPERM, latch));
+ waitLatch(latch);
+
+ cluster.stopAll();
+ }
+
+ private void sendTestTaskAndWait(final Node node) throws InterruptedException {
+ this.sendTestTaskAndWait(node, 0, RaftError.SUCCESS);
+ }
+
+ private void sendTestTaskAndWait(final Node node, final RaftError err) throws InterruptedException {
+ this.sendTestTaskAndWait(node, 0, err);
+ }
+
+ private void sendTestTaskAndWait(final Node node, final int start, final RaftError err) throws InterruptedException {
+ final CountDownLatch latch = new CountDownLatch(10);
+ for (int i = start; i < start + 10; i++) {
+ final ByteBuffer data = ByteBuffer.wrap(("hello" + i).getBytes());
+ final Task task = new Task(data, new ExpectClosure(err, latch));
+ node.apply(task);
+ }
+ waitLatch(latch);
+ }
+
+ @SuppressWarnings("SameParameterValue")
+ private void sendTestTaskAndWait(final String prefix, final Node node, final int code) throws InterruptedException {
+ final CountDownLatch latch = new CountDownLatch(10);
+ for (int i = 0; i < 10; i++) {
+ final ByteBuffer data = ByteBuffer.wrap((prefix + i).getBytes());
+ final Task task = new Task(data, new ExpectClosure(code, null, latch));
+ node.apply(task);
+ }
+ waitLatch(latch);
+ }
+
+ @Test
+ public void testTripleNodesWithReplicatorStateListener() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ final UserReplicatorStateListener listener1 = new UserReplicatorStateListener();
+ final UserReplicatorStateListener listener2 = new UserReplicatorStateListener();
+
+ for (Node node : cluster.getNodes()) {
+ node.addReplicatorStateListener(listener1);
+ node.addReplicatorStateListener(listener2);
+
+ }
+ // elect leader
+ cluster.waitLeader();
+ assertEquals(4, this.startedCounter.get());
+ assertEquals(2, cluster.getLeader().getReplicatorStatueListeners().size());
+ assertEquals(2, cluster.getFollowers().get(0).getReplicatorStatueListeners().size());
+ assertEquals(2, cluster.getFollowers().get(1).getReplicatorStatueListeners().size());
+
+ for (Node node : cluster.getNodes()) {
+ node.removeReplicatorStateListener(listener1);
+ }
+ assertEquals(1, cluster.getLeader().getReplicatorStatueListeners().size());
+ assertEquals(1, cluster.getFollowers().get(0).getReplicatorStatueListeners().size());
+ assertEquals(1, cluster.getFollowers().get(1).getReplicatorStatueListeners().size());
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testVoteTimedoutStepDown() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(3, leader.listPeers().size());
+ // apply tasks to leader
+ this.sendTestTaskAndWait(leader);
+
+ // Stop all followers
+ List<Node> followers = cluster.getFollowers();
+ assertFalse(followers.isEmpty());
+ for (Node node : followers) {
+ assertTrue(cluster.stop(node.getNodeId().getPeerId().getEndpoint()));
+ }
+
+ // Wait leader to step down.
+ while (leader.isLeader()) {
+ Thread.sleep(10);
+ }
+
+ // old leader try to elect self, it should fail.
+ ((NodeImpl) leader).tryElectSelf();
+ Thread.sleep(1500);
+ // Start followers
+ for (Node node : followers) {
+ assertTrue(cluster.start(node.getNodeId().getPeerId().getEndpoint()));
+ }
+
+ cluster.ensureSame(-1);
+ cluster.stopAll();
+ }
+
+ class UserReplicatorStateListener implements Replicator.ReplicatorStateListener {
+ @Override
+ public void onCreated(final PeerId peer) {
+ LOG.info("Replicator has created");
+ NodeTest.this.startedCounter.incrementAndGet();
+ }
+
+ @Override
+ public void onError(final PeerId peer, final Status status) {
+ LOG.info("Replicator has errors");
+ }
+
+ @Override
+ public void onDestroyed(final PeerId peer) {
+ LOG.info("Replicator has been destroyed");
+ NodeTest.this.stoppedCounter.incrementAndGet();
+ }
+ }
+
+ @Test
+ public void testLeaderTransferWithReplicatorStateListener() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 300);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+ cluster.waitLeader();
+ final UserReplicatorStateListener listener = new UserReplicatorStateListener();
+ for (Node node : cluster.getNodes()) {
+ node.addReplicatorStateListener(listener);
+ }
+ Node leader = cluster.getLeader();
+ this.sendTestTaskAndWait(leader);
+ Thread.sleep(100);
+ final List<Node> followers = cluster.getFollowers();
+
+ final PeerId targetPeer = followers.get(0).getNodeId().getPeerId().copy();
+ LOG.info("Transfer leadership from {} to {}", leader, targetPeer);
+ assertTrue(leader.transferLeadershipTo(targetPeer).isOk());
+ Thread.sleep(1000);
+ cluster.waitLeader();
+ assertEquals(2, this.startedCounter.get());
+
+ for (Node node : cluster.getNodes()) {
+ node.clearReplicatorStateListeners();
+ }
+ assertEquals(0, cluster.getLeader().getReplicatorStatueListeners().size());
+ assertEquals(0, cluster.getFollowers().get(0).getReplicatorStatueListeners().size());
+ assertEquals(0, cluster.getFollowers().get(1).getReplicatorStatueListeners().size());
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testTripleNodes() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(3, leader.listPeers().size());
+ // apply tasks to leader
+ this.sendTestTaskAndWait(leader);
+
+ {
+ final ByteBuffer data = ByteBuffer.wrap("no closure".getBytes());
+ final Task task = new Task(data, null);
+ leader.apply(task);
+ }
+
+ {
+ // task with TaskClosure
+ final ByteBuffer data = ByteBuffer.wrap("task closure".getBytes());
+ final Vector<String> cbs = new Vector<>();
+ final CountDownLatch latch = new CountDownLatch(1);
+ final Task task = new Task(data, new TaskClosure() {
+
+ @Override
+ public void run(final Status status) {
+ cbs.add("apply");
+ latch.countDown();
+ }
+
+ @Override
+ public void onCommitted() {
+ cbs.add("commit");
+
+ }
+ });
+ leader.apply(task);
+ latch.await();
+ assertEquals(2, cbs.size());
+ assertEquals("commit", cbs.get(0));
+ assertEquals("apply", cbs.get(1));
+ }
+
+ cluster.ensureSame(-1);
+ assertEquals(2, cluster.getFollowers().size());
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testSingleNodeWithLearner() throws Exception {
+ final Endpoint addr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+ final PeerId peer = new PeerId(addr, 0);
+
+ final Endpoint learnerAddr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT + 1);
+ final PeerId learnerPeer = new PeerId(learnerAddr, 0);
+
+ NodeManager.getInstance().addAddress(addr);
+ NodeManager.getInstance().addAddress(learnerAddr);
+ MockStateMachine learnerFsm = null;
+ Node learner = null;
+ RaftGroupService learnerServer = null;
+ {
+ // Start learner
+ final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+ learnerFsm = new MockStateMachine(learnerAddr);
+ nodeOptions.setFsm(learnerFsm);
+ nodeOptions.setLogUri(this.dataPath + File.separator + "log1");
+ nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta1");
+ nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot1");
+ nodeOptions.setInitialConf(new Configuration(Collections.singletonList(peer), Collections
+ .singletonList(learnerPeer)));
+
+ final RpcServer rpcServer = RaftRpcServerFactory.createRaftRpcServer(learnerAddr);
+ learnerServer = new RaftGroupService("unittest", new PeerId(learnerAddr, 0), nodeOptions, rpcServer);
+ learner = learnerServer.start();
+ }
+
+ {
+ // Start leader
+ final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+ final MockStateMachine fsm = new MockStateMachine(addr);
+ nodeOptions.setFsm(fsm);
+ nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+ nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+ nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+ nodeOptions.setInitialConf(new Configuration(Collections.singletonList(peer), Collections
+ .singletonList(learnerPeer)));
+ final Node node = new NodeImpl("unittest", peer);
+ assertTrue(node.init(nodeOptions));
+
+ assertEquals(1, node.listPeers().size());
+ assertTrue(node.listPeers().contains(peer));
+ while (!node.isLeader()) {
+ ;
+ }
+ sendTestTaskAndWait(node);
+ assertEquals(10, fsm.getLogs().size());
+ int i = 0;
+ for (final ByteBuffer data : fsm.getLogs()) {
+ assertEquals("hello" + i++, new String(data.array()));
+ }
+ Thread.sleep(1000); //wait for entries to be replicated to learner.
+ node.shutdown();
+ node.join();
+ }
+ {
+ // assert learner fsm
+ assertEquals(10, learnerFsm.getLogs().size());
+ int i = 0;
+ for (final ByteBuffer data : learnerFsm.getLogs()) {
+ assertEquals("hello" + i++, new String(data.array()));
+ }
+ learnerServer.shutdown();
+ learnerServer.join();
+ }
+ }
+
+ @Test
+ public void testResetLearners() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final LinkedHashSet<PeerId> learners = new LinkedHashSet<>();
+
+ for (int i = 0; i < 3; i++) {
+ learners.add(new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 3 + i));
+ }
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers, learners, 300);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+ for (final PeerId peer : learners) {
+ assertTrue(cluster.startLearner(peer));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ Node leader = cluster.getLeader();
+
+ assertEquals(3, leader.listAlivePeers().size());
+ assertEquals(3, leader.listAliveLearners().size());
+
+ this.sendTestTaskAndWait(leader);
+ Thread.sleep(500);
+ List<MockStateMachine> fsms = cluster.getFsms();
+ assertEquals(6, fsms.size());
+ cluster.ensureSame();
+
+ {
+ // Reset learners to 2 nodes
+ PeerId learnerPeer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 3);
+ learners.remove(learnerPeer);
+ assertEquals(2, learners.size());
+
+ SynchronizedClosure done = new SynchronizedClosure();
+ leader.resetLearners(new ArrayList<>(learners), done);
+ assertTrue(done.await().isOk());
+ assertEquals(2, leader.listAliveLearners().size());
+ assertEquals(2, leader.listLearners().size());
+ this.sendTestTaskAndWait(leader);
+ Thread.sleep(500);
+
+ assertEquals(6, fsms.size());
+
+ MockStateMachine fsm = fsms.remove(3); // get the removed learner's fsm
+ assertEquals(fsm.getAddress(), learnerPeer.getEndpoint());
+ // Ensure no more logs replicated to the removed learner.
+ assertTrue(cluster.getLeaderFsm().getLogs().size() > fsm.getLogs().size());
+ assertEquals(cluster.getLeaderFsm().getLogs().size(), 2 * fsm.getLogs().size());
+ }
+ {
+ // remove another learner
+ PeerId learnerPeer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 4);
+ SynchronizedClosure done = new SynchronizedClosure();
+ leader.removeLearners(Arrays.asList(learnerPeer), done);
+ assertTrue(done.await().isOk());
+
+ this.sendTestTaskAndWait(leader);
+ Thread.sleep(500);
+ MockStateMachine fsm = fsms.remove(3); // get the removed learner's fsm
+ assertEquals(fsm.getAddress(), learnerPeer.getEndpoint());
+ // Ensure no more logs replicated to the removed learner.
+ assertTrue(cluster.getLeaderFsm().getLogs().size() > fsm.getLogs().size());
+ assertEquals(cluster.getLeaderFsm().getLogs().size(), fsm.getLogs().size() / 2 * 3);
+ }
+
+ assertEquals(3, leader.listAlivePeers().size());
+ assertEquals(1, leader.listAliveLearners().size());
+ assertEquals(1, leader.listLearners().size());
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testTripleNodesWithStaticLearners() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ LinkedHashSet<PeerId> learners = new LinkedHashSet<>();
+ PeerId learnerPeer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 3);
+ learners.add(learnerPeer);
+ cluster.setLearners(learners);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+ final Node leader = cluster.getLeader();
+
+ assertEquals(3, leader.listPeers().size());
+ assertEquals(leader.listLearners().size(), 1);
+ assertTrue(leader.listLearners().contains(learnerPeer));
+ assertTrue(leader.listAliveLearners().isEmpty());
+
+ // start learner after cluster setup.
+ assertTrue(cluster.start(learnerPeer.getEndpoint()));
+
+ Thread.sleep(1000);
+
+ assertEquals(3, leader.listPeers().size());
+ assertEquals(leader.listLearners().size(), 1);
+ assertEquals(leader.listAliveLearners().size(), 1);
+
+ // apply tasks to leader
+ this.sendTestTaskAndWait(leader);
+
+ cluster.ensureSame();
+ assertEquals(4, cluster.getFsms().size());
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testTripleNodesWithLearners() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(3, leader.listPeers().size());
+ assertTrue(leader.listLearners().isEmpty());
+ assertTrue(leader.listAliveLearners().isEmpty());
+
+ {
+ // Adds a learner
+ SynchronizedClosure done = new SynchronizedClosure();
+ PeerId learnerPeer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 3);
+ // Start learner
+ assertTrue(cluster.startLearner(learnerPeer));
+ leader.addLearners(Arrays.asList(learnerPeer), done);
+ assertTrue(done.await().isOk());
+ assertEquals(1, leader.listAliveLearners().size());
+ assertEquals(1, leader.listLearners().size());
+ }
+
+ // apply tasks to leader
+ this.sendTestTaskAndWait(leader);
+
+ {
+ final ByteBuffer data = ByteBuffer.wrap("no closure".getBytes());
+ final Task task = new Task(data, null);
+ leader.apply(task);
+ }
+
+ {
+ // task with TaskClosure
+ final ByteBuffer data = ByteBuffer.wrap("task closure".getBytes());
+ final Vector<String> cbs = new Vector<>();
+ final CountDownLatch latch = new CountDownLatch(1);
+ final Task task = new Task(data, new TaskClosure() {
+
+ @Override
+ public void run(final Status status) {
+ cbs.add("apply");
+ latch.countDown();
+ }
+
+ @Override
+ public void onCommitted() {
+ cbs.add("commit");
+
+ }
+ });
+ leader.apply(task);
+ latch.await();
+ assertEquals(2, cbs.size());
+ assertEquals("commit", cbs.get(0));
+ assertEquals("apply", cbs.get(1));
+ }
+
+ assertEquals(4, cluster.getFsms().size());
+ assertEquals(2, cluster.getFollowers().size());
+ assertEquals(1, cluster.getLearners().size());
+ cluster.ensureSame(-1);
+
+ {
+ // Adds another learner
+ SynchronizedClosure done = new SynchronizedClosure();
+ PeerId learnerPeer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 4);
+ // Start learner
+ assertTrue(cluster.startLearner(learnerPeer));
+ leader.addLearners(Arrays.asList(learnerPeer), done);
+ assertTrue(done.await().isOk());
+ assertEquals(2, leader.listAliveLearners().size());
+ assertEquals(2, leader.listLearners().size());
+ }
+ {
+ // stop two followers
+ for (Node follower : cluster.getFollowers()) {
+ assertTrue(cluster.stop(follower.getNodeId().getPeerId().getEndpoint()));
+ }
+ // send a new task
+ final ByteBuffer data = ByteBuffer.wrap("task closure".getBytes());
+ SynchronizedClosure done = new SynchronizedClosure();
+ leader.apply(new Task(data, done));
+ // should fail
+ assertFalse(done.await().isOk());
+ assertEquals(RaftError.EPERM, done.getStatus().getRaftError());
+ // One peer with two learners.
+ assertEquals(3, cluster.getFsms().size());
+ cluster.ensureSame(-1);
+ }
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testNodesWithPriorityElection() throws Exception {
+
+ List<Integer> priorities = new ArrayList<>();
+ priorities.add(100);
+ priorities.add(40);
+ priorities.add(40);
+
+ final List<PeerId> peers = TestUtils.generatePriorityPeers(3, priorities);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), peer.getPriority()));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(3, leader.listPeers().size());
+ assertEquals(100, leader.getNodeTargetPriority());
+ assertEquals(100, leader.getLeaderId().getPriority());
+ assertEquals(2, cluster.getFollowers().size());
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testNodesWithPartPriorityElection() throws Exception {
+
+ List<Integer> priorities = new ArrayList<>();
+ priorities.add(100);
+ priorities.add(40);
+ priorities.add(-1);
+
+ final List<PeerId> peers = TestUtils.generatePriorityPeers(3, priorities);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), peer.getPriority()));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(3, leader.listPeers().size());
+ assertEquals(2, cluster.getFollowers().size());
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testNodesWithSpecialPriorityElection() throws Exception {
+
+ List<Integer> priorities = new ArrayList<Integer>();
+ priorities.add(0);
+ priorities.add(0);
+ priorities.add(-1);
+
+ final List<PeerId> peers = TestUtils.generatePriorityPeers(3, priorities);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), peer.getPriority()));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(3, leader.listPeers().size());
+ assertEquals(2, cluster.getFollowers().size());
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testNodesWithZeroValPriorityElection() throws Exception {
+
+ List<Integer> priorities = new ArrayList<Integer>();
+ priorities.add(50);
+ priorities.add(0);
+ priorities.add(0);
+
+ final List<PeerId> peers = TestUtils.generatePriorityPeers(3, priorities);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), peer.getPriority()));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(3, leader.listPeers().size());
+ assertEquals(2, cluster.getFollowers().size());
+ assertEquals(50, leader.getNodeTargetPriority());
+ assertEquals(50, leader.getLeaderId().getPriority());
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testNoLeaderWithZeroValPriorityElection() throws Exception {
+ List<Integer> priorities = new ArrayList<>();
+ priorities.add(0);
+ priorities.add(0);
+ priorities.add(0);
+
+ final List<PeerId> peers = TestUtils.generatePriorityPeers(3, priorities);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), peer.getPriority()));
+ }
+
+ Thread.sleep(200);
+
+ final List<Node> followers = cluster.getFollowers();
+ assertEquals(3, followers.size());
+
+ for (Node follower : followers) {
+ assertEquals(0, follower.getNodeId().getPeerId().getPriority());
+ }
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testLeaderStopAndReElectWithPriority() throws Exception {
+ final List<Integer> priorities = new ArrayList<>();
+ priorities.add(100);
+ priorities.add(60);
+ priorities.add(10);
+
+ final List<PeerId> peers = TestUtils.generatePriorityPeers(3, priorities);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), peer.getPriority()));
+ }
+
+ cluster.waitLeader();
+ Node leader = cluster.getLeader();
+
+ assertNotNull(leader);
+ assertEquals(100, leader.getNodeId().getPeerId().getPriority());
+ assertEquals(100, leader.getNodeTargetPriority());
+
+ // apply tasks to leader
+ sendTestTaskAndWait(leader);
+
+ // stop leader
+ assertTrue(cluster.stop(leader.getNodeId().getPeerId().getEndpoint()));
+
+ // elect new leader
+ cluster.waitLeader();
+ leader = cluster.getLeader();
+
+ assertNotNull(leader);
+
+ // get current leader priority value
+ int leaderPriority = leader.getNodeId().getPeerId().getPriority();
+
+ // get current leader log size
+ int peer1LogSize = cluster.getFsmByPeer(peers.get(1)).getLogs().size();
+ int peer2LogSize = cluster.getFsmByPeer(peers.get(2)).getLogs().size();
+
+ // if the leader is lower priority value
+ if (leaderPriority == 10) {
+ // we just compare the two peers' log size value;
+ assertTrue(peer2LogSize > peer1LogSize);
+ } else {
+ assertEquals(60, leader.getNodeId().getPeerId().getPriority());
+ assertEquals(100, leader.getNodeTargetPriority());
+ }
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testRemoveLeaderWithPriority() throws Exception {
+ final List<Integer> priorities = new ArrayList<Integer>();
+ priorities.add(100);
+ priorities.add(60);
+ priorities.add(10);
+
+ final List<PeerId> peers = TestUtils.generatePriorityPeers(3, priorities);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), peer.getPriority()));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ // get leader
+ Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(100, leader.getNodeTargetPriority());
+ assertEquals(100, leader.getNodeId().getPeerId().getPriority());
+
+ final List<Node> followers = cluster.getFollowers();
+ assertEquals(2, followers.size());
+
+ final PeerId oldLeader = leader.getNodeId().getPeerId().copy();
+ final Endpoint oldLeaderAddr = oldLeader.getEndpoint();
+
+ // remove old leader
+ LOG.info("Remove old leader {}", oldLeader);
+ CountDownLatch latch = new CountDownLatch(1);
+ leader.removePeer(oldLeader, new ExpectClosure(latch));
+ waitLatch(latch);
+ assertEquals(60, leader.getNodeTargetPriority());
+
+ // stop and clean old leader
+ LOG.info("Stop and clean old leader {}", oldLeader);
+ assertTrue(cluster.stop(oldLeaderAddr));
+ cluster.clean(oldLeaderAddr);
+
+ // elect new leader
+ cluster.waitLeader();
+ leader = cluster.getLeader();
+ LOG.info("New leader is {}", leader);
+ assertNotNull(leader);
+ assertNotSame(leader, oldLeader);
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testTripleNodesV1V2Codec() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (int i = 0; i < peers.size(); i++) {
+ // Peer3 use codec v1
+ if (i == 2) {
+ cluster.setRaftServiceFactory(new V1JRaftServiceFactory());
+ }
+ assertTrue(cluster.start(peers.get(i).getEndpoint()));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ // get leader
+ Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(3, leader.listPeers().size());
+ // apply tasks to leader
+ this.sendTestTaskAndWait(leader);
+
+ {
+ final ByteBuffer data = ByteBuffer.wrap("no closure".getBytes());
+ final Task task = new Task(data, null);
+ leader.apply(task);
+ }
+
+ {
+ // task with TaskClosure
+ final ByteBuffer data = ByteBuffer.wrap("task closure".getBytes());
+ final Vector<String> cbs = new Vector<>();
+ final CountDownLatch latch = new CountDownLatch(1);
+ final Task task = new Task(data, new TaskClosure() {
+
+ @Override
+ public void run(final Status status) {
+ cbs.add("apply");
+ latch.countDown();
+ }
+
+ @Override
+ public void onCommitted() {
+ cbs.add("commit");
+
+ }
+ });
+ leader.apply(task);
+ latch.await();
+ assertEquals(2, cbs.size());
+ assertEquals("commit", cbs.get(0));
+ assertEquals("apply", cbs.get(1));
+ }
+
+ cluster.ensureSame(-1);
+ assertEquals(2, cluster.getFollowers().size());
+
+ // transfer the leader to v1 codec peer
+ assertTrue(leader.transferLeadershipTo(peers.get(2)).isOk());
+ cluster.waitLeader();
+ leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(leader.getLeaderId(), peers.get(2));
+ // apply tasks to leader
+ this.sendTestTaskAndWait(leader);
+ cluster.ensureSame();
+ cluster.stopAll();
+
+ // start the cluster with v2 codec, should work
+ final TestCluster newCluster = new TestCluster("unittest", this.dataPath, peers);
+ for (int i = 0; i < peers.size(); i++) {
+ assertTrue(newCluster.start(peers.get(i).getEndpoint()));
+ }
+
+ // elect leader
+ newCluster.waitLeader();
+ newCluster.ensureSame();
+ leader = newCluster.getLeader();
+ assertNotNull(leader);
+ // apply new tasks
+ this.sendTestTaskAndWait(leader);
+ newCluster.ensureSame();
+ newCluster.stopAll();
+ }
+
+ @Test
+ public void testChecksum() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ // start with checksum validation
+ {
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ final RaftOptions raftOptions = new RaftOptions();
+ raftOptions.setEnableLogEntryChecksum(true);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), false, 300, true, null, raftOptions));
+ }
+
+ cluster.waitLeader();
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(3, leader.listPeers().size());
+ this.sendTestTaskAndWait(leader);
+ cluster.ensureSame();
+
+ cluster.stopAll();
+ }
+
+ // restart with peer3 enable checksum validation
+ {
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ RaftOptions raftOptions = new RaftOptions();
+ raftOptions.setEnableLogEntryChecksum(false);
+ for (final PeerId peer : peers) {
+ if (peer.equals(peers.get(2))) {
+ raftOptions = new RaftOptions();
+ raftOptions.setEnableLogEntryChecksum(true);
+ }
+ assertTrue(cluster.start(peer.getEndpoint(), false, 300, true, null, raftOptions));
+ }
+
+ cluster.waitLeader();
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(3, leader.listPeers().size());
+ this.sendTestTaskAndWait(leader);
+ cluster.ensureSame();
+
+ cluster.stopAll();
+ }
+
+ // restart with no checksum validation
+ {
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ final RaftOptions raftOptions = new RaftOptions();
+ raftOptions.setEnableLogEntryChecksum(false);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), false, 300, true, null, raftOptions));
+ }
+
+ cluster.waitLeader();
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(3, leader.listPeers().size());
+ this.sendTestTaskAndWait(leader);
+ cluster.ensureSame();
+
+ cluster.stopAll();
+ }
+
+ // restart with all peers enable checksum validation
+ {
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ final RaftOptions raftOptions = new RaftOptions();
+ raftOptions.setEnableLogEntryChecksum(true);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), false, 300, true, null, raftOptions));
+ }
+
+ cluster.waitLeader();
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(3, leader.listPeers().size());
+ this.sendTestTaskAndWait(leader);
+ cluster.ensureSame();
+
+ cluster.stopAll();
+ }
+
+ }
+
+ @Test
+ public void testReadIndex() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), false, 300, true));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(3, leader.listPeers().size());
+ // apply tasks to leader
+ this.sendTestTaskAndWait(leader);
+
+ // first call will fail-fast when no connection
+ if (!assertReadIndex(leader, 11)) {
+ assertTrue(assertReadIndex(leader, 11));
+ }
+
+ // read from follower
+ for (final Node follower : cluster.getFollowers()) {
+ assertNotNull(follower);
+ assertReadIndex(follower, 11);
+ }
+
+ // read with null request context
+ final CountDownLatch latch = new CountDownLatch(1);
+ leader.readIndex(null, new ReadIndexClosure() {
+
+ @Override
+ public void run(final Status status, final long index, final byte[] reqCtx) {
+ assertNull(reqCtx);
+ assertTrue(status.isOk());
+ latch.countDown();
+ }
+ });
+ latch.await();
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testReadIndexTimeout() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), false, 300, true));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(3, leader.listPeers().size());
+ // apply tasks to leader
+ sendTestTaskAndWait(leader);
+
+ // first call will fail-fast when no connection
+ if (!assertReadIndex(leader, 11)) {
+ assertTrue(assertReadIndex(leader, 11));
+ }
+
+ // read from follower
+ for (final Node follower : cluster.getFollowers()) {
+ assertNotNull(follower);
+ assertReadIndex(follower, 11);
+ }
+
+ // read with null request context
+ final CountDownLatch latch = new CountDownLatch(1);
+ final long start = System.currentTimeMillis();
+ leader.readIndex(null, new ReadIndexClosure(0) {
+
+ @Override
+ public void run(final Status status, final long index, final byte[] reqCtx) {
+ assertNull(reqCtx);
+ if (status.isOk()) {
+ System.err.println("Read-index so fast: " + (System.currentTimeMillis() - start) + "ms");
+ } else {
+ assertEquals(status, new Status(RaftError.ETIMEDOUT, "read-index request timeout"));
+ assertEquals(index, -1);
+ }
+ latch.countDown();
+ }
+ });
+ latch.await();
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testReadIndexFromLearner() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), false, 300, true));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(3, leader.listPeers().size());
+ // apply tasks to leader
+ this.sendTestTaskAndWait(leader);
+
+ {
+ // Adds a learner
+ SynchronizedClosure done = new SynchronizedClosure();
+ PeerId learnerPeer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 3);
+ // Start learner
+ assertTrue(cluster.startLearner(learnerPeer));
+ leader.addLearners(Arrays.asList(learnerPeer), done);
+ assertTrue(done.await().isOk());
+ assertEquals(1, leader.listAliveLearners().size());
+ assertEquals(1, leader.listLearners().size());
+ }
+
+ Thread.sleep(100);
+ // read from learner
+ Node learner = cluster.getNodes().get(3);
+ assertNotNull(leader);
+ assertReadIndex(learner, 12);
+ assertReadIndex(learner, 12);
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testReadIndexChaos() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), false, 300, true));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(3, leader.listPeers().size());
+
+ final CountDownLatch latch = new CountDownLatch(10);
+ for (int i = 0; i < 10; i++) {
+ new Thread() {
+ @Override
+ public void run() {
+ try {
+ for (int i = 0; i < 100; i++) {
+ try {
+ sendTestTaskAndWait(leader);
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ readIndexRandom(cluster);
+ }
+ } finally {
+ latch.countDown();
+ }
+ }
+
+ private void readIndexRandom(final TestCluster cluster) {
+ final CountDownLatch readLatch = new CountDownLatch(1);
+ final byte[] requestContext = TestUtils.getRandomBytes();
+ cluster.getNodes().get(ThreadLocalRandom.current().nextInt(3))
+ .readIndex(requestContext, new ReadIndexClosure() {
+
+ @Override
+ public void run(final Status status, final long index, final byte[] reqCtx) {
+ if (status.isOk()) {
+ assertTrue(status.toString(), status.isOk());
+ assertTrue(index > 0);
+ assertArrayEquals(requestContext, reqCtx);
+ }
+ readLatch.countDown();
+ }
+ });
+ try {
+ readLatch.await();
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }.start();
+ }
+
+ latch.await();
+
+ cluster.ensureSame();
+
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertEquals(10000, fsm.getLogs().size());
+ }
+
+ cluster.stopAll();
+ }
+
+ @SuppressWarnings({ "unused", "SameParameterValue" })
+ private boolean assertReadIndex(final Node node, final int index) throws InterruptedException {
+ final CountDownLatch latch = new CountDownLatch(1);
+ final byte[] requestContext = TestUtils.getRandomBytes();
+ final AtomicBoolean success = new AtomicBoolean(false);
+ node.readIndex(requestContext, new ReadIndexClosure() {
+
+ @Override
+ public void run(final Status status, final long theIndex, final byte[] reqCtx) {
+ if (status.isOk()) {
+ assertEquals(index, theIndex);
+ assertArrayEquals(requestContext, reqCtx);
+ success.set(true);
+ } else {
+ assertTrue(status.getErrorMsg(), status.getErrorMsg().contains("RPC exception:Check connection["));
+ assertTrue(status.getErrorMsg(), status.getErrorMsg().contains("] fail and try to create new one"));
+ }
+ latch.countDown();
+ }
+ });
+ latch.await();
+ return success.get();
+ }
+
+ @Test
+ public void testNodeMetrics() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), false, 300, true));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertEquals(3, leader.listPeers().size());
+ // apply tasks to leader
+ this.sendTestTaskAndWait(leader);
+
+ {
+ final ByteBuffer data = ByteBuffer.wrap("no closure".getBytes());
+ final Task task = new Task(data, null);
+ leader.apply(task);
+ }
+
+ cluster.ensureSame(-1);
+ for (final Node node : cluster.getNodes()) {
+ System.out.println("-------------" + node.getNodeId() + "-------------");
+ final ConsoleReporter reporter = ConsoleReporter.forRegistry(node.getNodeMetrics().getMetricRegistry())
+ .build();
+ reporter.report();
+ reporter.close();
+ System.out.println();
+ }
+ // TODO check http status
+ assertEquals(2, cluster.getFollowers().size());
+ cluster.stopAll();
+ // System.out.println(node.getNodeMetrics().getMetrics());
+ }
+
+ @Test
+ public void testLeaderFail() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ // get leader
+ Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ LOG.info("Current leader is {}", leader.getLeaderId());
+ // apply tasks to leader
+ this.sendTestTaskAndWait(leader);
+
+ // stop leader
+ LOG.warn("Stop leader {}", leader.getNodeId().getPeerId());
+ final PeerId oldLeader = leader.getNodeId().getPeerId();
+ assertTrue(cluster.stop(leader.getNodeId().getPeerId().getEndpoint()));
+
+ // apply something when follower
+ final List<Node> followers = cluster.getFollowers();
+ assertFalse(followers.isEmpty());
+ this.sendTestTaskAndWait("follower apply ", followers.get(0), -1);
+
+ // elect new leader
+ cluster.waitLeader();
+ leader = cluster.getLeader();
+ LOG.info("Eelect new leader is {}", leader.getLeaderId());
+ // apply tasks to new leader
+ CountDownLatch latch = new CountDownLatch(10);
+ for (int i = 10; i < 20; i++) {
+ final ByteBuffer data = ByteBuffer.wrap(("hello" + i).getBytes());
+ final Task task = new Task(data, new ExpectClosure(latch));
+ leader.apply(task);
+ }
+ waitLatch(latch);
+
+ // restart old leader
+ LOG.info("restart old leader {}", oldLeader);
+ assertTrue(cluster.start(oldLeader.getEndpoint()));
+ // apply something
+ latch = new CountDownLatch(10);
+ for (int i = 20; i < 30; i++) {
+ final ByteBuffer data = ByteBuffer.wrap(("hello" + i).getBytes());
+ final Task task = new Task(data, new ExpectClosure(latch));
+ leader.apply(task);
+ }
+ waitLatch(latch);
+
+ // stop and clean old leader
+ cluster.stop(oldLeader.getEndpoint());
+ cluster.clean(oldLeader.getEndpoint());
+
+ // restart old leader
+ LOG.info("restart old leader {}", oldLeader);
+ assertTrue(cluster.start(oldLeader.getEndpoint()));
+ assertTrue(cluster.ensureSame(-1));
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertEquals(30, fsm.getLogs().size());
+ }
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testJoinNodes() throws Exception {
+ final PeerId peer0 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+ final PeerId peer1 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 1);
+ final PeerId peer2 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 2);
+ final PeerId peer3 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 3);
+
+ final ArrayList<PeerId> peers = new ArrayList<>();
+ peers.add(peer0);
+
+ // start single cluster
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ assertTrue(cluster.start(peer0.getEndpoint()));
+
+ cluster.waitLeader();
+
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ Assert.assertEquals(leader.getNodeId().getPeerId(), peer0);
+ this.sendTestTaskAndWait(leader);
+
+ // start peer1
+ assertTrue(cluster.start(peer1.getEndpoint(), true, 300));
+ // add peer1
+ CountDownLatch latch = new CountDownLatch(1);
+ peers.add(peer1);
+ leader.addPeer(peer1, new ExpectClosure(latch));
+ waitLatch(latch);
+
+ cluster.ensureSame(-1);
+ assertEquals(2, cluster.getFsms().size());
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertEquals(10, fsm.getLogs().size());
+ }
+
+ // add peer2 but not start
+ peers.add(peer2);
+ latch = new CountDownLatch(1);
+ leader.addPeer(peer2, new ExpectClosure(RaftError.ECATCHUP, latch));
+ waitLatch(latch);
+
+ // start peer2 after 2 seconds
+ Thread.sleep(2000);
+ assertTrue(cluster.start(peer2.getEndpoint(), true, 300));
+
+ Thread.sleep(10000);
+
+ // re-add peer2
+ latch = new CountDownLatch(2);
+ leader.addPeer(peer2, new ExpectClosure(latch));
+ // concurrent configuration change
+ leader.addPeer(peer3, new ExpectClosure(RaftError.EBUSY, latch));
+ waitLatch(latch);
+
+ // re-add peer2 directly
+
+ try {
+ leader.addPeer(peer2, new ExpectClosure(latch));
+ fail();
+ } catch (final IllegalArgumentException e) {
+ assertEquals("Peer already exists in current configuration", e.getMessage());
+ }
+
+ cluster.ensureSame();
+ assertEquals(3, cluster.getFsms().size());
+ assertEquals(2, cluster.getFollowers().size());
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertEquals(10, fsm.getLogs().size());
+ }
+ cluster.stopAll();
+ }
+
+ private void waitLatch(final CountDownLatch latch) throws InterruptedException {
+ assertTrue(latch.await(30, TimeUnit.SECONDS));
+ }
+
+ @Test
+ public void testRemoveFollower() throws Exception {
+ List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ // apply tasks to leader
+ this.sendTestTaskAndWait(leader);
+
+ cluster.ensureSame();
+
+ List<Node> followers = cluster.getFollowers();
+ assertEquals(2, followers.size());
+
+ final PeerId followerPeer = followers.get(0).getNodeId().getPeerId();
+ final Endpoint followerAddr = followerPeer.getEndpoint();
+
+ // stop and clean follower
+ LOG.info("Stop and clean follower {}", followerPeer);
+ assertTrue(cluster.stop(followerAddr));
+ cluster.clean(followerAddr);
+
+ // remove follower
+ LOG.info("Remove follower {}", followerPeer);
+ CountDownLatch latch = new CountDownLatch(1);
+ leader.removePeer(followerPeer, new ExpectClosure(latch));
+ waitLatch(latch);
+
+ this.sendTestTaskAndWait(leader, 10, RaftError.SUCCESS);
+ followers = cluster.getFollowers();
+ assertEquals(1, followers.size());
+
+ peers = TestUtils.generatePeers(3);
+ assertTrue(peers.remove(followerPeer));
+
+ // start follower
+ LOG.info("Start and add follower {}", followerPeer);
+ assertTrue(cluster.start(followerAddr));
+ // re-add follower
+ latch = new CountDownLatch(1);
+ leader.addPeer(followerPeer, new ExpectClosure(latch));
+ waitLatch(latch);
+
+ followers = cluster.getFollowers();
+ assertEquals(2, followers.size());
+
+ cluster.ensureSame();
+ assertEquals(3, cluster.getFsms().size());
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertEquals(20, fsm.getLogs().size());
+ }
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testRemoveLeader() throws Exception {
+ List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ // elect leader
+ cluster.waitLeader();
+
+ // get leader
+ Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ // apply tasks to leader
+ this.sendTestTaskAndWait(leader);
+
+ cluster.ensureSame();
+
+ List<Node> followers = cluster.getFollowers();
+ assertEquals(2, followers.size());
+
+ final PeerId oldLeader = leader.getNodeId().getPeerId().copy();
+ final Endpoint oldLeaderAddr = oldLeader.getEndpoint();
+
+ // remove old leader
+ LOG.info("Remove old leader {}", oldLeader);
+ CountDownLatch latch = new CountDownLatch(1);
+ leader.removePeer(oldLeader, new ExpectClosure(latch));
+ waitLatch(latch);
+ Thread.sleep(100);
+
+ // elect new leader
+ cluster.waitLeader();
+ leader = cluster.getLeader();
+ LOG.info("New leader is {}", leader);
+ assertNotNull(leader);
+ // apply tasks to new leader
+ this.sendTestTaskAndWait(leader, 10, RaftError.SUCCESS);
+
+ // stop and clean old leader
+ LOG.info("Stop and clean old leader {}", oldLeader);
+ assertTrue(cluster.stop(oldLeaderAddr));
+ cluster.clean(oldLeaderAddr);
+
+ // Add and start old leader
+ LOG.info("Start and add old leader {}", oldLeader);
+ assertTrue(cluster.start(oldLeaderAddr));
+
+ peers = TestUtils.generatePeers(3);
+ assertTrue(peers.remove(oldLeader));
+ latch = new CountDownLatch(1);
+ leader.addPeer(oldLeader, new ExpectClosure(latch));
+ waitLatch(latch);
+
+ followers = cluster.getFollowers();
+ assertEquals(2, followers.size());
+ cluster.ensureSame();
+ assertEquals(3, cluster.getFsms().size());
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertEquals(20, fsm.getLogs().size());
+ }
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testPreVote() throws Exception {
+ List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ cluster.waitLeader();
+ // get leader
+ Node leader = cluster.getLeader();
+ final long savedTerm = ((NodeImpl) leader).getCurrentTerm();
+ assertNotNull(leader);
+ // apply tasks to leader
+ this.sendTestTaskAndWait(leader);
+
+ cluster.ensureSame();
+
+ final List<Node> followers = cluster.getFollowers();
+ assertEquals(2, followers.size());
+
+ final PeerId followerPeer = followers.get(0).getNodeId().getPeerId();
+ final Endpoint followerAddr = followerPeer.getEndpoint();
+
+ // remove follower
+ LOG.info("Remove follower {}", followerPeer);
+ CountDownLatch latch = new CountDownLatch(1);
+ leader.removePeer(followerPeer, new ExpectClosure(latch));
+ waitLatch(latch);
+
+ this.sendTestTaskAndWait(leader, 10, RaftError.SUCCESS);
+
+ Thread.sleep(2000);
+
+ // add follower
+ LOG.info("Add follower {}", followerAddr);
+ peers = TestUtils.generatePeers(3);
+ assertTrue(peers.remove(followerPeer));
+ latch = new CountDownLatch(1);
+ leader.addPeer(followerPeer, new ExpectClosure(latch));
+ waitLatch(latch);
+ leader = cluster.getLeader();
+ assertNotNull(leader);
+ // leader term should not be changed.
+ assertEquals(savedTerm, ((NodeImpl) leader).getCurrentTerm());
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testSetPeer1() throws Exception {
+ final TestCluster cluster = new TestCluster("testSetPeer1", this.dataPath, new ArrayList<>());
+
+ final PeerId bootPeer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+ assertTrue(cluster.start(bootPeer.getEndpoint()));
+ final List<Node> nodes = cluster.getFollowers();
+ assertEquals(1, nodes.size());
+
+ final List<PeerId> peers = new ArrayList<>();
+ peers.add(bootPeer);
+ // reset peers from empty
+ assertTrue(nodes.get(0).resetPeers(new Configuration(peers)).isOk());
+ cluster.waitLeader();
+ assertNotNull(cluster.getLeader());
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testSetPeer2() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ cluster.waitLeader();
+ // get leader
+ Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ // apply tasks to leader
+ this.sendTestTaskAndWait(leader);
+
+ cluster.ensureSame();
+
+ final List<Node> followers = cluster.getFollowers();
+ assertEquals(2, followers.size());
+
+ final PeerId followerPeer1 = followers.get(0).getNodeId().getPeerId();
+ final Endpoint followerAddr1 = followerPeer1.getEndpoint();
+ final PeerId followerPeer2 = followers.get(1).getNodeId().getPeerId();
+ final Endpoint followerAddr2 = followerPeer2.getEndpoint();
+
+ LOG.info("Stop and clean follower {}", followerPeer1);
+ assertTrue(cluster.stop(followerAddr1));
+ cluster.clean(followerAddr1);
+
+ // apply tasks to leader again
+ this.sendTestTaskAndWait(leader, 10, RaftError.SUCCESS);
+ // set peer when no quorum die
+ final Endpoint leaderAddr = leader.getLeaderId().getEndpoint().copy();
+ LOG.info("Set peers to {}", leaderAddr);
+ final List<PeerId> newPeers = TestUtils.generatePeers(3);
+ assertTrue(newPeers.remove(followerPeer1));
+
+ LOG.info("Stop and clean follower {}", followerPeer2);
+ assertTrue(cluster.stop(followerAddr2));
+ cluster.clean(followerAddr2);
+
+ // leader will step-down, become follower
+ Thread.sleep(2000);
+ newPeers.clear();
+ newPeers.add(new PeerId(leaderAddr, 0));
+
+ // new peers equal to current conf
+ assertTrue(leader.resetPeers(new Configuration(peers)).isOk());
+ // set peer when quorum die
+ LOG.warn("Set peers to {}", leaderAddr);
+ assertTrue(leader.resetPeers(new Configuration(newPeers)).isOk());
+
+ cluster.waitLeader();
+ leader = cluster.getLeader();
+ assertNotNull(leader);
+ Assert.assertEquals(leaderAddr, leader.getNodeId().getPeerId().getEndpoint());
+
+ LOG.info("start follower {}", followerAddr1);
+ assertTrue(cluster.start(followerAddr1, true, 300));
+ LOG.info("start follower {}", followerAddr2);
+ assertTrue(cluster.start(followerAddr2, true, 300));
+
+ CountDownLatch latch = new CountDownLatch(1);
+ LOG.info("Add old follower {}", followerAddr1);
+ leader.addPeer(followerPeer1, new ExpectClosure(latch));
+ waitLatch(latch);
+
+ latch = new CountDownLatch(1);
+ LOG.info("Add old follower {}", followerAddr2);
+ leader.addPeer(followerPeer2, new ExpectClosure(latch));
+ waitLatch(latch);
+
+ newPeers.add(followerPeer1);
+ newPeers.add(followerPeer2);
+
+ cluster.ensureSame();
+ assertEquals(3, cluster.getFsms().size());
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertEquals(20, fsm.getLogs().size());
+ }
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testRestoreSnasphot() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ cluster.waitLeader();
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ // apply tasks to leader
+ this.sendTestTaskAndWait(leader);
+
+ cluster.ensureSame();
+ triggerLeaderSnapshot(cluster, leader);
+
+ // stop leader
+ final Endpoint leaderAddr = leader.getNodeId().getPeerId().getEndpoint().copy();
+ assertTrue(cluster.stop(leaderAddr));
+ Thread.sleep(2000);
+
+ // restart leader
+ cluster.waitLeader();
+ assertEquals(0, cluster.getLeaderFsm().getLoadSnapshotTimes());
+ assertTrue(cluster.start(leaderAddr));
+ cluster.ensureSame();
+ assertEquals(0, cluster.getLeaderFsm().getLoadSnapshotTimes());
+
+ cluster.stopAll();
+ }
+
+ private void triggerLeaderSnapshot(final TestCluster cluster, final Node leader) throws InterruptedException {
+ this.triggerLeaderSnapshot(cluster, leader, 1);
+ }
+
+ private void triggerLeaderSnapshot(final TestCluster cluster, final Node leader, final int times)
+ throws InterruptedException {
+ // trigger leader snapshot
+ // first snapshot will be triggered randomly
+ int snapshotTimes = cluster.getLeaderFsm().getSaveSnapshotTimes();
+ assertTrue("snapshotTimes=" + snapshotTimes + ", times=" + times, snapshotTimes == times - 1
+ || snapshotTimes == times);
+ final CountDownLatch latch = new CountDownLatch(1);
+ leader.snapshot(new ExpectClosure(latch));
+ waitLatch(latch);
+ assertEquals(snapshotTimes + 1, cluster.getLeaderFsm().getSaveSnapshotTimes());
+ }
+
+ @Test
+ public void testInstallSnapshotWithThrottle() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), false, 200, false, new ThroughputSnapshotThrottle(1024, 1)));
+ }
+
+ cluster.waitLeader();
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ // apply tasks to leader
+ this.sendTestTaskAndWait(leader);
+
+ cluster.ensureSame();
+
+ // stop follower1
+ final List<Node> followers = cluster.getFollowers();
+ assertEquals(2, followers.size());
+
+ final Endpoint followerAddr = followers.get(0).getNodeId().getPeerId().getEndpoint();
+ assertTrue(cluster.stop(followerAddr));
+
+ cluster.waitLeader();
+
+ // apply something more
+ this.sendTestTaskAndWait(leader, 10, RaftError.SUCCESS);
+
+ Thread.sleep(1000);
+
+ // trigger leader snapshot
+ triggerLeaderSnapshot(cluster, leader);
+ // apply something more
+ this.sendTestTaskAndWait(leader, 20, RaftError.SUCCESS);
+ // trigger leader snapshot
+ triggerLeaderSnapshot(cluster, leader, 2);
+
+ // wait leader to compact logs
+ Thread.sleep(1000);
+
+ // restart follower.
+ cluster.clean(followerAddr);
+ assertTrue(cluster.start(followerAddr, true, 300, false, new ThroughputSnapshotThrottle(1024, 1)));
+
+ Thread.sleep(2000);
+ cluster.ensureSame();
+
+ assertEquals(3, cluster.getFsms().size());
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertEquals(30, fsm.getLogs().size());
+ }
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testInstallLargeSnapshotWithThrottle() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(4);
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers.subList(0, 3));
+ for (int i = 0; i < peers.size() - 1; i++) {
+ final PeerId peer = peers.get(i);
+ final boolean started = cluster.start(peer.getEndpoint(), false, 200, false);
+ assertTrue(started);
+ }
+ cluster.waitLeader();
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ // apply tasks to leader
+ sendTestTaskAndWait(leader, 0, RaftError.SUCCESS);
+
+ cluster.ensureSame();
+
+ // apply something more
+ for (int i = 1; i < 100; i++) {
+ sendTestTaskAndWait(leader, i * 10, RaftError.SUCCESS);
+ }
+
+ Thread.sleep(1000);
+
+ // trigger leader snapshot
+ triggerLeaderSnapshot(cluster, leader);
+
+ // apply something more
+ for (int i = 100; i < 200; i++) {
+ sendTestTaskAndWait(leader, i * 10, RaftError.SUCCESS);
+ }
+ // trigger leader snapshot
+ triggerLeaderSnapshot(cluster, leader, 2);
+
+ // wait leader to compact logs
+ Thread.sleep(1000);
+
+ // add follower
+ final PeerId newPeer = peers.get(3);
+ final SnapshotThrottle snapshotThrottle = new ThroughputSnapshotThrottle(128, 1);
+ final boolean started = cluster.start(newPeer.getEndpoint(), true, 300, false, snapshotThrottle);
+ assertTrue(started);
+
+ final CountDownLatch latch = new CountDownLatch(1);
+ leader.addPeer(newPeer, status -> {
+ assertTrue(status.toString(), status.isOk());
+ latch.countDown();
+ });
+ waitLatch(latch);
+
+ cluster.ensureSame();
+
+ assertEquals(4, cluster.getFsms().size());
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertEquals(2000, fsm.getLogs().size());
+ }
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testInstallLargeSnapshot() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(4);
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers.subList(0, 3));
+ for (int i = 0; i < peers.size() - 1; i++) {
+ final PeerId peer = peers.get(i);
+ final boolean started = cluster.start(peer.getEndpoint(), false, 200, false);
+ assertTrue(started);
+ }
+ cluster.waitLeader();
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ // apply tasks to leader
+ sendTestTaskAndWait(leader, 0, RaftError.SUCCESS);
+
+ cluster.ensureSame();
+
+ // apply something more
+ for (int i = 1; i < 100; i++) {
+ sendTestTaskAndWait(leader, i * 10, RaftError.SUCCESS);
+ }
+
+ Thread.sleep(1000);
+
+ // trigger leader snapshot
+ triggerLeaderSnapshot(cluster, leader);
+
+ // apply something more
+ for (int i = 100; i < 200; i++) {
+ sendTestTaskAndWait(leader, i * 10, RaftError.SUCCESS);
+ }
+ // trigger leader snapshot
+ triggerLeaderSnapshot(cluster, leader, 2);
+
+ // wait leader to compact logs
+ Thread.sleep(1000);
+
+ // add follower
+ final PeerId newPeer = peers.get(3);
+ final RaftOptions raftOptions = new RaftOptions();
+ raftOptions.setMaxByteCountPerRpc(128);
+ final boolean started = cluster.start(newPeer.getEndpoint(), true, 300, false, null, raftOptions);
+ assertTrue(started);
+
+ final CountDownLatch latch = new CountDownLatch(1);
+ leader.addPeer(newPeer, status -> {
+ assertTrue(status.toString(), status.isOk());
+ latch.countDown();
+ });
+ waitLatch(latch);
+
+ cluster.ensureSame();
+
+ assertEquals(4, cluster.getFsms().size());
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertEquals(2000, fsm.getLogs().size());
+ }
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testInstallSnapshot() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ cluster.waitLeader();
+ // get leader
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ // apply tasks to leader
+ this.sendTestTaskAndWait(leader);
+
+ cluster.ensureSame();
+
+ // stop follower1
+ final List<Node> followers = cluster.getFollowers();
+ assertEquals(2, followers.size());
+
+ final Endpoint followerAddr = followers.get(0).getNodeId().getPeerId().getEndpoint();
+ assertTrue(cluster.stop(followerAddr));
+
+ // apply something more
+ this.sendTestTaskAndWait(leader, 10, RaftError.SUCCESS);
+
+ // trigger leader snapshot
+ triggerLeaderSnapshot(cluster, leader);
+ // apply something more
+ this.sendTestTaskAndWait(leader, 20, RaftError.SUCCESS);
+ triggerLeaderSnapshot(cluster, leader, 2);
+
+ // wait leader to compact logs
+ Thread.sleep(50);
+
+ //restart follower.
+ cluster.clean(followerAddr);
+ assertTrue(cluster.start(followerAddr, true, 300));
+
+ Thread.sleep(2000);
+ cluster.ensureSame();
+
+ assertEquals(3, cluster.getFsms().size());
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertEquals(30, fsm.getLogs().size());
+ }
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testNoSnapshot() throws Exception {
+ final Endpoint addr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+ NodeManager.getInstance().addAddress(addr);
+ final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+ final MockStateMachine fsm = new MockStateMachine(addr);
+ nodeOptions.setFsm(fsm);
+ nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+ nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+ nodeOptions.setInitialConf(new Configuration(Collections.singletonList(new PeerId(addr, 0))));
+
+ final Node node = new NodeImpl("unittest", new PeerId(addr, 0));
+ assertTrue(node.init(nodeOptions));
+ // wait node elect self as leader
+
+ Thread.sleep(2000);
+
+ this.sendTestTaskAndWait(node);
+
+ assertEquals(0, fsm.getSaveSnapshotTimes());
+ // do snapshot but returns error
+ CountDownLatch latch = new CountDownLatch(1);
+ node.snapshot(new ExpectClosure(RaftError.EINVAL, "Snapshot is not supported", latch));
+ waitLatch(latch);
+ assertEquals(0, fsm.getSaveSnapshotTimes());
+
+ latch = new CountDownLatch(1);
+ node.shutdown(new ExpectClosure(latch));
+ node.join();
+ waitLatch(latch);
+ }
+
+ @Test
+ public void testAutoSnapshot() throws Exception {
+ final Endpoint addr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+ NodeManager.getInstance().addAddress(addr);
+ final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+ final MockStateMachine fsm = new MockStateMachine(addr);
+ nodeOptions.setFsm(fsm);
+ nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+ nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+ nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+ nodeOptions.setSnapshotIntervalSecs(10);
+ nodeOptions.setInitialConf(new Configuration(Collections.singletonList(new PeerId(addr, 0))));
+
+ final Node node = new NodeImpl("unittest", new PeerId(addr, 0));
+ assertTrue(node.init(nodeOptions));
+ // wait node elect self as leader
+ Thread.sleep(2000);
+
+ sendTestTaskAndWait(node);
+
+ // wait for auto snapshot
+ Thread.sleep(10000);
+ // first snapshot will be triggered randomly
+ final int times = fsm.getSaveSnapshotTimes();
+ assertTrue("snapshotTimes=" + times, times >= 1);
+ assertTrue(fsm.getSnapshotIndex() > 0);
+
+ final CountDownLatch latch = new CountDownLatch(1);
+ node.shutdown(new ExpectClosure(latch));
+ node.join();
+ waitLatch(latch);
+ }
+
+ @Test
+ public void testLeaderShouldNotChange() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ cluster.waitLeader();
+ // get leader
+ final Node leader0 = cluster.getLeader();
+ assertNotNull(leader0);
+ final long savedTerm = ((NodeImpl) leader0).getCurrentTerm();
+ LOG.info("Current leader is {}, term is {}", leader0, savedTerm);
+ Thread.sleep(5000);
+ cluster.waitLeader();
+ final Node leader1 = cluster.getLeader();
+ assertNotNull(leader1);
+ LOG.info("Current leader is {}", leader1);
+ assertEquals(savedTerm, ((NodeImpl) leader1).getCurrentTerm());
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testRecoverFollower() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ cluster.waitLeader();
+
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+
+ Thread.sleep(100);
+
+ final List<Node> followers = cluster.getFollowers();
+ assertEquals(2, followers.size());
+
+ final Endpoint followerAddr = followers.get(0).getNodeId().getPeerId().getEndpoint().copy();
+ assertTrue(cluster.stop(followerAddr));
+
+ this.sendTestTaskAndWait(leader);
+
+ for (int i = 10; i < 30; i++) {
+ final ByteBuffer data = ByteBuffer.wrap(("no clusre" + i).getBytes());
+ final Task task = new Task(data, null);
+ leader.apply(task);
+ }
+ // wait leader to compact logs
+ Thread.sleep(5000);
+ // restart follower
+ assertTrue(cluster.start(followerAddr));
+ assertTrue(cluster.ensureSame(30));
+ assertEquals(3, cluster.getFsms().size());
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertEquals(30, fsm.getLogs().size());
+ }
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testLeaderTransfer() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 300);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ cluster.waitLeader();
+
+ Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ this.sendTestTaskAndWait(leader);
+
+ Thread.sleep(100);
+
+ final List<Node> followers = cluster.getFollowers();
+ assertEquals(2, followers.size());
+
+ final PeerId targetPeer = followers.get(0).getNodeId().getPeerId().copy();
+ LOG.info("Transfer leadership from {} to {}", leader, targetPeer);
+ assertTrue(leader.transferLeadershipTo(targetPeer).isOk());
+ Thread.sleep(1000);
+ cluster.waitLeader();
+ leader = cluster.getLeader();
+ Assert.assertEquals(leader.getNodeId().getPeerId(), targetPeer);
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testLeaderTransferBeforeLogIsCompleted() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 300);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), false, 1));
+ }
+
+ cluster.waitLeader();
+
+ Node leader = cluster.getLeader();
+ assertNotNull(leader);
+
+ Thread.sleep(100);
+
+ final List<Node> followers = cluster.getFollowers();
+ assertEquals(2, followers.size());
+
+ final PeerId targetPeer = followers.get(0).getNodeId().getPeerId().copy();
+ assertTrue(cluster.stop(targetPeer.getEndpoint()));
+ this.sendTestTaskAndWait(leader);
+ LOG.info("Transfer leadership from {} to {}", leader, targetPeer);
+ assertTrue(leader.transferLeadershipTo(targetPeer).isOk());
+ final CountDownLatch latch = new CountDownLatch(1);
+ final Task task = new Task(ByteBuffer.wrap("aaaaa".getBytes()), new ExpectClosure(RaftError.EBUSY, latch));
+ leader.apply(task);
+ waitLatch(latch);
+
+ assertTrue(cluster.start(targetPeer.getEndpoint()));
+ Thread.sleep(5000);
+ cluster.waitLeader();
+ leader = cluster.getLeader();
+ Assert.assertEquals(targetPeer, leader.getNodeId().getPeerId());
+ assertTrue(cluster.ensureSame(5));
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testLeaderTransferResumeOnFailure() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 300);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint(), false, 1));
+ }
+
+ cluster.waitLeader();
+
+ Node leader = cluster.getLeader();
+ assertNotNull(leader);
+
+ final List<Node> followers = cluster.getFollowers();
+ assertEquals(2, followers.size());
+
+ final PeerId targetPeer = followers.get(0).getNodeId().getPeerId().copy();
+ assertTrue(cluster.stop(targetPeer.getEndpoint()));
+
+ this.sendTestTaskAndWait(leader);
+
+ assertTrue(leader.transferLeadershipTo(targetPeer).isOk());
+ final Node savedLeader = leader;
+ //try to apply task when transferring leadership
+ CountDownLatch latch = new CountDownLatch(1);
+ Task task = new Task(ByteBuffer.wrap("aaaaa".getBytes()), new ExpectClosure(RaftError.EBUSY, latch));
+ leader.apply(task);
+ waitLatch(latch);
+
+ Thread.sleep(100);
+ cluster.waitLeader();
+ leader = cluster.getLeader();
+ assertSame(leader, savedLeader);
+
+ // restart target peer
+ assertTrue(cluster.start(targetPeer.getEndpoint()));
+ Thread.sleep(100);
+ // retry apply task
+ latch = new CountDownLatch(1);
+ task = new Task(ByteBuffer.wrap("aaaaa".getBytes()), new ExpectClosure(latch));
+ leader.apply(task);
+ waitLatch(latch);
+
+ assertTrue(cluster.ensureSame(5));
+ cluster.stopAll();
+ }
+
+ /**
+ * mock state machine that fails to load snapshot.
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-23 11:45:29 AM
+ */
+ static class MockFSM1 extends MockStateMachine {
+
+ public MockFSM1() {
+ this(new Endpoint(Utils.IP_ANY, 0));
+ }
+
+ public MockFSM1(final Endpoint address) {
+ super(address);
+ }
+
+ @Override
+ public boolean onSnapshotLoad(final SnapshotReader reader) {
+ return false;
+ }
+
+ }
+
+ @Test
+ public void testShutdownAndJoinWorkAfterInitFails() throws Exception {
+ final Endpoint addr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+ NodeManager.getInstance().addAddress(addr);
+ {
+ final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+ final MockStateMachine fsm = new MockStateMachine(addr);
+ nodeOptions.setFsm(fsm);
+ nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+ nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+ nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+ nodeOptions.setSnapshotIntervalSecs(10);
+ nodeOptions.setInitialConf(new Configuration(Collections.singletonList(new PeerId(addr, 0))));
+
+ final Node node = new NodeImpl("unittest", new PeerId(addr, 0));
+ assertTrue(node.init(nodeOptions));
+ Thread.sleep(1000);
+ this.sendTestTaskAndWait(node);
+
+ // save snapshot
+ final CountDownLatch latch = new CountDownLatch(1);
+ node.snapshot(new ExpectClosure(latch));
+ waitLatch(latch);
+ node.shutdown();
+ node.join();
+ }
+ {
+ final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+ final MockStateMachine fsm = new MockFSM1(addr);
+ nodeOptions.setFsm(fsm);
+ nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+ nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+ nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+ nodeOptions.setSnapshotIntervalSecs(10);
+ nodeOptions.setInitialConf(new Configuration(Collections.singletonList(new PeerId(addr, 0))));
+
+ final Node node = new NodeImpl("unittest", new PeerId(addr, 0));
+ assertFalse(node.init(nodeOptions));
+ node.shutdown();
+ node.join();
+ }
+ }
+
+ @Test
+ public void testShuttingDownLeaderTriggerTimeoutNow() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 300);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ cluster.waitLeader();
+
+ Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ final Node oldLeader = leader;
+
+ LOG.info("Shutdown leader {}", leader);
+ leader.shutdown();
+ leader.join();
+
+ Thread.sleep(100);
+ leader = cluster.getLeader();
+ cluster.waitLeader();
+ assertNotNull(leader);
+ assertNotSame(leader, oldLeader);
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testRemovingLeaderTriggerTimeoutNow() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 300);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ cluster.waitLeader();
+
+ Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ final Node oldLeader = leader;
+
+ final CountDownLatch latch = new CountDownLatch(1);
+ oldLeader.removePeer(oldLeader.getNodeId().getPeerId(), new ExpectClosure(latch));
+ waitLatch(latch);
+
+ Thread.sleep(100);
+ leader = cluster.getLeader();
+ assertNotNull(leader);
+ assertNotSame(leader, oldLeader);
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testTransferShouldWorkAfterInstallSnapshot() throws Exception {
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 1000);
+
+ for (int i = 0; i < peers.size() - 1; i++) {
+ assertTrue(cluster.start(peers.get(i).getEndpoint()));
+ }
+
+ cluster.waitLeader();
+
+ Node leader = cluster.getLeader();
+ assertNotNull(leader);
+
+ this.sendTestTaskAndWait(leader);
+
+ final List<Node> followers = cluster.getFollowers();
+ assertEquals(1, followers.size());
+
+ final PeerId follower = followers.get(0).getNodeId().getPeerId();
+ assertTrue(leader.transferLeadershipTo(follower).isOk());
+ Thread.sleep(2000);
+ leader = cluster.getLeader();
+ Assert.assertEquals(follower, leader.getNodeId().getPeerId());
+
+ CountDownLatch latch = new CountDownLatch(1);
+ leader.snapshot(new ExpectClosure(latch));
+ waitLatch(latch);
+ latch = new CountDownLatch(1);
+ leader.snapshot(new ExpectClosure(latch));
+ waitLatch(latch);
+
+ // start the last peer which should be recover with snapshot.
+ final PeerId lastPeer = peers.get(2);
+ assertTrue(cluster.start(lastPeer.getEndpoint()));
+ Thread.sleep(5000);
+ assertTrue(leader.transferLeadershipTo(lastPeer).isOk());
+ Thread.sleep(2000);
+ leader = cluster.getLeader();
+ Assert.assertEquals(lastPeer, leader.getNodeId().getPeerId());
+ assertEquals(3, cluster.getFsms().size());
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertEquals(10, fsm.getLogs().size());
+ }
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testAppendEntriesWhenFollowerIsInErrorState() throws Exception {
+ // start five nodes
+ final List<PeerId> peers = TestUtils.generatePeers(5);
+
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 1000);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ cluster.waitLeader();
+ final Node oldLeader = cluster.getLeader();
+ assertNotNull(oldLeader);
+ // apply something
+ this.sendTestTaskAndWait(oldLeader);
+
+ // set one follower into error state
+ final List<Node> followers = cluster.getFollowers();
+ assertEquals(4, followers.size());
+ final Node errorNode = followers.get(0);
+ final PeerId errorPeer = errorNode.getNodeId().getPeerId().copy();
+ final Endpoint errorFollowerAddr = errorPeer.getEndpoint();
+ LOG.info("Set follower {} into error state", errorNode);
+ ((NodeImpl) errorNode).onError(new RaftException(EnumOutter.ErrorType.ERROR_TYPE_STATE_MACHINE, new Status(-1,
+ "Follower has something wrong.")));
+
+ // increase term by stopping leader and electing a new leader again
+ final Endpoint oldLeaderAddr = oldLeader.getNodeId().getPeerId().getEndpoint().copy();
+ assertTrue(cluster.stop(oldLeaderAddr));
+ cluster.waitLeader();
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ LOG.info("Elect a new leader {}", leader);
+ // apply something again
+ this.sendTestTaskAndWait(leader, 10, RaftError.SUCCESS);
+
+ // stop error follower
+ Thread.sleep(20);
+ LOG.info("Stop error follower {}", errorNode);
+ assertTrue(cluster.stop(errorFollowerAddr));
+ // restart error and old leader
+ LOG.info("Restart error follower {} and old leader {}", errorFollowerAddr, oldLeaderAddr);
+
+ assertTrue(cluster.start(errorFollowerAddr));
+ assertTrue(cluster.start(oldLeaderAddr));
+ cluster.ensureSame();
+ assertEquals(5, cluster.getFsms().size());
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertEquals(20, fsm.getLogs().size());
+ }
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testFollowerStartStopFollowing() throws Exception {
+ // start five nodes
+ final List<PeerId> peers = TestUtils.generatePeers(5);
+
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 1000);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+ cluster.waitLeader();
+ final Node firstLeader = cluster.getLeader();
+ assertNotNull(firstLeader);
+ // apply something
+ this.sendTestTaskAndWait(firstLeader);
+
+ // assert follow times
+ final List<Node> firstFollowers = cluster.getFollowers();
+ assertEquals(4, firstFollowers.size());
+ for (final Node node : firstFollowers) {
+ assertEquals(1, ((MockStateMachine) node.getOptions().getFsm()).getOnStartFollowingTimes());
+ assertEquals(0, ((MockStateMachine) node.getOptions().getFsm()).getOnStopFollowingTimes());
+ }
+
+ // stop leader and elect new one
+ final Endpoint fstLeaderAddr = firstLeader.getNodeId().getPeerId().getEndpoint();
+ assertTrue(cluster.stop(fstLeaderAddr));
+ cluster.waitLeader();
+ final Node secondLeader = cluster.getLeader();
+ assertNotNull(secondLeader);
+ this.sendTestTaskAndWait(secondLeader, 10, RaftError.SUCCESS);
+
+ // ensure start/stop following times
+ final List<Node> secondFollowers = cluster.getFollowers();
+ assertEquals(3, secondFollowers.size());
+ for (final Node node : secondFollowers) {
+ assertEquals(2, ((MockStateMachine) node.getOptions().getFsm()).getOnStartFollowingTimes());
+ assertEquals(1, ((MockStateMachine) node.getOptions().getFsm()).getOnStopFollowingTimes());
+ }
+
+ // transfer leadership to a follower
+ final PeerId targetPeer = secondFollowers.get(0).getNodeId().getPeerId().copy();
+ assertTrue(secondLeader.transferLeadershipTo(targetPeer).isOk());
+ Thread.sleep(100);
+ cluster.waitLeader();
+ final Node thirdLeader = cluster.getLeader();
+ Assert.assertEquals(targetPeer, thirdLeader.getNodeId().getPeerId());
+ this.sendTestTaskAndWait(thirdLeader, 20, RaftError.SUCCESS);
+
+ final List<Node> thirdFollowers = cluster.getFollowers();
+ assertEquals(3, thirdFollowers.size());
+ for (int i = 0; i < 3; i++) {
+ if (thirdFollowers.get(i).getNodeId().getPeerId().equals(secondLeader.getNodeId().getPeerId())) {
+ assertEquals(2,
+ ((MockStateMachine) thirdFollowers.get(i).getOptions().getFsm()).getOnStartFollowingTimes());
+ assertEquals(1,
+ ((MockStateMachine) thirdFollowers.get(i).getOptions().getFsm()).getOnStopFollowingTimes());
+ continue;
+ }
+ assertEquals(3, ((MockStateMachine) thirdFollowers.get(i).getOptions().getFsm()).getOnStartFollowingTimes());
+ assertEquals(2, ((MockStateMachine) thirdFollowers.get(i).getOptions().getFsm()).getOnStopFollowingTimes());
+ }
+
+ cluster.ensureSame();
+ cluster.stopAll();
+ }
+
+ @Test
+ public void readCommittedUserLog() throws Exception {
+ // setup cluster
+ final List<PeerId> peers = TestUtils.generatePeers(3);
+
+ final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 1000);
+
+ for (final PeerId peer : peers) {
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+ cluster.waitLeader();
+
+ final Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ this.sendTestTaskAndWait(leader);
+
+ // index == 1 is a CONFIGURATION log, so real_index will be 2 when returned.
+ UserLog userLog = leader.readCommittedUserLog(1);
+ assertNotNull(userLog);
+ assertEquals(2, userLog.getIndex());
+ assertEquals("hello0", new String(userLog.getData().array()));
+
+ // index == 5 is a DATA log(a user log)
+ userLog = leader.readCommittedUserLog(5);
+ assertNotNull(userLog);
+ assertEquals(5, userLog.getIndex());
+ assertEquals("hello3", new String(userLog.getData().array()));
+
+ // index == 15 is greater than last_committed_index
+ try {
+ assertNull(leader.readCommittedUserLog(15));
+ fail();
+ } catch (final LogIndexOutOfBoundsException e) {
+ assertEquals(e.getMessage(), "Request index 15 is greater than lastAppliedIndex: 11");
+ }
+
+ // index == 0 invalid request
+ try {
+ assertNull(leader.readCommittedUserLog(0));
+ fail();
+ } catch (final LogIndexOutOfBoundsException e) {
+ assertEquals(e.getMessage(), "Request index is invalid: 0");
+ }
+ LOG.info("Trigger leader snapshot");
+ CountDownLatch latch = new CountDownLatch(1);
+ leader.snapshot(new ExpectClosure(latch));
+ waitLatch(latch);
+
+ // remove and add a peer to add two CONFIGURATION logs
+ final List<Node> followers = cluster.getFollowers();
+ assertEquals(2, followers.size());
+ final Node testFollower = followers.get(0);
+ latch = new CountDownLatch(1);
+ leader.removePeer(testFollower.getNodeId().getPeerId(), new ExpectClosure(latch));
+ waitLatch(latch);
+ latch = new CountDownLatch(1);
+ leader.addPeer(testFollower.getNodeId().getPeerId(), new ExpectClosure(latch));
+ waitLatch(latch);
+
+ this.sendTestTaskAndWait(leader, 10, RaftError.SUCCESS);
+
+ // trigger leader snapshot for the second time, after this the log of index 1~11 will be deleted.
+ LOG.info("Trigger leader snapshot");
+ latch = new CountDownLatch(1);
+ leader.snapshot(new ExpectClosure(latch));
+ waitLatch(latch);
+ Thread.sleep(100);
+
+ // index == 5 log has been deleted in log_storage.
+ try {
+ leader.readCommittedUserLog(5);
+ fail();
+ } catch (final LogNotFoundException e) {
+ assertEquals("User log is deleted at index: 5", e.getMessage());
+ }
+
+ // index == 12、index == 13、index=14、index=15 are 4 CONFIGURATION logs(joint consensus), so real_index will be 16 when returned.
+ userLog = leader.readCommittedUserLog(12);
+ assertNotNull(userLog);
+ assertEquals(16, userLog.getIndex());
+ assertEquals("hello10", new String(userLog.getData().array()));
+
+ // now index == 17 is a user log
+ userLog = leader.readCommittedUserLog(17);
+ assertNotNull(userLog);
+ assertEquals(17, userLog.getIndex());
+ assertEquals("hello11", new String(userLog.getData().array()));
+
+ cluster.ensureSame();
+ assertEquals(3, cluster.getFsms().size());
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertEquals(20, fsm.getLogs().size());
+ for (int i = 0; i < 20; i++) {
+ assertEquals("hello" + i, new String(fsm.getLogs().get(i).array()));
+ }
+ }
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testBootStrapWithSnapshot() throws Exception {
+ final Endpoint addr = JRaftUtils.getEndPoint("127.0.0.1:5006");
+ final MockStateMachine fsm = new MockStateMachine(addr);
+
+ for (char ch = 'a'; ch <= 'z'; ch++) {
+ fsm.getLogs().add(ByteBuffer.wrap(new byte[] { (byte) ch }));
+ }
+
+ final BootstrapOptions opts = new BootstrapOptions();
+ opts.setLastLogIndex(fsm.getLogs().size());
+ opts.setRaftMetaUri(this.dataPath + File.separator + "meta");
+ opts.setLogUri(this.dataPath + File.separator + "log");
+ opts.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+ opts.setGroupConf(JRaftUtils.getConfiguration("127.0.0.1:5006"));
+ opts.setFsm(fsm);
+
+ NodeManager.getInstance().addAddress(addr);
+ assertTrue(JRaftUtils.bootstrap(opts));
+
+ final NodeOptions nodeOpts = createNodeOptionsWithSharedTimer();
+ nodeOpts.setRaftMetaUri(this.dataPath + File.separator + "meta");
+ nodeOpts.setLogUri(this.dataPath + File.separator + "log");
+ nodeOpts.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+ nodeOpts.setFsm(fsm);
+
+ final NodeImpl node = new NodeImpl("test", new PeerId(addr, 0));
+ assertTrue(node.init(nodeOpts));
+ assertEquals(26, fsm.getLogs().size());
+
+ for (int i = 0; i < 26; i++) {
+ assertEquals('a' + i, fsm.getLogs().get(i).get());
+ }
+
+ while (!node.isLeader()) {
+ Thread.sleep(20);
+ }
+ this.sendTestTaskAndWait(node);
+ assertEquals(36, fsm.getLogs().size());
+ node.shutdown();
+ node.join();
+ }
+
+ @Test
+ public void testBootStrapWithoutSnapshot() throws Exception {
+ final Endpoint addr = JRaftUtils.getEndPoint("127.0.0.1:5006");
+ final MockStateMachine fsm = new MockStateMachine(addr);
+
+ final BootstrapOptions opts = new BootstrapOptions();
+ opts.setLastLogIndex(0);
+ opts.setRaftMetaUri(this.dataPath + File.separator + "meta");
+ opts.setLogUri(this.dataPath + File.separator + "log");
+ opts.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+ opts.setGroupConf(JRaftUtils.getConfiguration("127.0.0.1:5006"));
+ opts.setFsm(fsm);
+
+ NodeManager.getInstance().addAddress(addr);
+ assertTrue(JRaftUtils.bootstrap(opts));
+
+ final NodeOptions nodeOpts = createNodeOptionsWithSharedTimer();
+ nodeOpts.setRaftMetaUri(this.dataPath + File.separator + "meta");
+ nodeOpts.setLogUri(this.dataPath + File.separator + "log");
+ nodeOpts.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+ nodeOpts.setFsm(fsm);
+
+ final NodeImpl node = new NodeImpl("test", new PeerId(addr, 0));
+ assertTrue(node.init(nodeOpts));
+ while (!node.isLeader()) {
+ Thread.sleep(20);
+ }
+ this.sendTestTaskAndWait(node);
+ assertEquals(10, fsm.getLogs().size());
+ node.shutdown();
+ node.join();
+ }
+
+ @Test
+ public void testChangePeers() throws Exception {
+ final PeerId peer0 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+ final TestCluster cluster = new TestCluster("testChangePeers", this.dataPath, Collections.singletonList(peer0));
+ assertTrue(cluster.start(peer0.getEndpoint()));
+
+ cluster.waitLeader();
+ Node leader = cluster.getLeader();
+ this.sendTestTaskAndWait(leader);
+
+ for (int i = 1; i < 10; i++) {
+ final PeerId peer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + i);
+ assertTrue(cluster.start(peer.getEndpoint(), true, 300));
+ }
+ for (int i = 0; i < 9; i++) {
+ cluster.waitLeader();
+ leader = cluster.getLeader();
+ assertNotNull(leader);
+ PeerId peer = new PeerId(TestUtils.getMyIp(), peer0.getEndpoint().getPort() + i);
+ Assert.assertEquals(peer, leader.getNodeId().getPeerId());
+ peer = new PeerId(TestUtils.getMyIp(), peer0.getEndpoint().getPort() + i + 1);
+ final SynchronizedClosure done = new SynchronizedClosure();
+ leader.changePeers(new Configuration(Collections.singletonList(peer)), done);
+ assertTrue(done.await().isOk());
+ }
+ assertTrue(cluster.ensureSame());
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testChangePeersAddMultiNodes() throws Exception {
+ final PeerId peer0 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+ final TestCluster cluster = new TestCluster("testChangePeers", this.dataPath, Collections.singletonList(peer0));
+ assertTrue(cluster.start(peer0.getEndpoint()));
+
+ cluster.waitLeader();
+ final Node leader = cluster.getLeader();
+ this.sendTestTaskAndWait(leader);
+
+ final Configuration conf = new Configuration();
+ for (int i = 0; i < 3; i++) {
+ final PeerId peer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + i);
+ conf.addPeer(peer);
+ }
+
+ PeerId peer = new PeerId(TestUtils.getMyIp(), peer0.getEndpoint().getPort() + 1);
+ // fail, because the peers are not started.
+ final SynchronizedClosure done = new SynchronizedClosure();
+ leader.changePeers(new Configuration(Collections.singletonList(peer)), done);
+ Assert.assertEquals(RaftError.ECATCHUP, done.await().getRaftError());
+
+ // start peer1
+ assertTrue(cluster.start(peer.getEndpoint()));
+ // still fail, because peer2 is not started
+ done.reset();
+ leader.changePeers(conf, done);
+ Assert.assertEquals(RaftError.ECATCHUP, done.await().getRaftError());
+ // start peer2
+ peer = new PeerId(TestUtils.getMyIp(), peer0.getEndpoint().getPort() + 2);
+ assertTrue(cluster.start(peer.getEndpoint()));
+ done.reset();
+ // works
+ leader.changePeers(conf, done);
+ assertTrue(done.await().isOk());
+
+ assertTrue(cluster.ensureSame());
+ assertEquals(3, cluster.getFsms().size());
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertEquals(10, fsm.getLogs().size());
+ }
+
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testChangePeersStepsDownInJointConsensus() throws Exception {
+ final List<PeerId> peers = new ArrayList<>();
+ final PeerId peer0 = JRaftUtils.getPeerId("127.0.0.1:5006");
+ final PeerId peer1 = JRaftUtils.getPeerId("127.0.0.1:5007");
+ final PeerId peer2 = JRaftUtils.getPeerId("127.0.0.1:5008");
+ final PeerId peer3 = JRaftUtils.getPeerId("127.0.0.1:5009");
+
+ // start single cluster
+ peers.add(peer0);
+ final TestCluster cluster = new TestCluster("testChangePeersStepsDownInJointConsensus", this.dataPath, peers);
+ assertTrue(cluster.start(peer0.getEndpoint()));
+
+ cluster.waitLeader();
+ Node leader = cluster.getLeader();
+ assertNotNull(leader);
+ this.sendTestTaskAndWait(leader);
+
+ // start peer1-3
+ assertTrue(cluster.start(peer1.getEndpoint()));
+ assertTrue(cluster.start(peer2.getEndpoint()));
+ assertTrue(cluster.start(peer3.getEndpoint()));
+
+ final Configuration conf = new Configuration();
+ conf.addPeer(peer0);
+ conf.addPeer(peer1);
+ conf.addPeer(peer2);
+ conf.addPeer(peer3);
+
+ // change peers
+ final SynchronizedClosure done = new SynchronizedClosure();
+ leader.changePeers(conf, done);
+ assertTrue(done.await().isOk());
+
+ // stop peer3
+ assertTrue(cluster.stop(peer3.getEndpoint()));
+
+ conf.removePeer(peer0);
+ conf.removePeer(peer1);
+
+ // Change peers to [peer2, peer3], which must fail since peer3 is stopped
+ done.reset();
+ leader.changePeers(conf, done);
+ Assert.assertEquals(RaftError.EPERM, done.await().getRaftError());
+ LOG.info(done.getStatus().toString());
+
+ assertFalse(((NodeImpl) leader).getConf().isStable());
+
+ leader = cluster.getLeader();
+ assertNull(leader);
+
+ assertTrue(cluster.start(peer3.getEndpoint()));
+ Thread.sleep(1000);
+ cluster.waitLeader();
+ leader = cluster.getLeader();
+ final List<PeerId> thePeers = leader.listPeers();
+ assertTrue(thePeers.size() > 0);
+ assertEquals(conf.getPeerSet(), new HashSet<>(thePeers));
+
+ cluster.stopAll();
+ }
+
+ static class ChangeArg {
+ TestCluster c;
+ List<PeerId> peers;
+ volatile boolean stop;
+ boolean dontRemoveFirstPeer;
+
+ public ChangeArg(final TestCluster c, final List<PeerId> peers, final boolean stop,
+ final boolean dontRemoveFirstPeer) {
+ super();
+ this.c = c;
+ this.peers = peers;
+ this.stop = stop;
+ this.dontRemoveFirstPeer = dontRemoveFirstPeer;
+ }
+
+ }
+
+ private Future<?> startChangePeersThread(final ChangeArg arg) {
+
+ final Set<RaftError> expectedErrors = new HashSet<>();
+ expectedErrors.add(RaftError.EBUSY);
+ expectedErrors.add(RaftError.EPERM);
+ expectedErrors.add(RaftError.ECATCHUP);
+
+ return Utils.runInThread(() -> {
+ try {
+ while (!arg.stop) {
+ arg.c.waitLeader();
+ final Node leader = arg.c.getLeader();
+ if (leader == null) {
+ continue;
+ }
+ // select peers in random
+ final Configuration conf = new Configuration();
+ if (arg.dontRemoveFirstPeer) {
+ conf.addPeer(arg.peers.get(0));
+ }
+ for (int i = 0; i < arg.peers.size(); i++) {
+ final boolean select = ThreadLocalRandom.current().nextInt(64) < 32;
+ if (select && !conf.contains(arg.peers.get(i))) {
+ conf.addPeer(arg.peers.get(i));
+ }
+ }
+ if (conf.isEmpty()) {
+ LOG.warn("No peer has been selected");
+ continue;
+ }
+ final SynchronizedClosure done = new SynchronizedClosure();
+ leader.changePeers(conf, done);
+ done.await();
+ assertTrue(done.getStatus().toString(),
+ done.getStatus().isOk() || expectedErrors.contains(done.getStatus().getRaftError()));
+ }
+ } catch (final InterruptedException e) {
+ LOG.error("ChangePeersThread is interrupted", e);
+ }
+ });
+ }
+
+ @Test
+ public void testChangePeersChaosWithSnapshot() throws Exception {
+ // start cluster
+ final List<PeerId> peers = new ArrayList<>();
+ peers.add(new PeerId("127.0.0.1", TestUtils.INIT_PORT));
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers, 1000);
+ assertTrue(cluster.start(peers.get(0).getEndpoint(), false, 2));
+ // start other peers
+ for (int i = 1; i < 10; i++) {
+ final PeerId peer = new PeerId("127.0.0.1", TestUtils.INIT_PORT + i);
+ peers.add(peer);
+ assertTrue(cluster.start(peer.getEndpoint()));
+ }
+
+ final ChangeArg arg = new ChangeArg(cluster, peers, false, false);
+
+ final Future<?> future = startChangePeersThread(arg);
+ for (int i = 0; i < 5000;) {
+ cluster.waitLeader();
+ final Node leader = cluster.getLeader();
+ if (leader == null) {
+ continue;
+ }
+ final SynchronizedClosure done = new SynchronizedClosure();
+ final Task task = new Task(ByteBuffer.wrap(("hello" + i).getBytes()), done);
+ leader.apply(task);
+ final Status status = done.await();
+ if (status.isOk()) {
+ if (++i % 100 == 0) {
+ System.out.println("Progress:" + i);
+ }
+ } else {
+ assertEquals(RaftError.EPERM, status.getRaftError());
+ }
+ }
+ arg.stop = true;
+ future.get();
+ cluster.waitLeader();
+ final SynchronizedClosure done = new SynchronizedClosure();
+ final Node leader = cluster.getLeader();
+ leader.changePeers(new Configuration(peers), done);
+ final Status st = done.await();
+ assertTrue(st.getErrorMsg(), st.isOk());
+ cluster.ensureSame();
+ assertEquals(10, cluster.getFsms().size());
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertTrue(fsm.getLogs().size() >= 5000);
+ }
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testChangePeersChaosWithoutSnapshot() throws Exception {
+ // start cluster
+ final List<PeerId> peers = new ArrayList<>();
+ peers.add(new PeerId("127.0.0.1", TestUtils.INIT_PORT));
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers, 1000);
+ assertTrue(cluster.start(peers.get(0).getEndpoint(), false, 100000));
+ // start other peers
+ for (int i = 1; i < 10; i++) {
+ final PeerId peer = new PeerId("127.0.0.1", TestUtils.INIT_PORT + i);
+ peers.add(peer);
+ assertTrue(cluster.start(peer.getEndpoint(), true, 10000));
+ }
+
+ final ChangeArg arg = new ChangeArg(cluster, peers, false, true);
+
+ final Future<?> future = startChangePeersThread(arg);
+ final int tasks = 5000;
+ for (int i = 0; i < tasks;) {
+ cluster.waitLeader();
+ final Node leader = cluster.getLeader();
+ if (leader == null) {
+ continue;
+ }
+ final SynchronizedClosure done = new SynchronizedClosure();
+ final Task task = new Task(ByteBuffer.wrap(("hello" + i).getBytes()), done);
+ leader.apply(task);
+ final Status status = done.await();
+ if (status.isOk()) {
+ if (++i % 100 == 0) {
+ System.out.println("Progress:" + i);
+ }
+ } else {
+ assertEquals(RaftError.EPERM, status.getRaftError());
+ }
+ }
+ arg.stop = true;
+ future.get();
+ cluster.waitLeader();
+ final SynchronizedClosure done = new SynchronizedClosure();
+ final Node leader = cluster.getLeader();
+ leader.changePeers(new Configuration(peers), done);
+ assertTrue(done.await().isOk());
+ cluster.ensureSame();
+ assertEquals(10, cluster.getFsms().size());
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ assertTrue(fsm.getLogs().size() >= tasks);
+ assertTrue(fsm.getLogs().size() - tasks < 100);
+ }
+ cluster.stopAll();
+ }
+
+ @Test
+ public void testChangePeersChaosApplyTasks() throws Exception {
+ // start cluster
+ final List<PeerId> peers = new ArrayList<>();
+ peers.add(new PeerId("127.0.0.1", TestUtils.INIT_PORT));
+ final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers, 1000);
+ assertTrue(cluster.start(peers.get(0).getEndpoint(), false, 100000));
+ // start other peers
+ for (int i = 1; i < 10; i++) {
+ final PeerId peer = new PeerId("127.0.0.1", TestUtils.INIT_PORT + i);
+ peers.add(peer);
+ assertTrue(cluster.start(peer.getEndpoint(), true, 100000));
+ }
+
+ final int threads = 3;
+ final List<ChangeArg> args = new ArrayList<>();
+ final List<Future<?>> futures = new ArrayList<>();
+ final CountDownLatch latch = new CountDownLatch(threads);
+ for (int t = 0; t < threads; t++) {
+ final ChangeArg arg = new ChangeArg(cluster, peers, false, true);
+ args.add(arg);
+ futures.add(startChangePeersThread(arg));
+
+ Utils.runInThread(() -> {
+ try {
+ for (int i = 0; i < 5000;) {
+ cluster.waitLeader();
+ final Node leader = cluster.getLeader();
+ if (leader == null) {
+ continue;
+ }
+ final SynchronizedClosure done = new SynchronizedClosure();
+ final Task task = new Task(ByteBuffer.wrap(("hello" + i).getBytes()), done);
+ leader.apply(task);
+ final Status status = done.await();
+ if (status.isOk()) {
+ if (++i % 100 == 0) {
+ System.out.println("Progress:" + i);
+ }
+ } else {
+ assertEquals(RaftError.EPERM, status.getRaftError());
+ }
+ }
+ } catch (final Exception e) {
+ e.printStackTrace();
+ } finally {
+ latch.countDown();
+ }
+ });
+ }
+
+ latch.await();
+ for (final ChangeArg arg : args) {
+ arg.stop = true;
+ }
+ for (final Future<?> future : futures) {
+ future.get();
+ }
+
+ cluster.waitLeader();
+ final SynchronizedClosure done = new SynchronizedClosure();
+ final Node leader = cluster.getLeader();
+ leader.changePeers(new Configuration(peers), done);
+ assertTrue(done.await().isOk());
+ cluster.ensureSame();
+ assertEquals(10, cluster.getFsms().size());
+ try {
+ for (final MockStateMachine fsm : cluster.getFsms()) {
+ final int logSize = fsm.getLogs().size();
+ assertTrue("logSize= " + logSize, logSize >= 5000 * threads);
+ assertTrue("logSize= " + logSize, logSize - 5000 * threads < 100);
+ }
+ } finally {
+ cluster.stopAll();
+ }
+ }
+
+ private NodeOptions createNodeOptionsWithSharedTimer() {
+ final NodeOptions options = new NodeOptions();
+ options.setSharedElectionTimer(true);
+ options.setSharedVoteTimer(true);
+ return options;
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/core/ReadOnlyServiceTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/ReadOnlyServiceTest.java
new file mode 100644
index 0000000..1de66ce
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/ReadOnlyServiceTest.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.concurrent.CountDownLatch;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.ArgumentMatcher;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.FSMCaller;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.closure.ReadIndexClosure;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.entity.ReadIndexState;
+import com.alipay.sofa.jraft.entity.ReadIndexStatus;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.option.ReadOnlyServiceOptions;
+import com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest;
+import com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse;
+import com.alipay.sofa.jraft.rpc.RpcResponseClosure;
+import com.alipay.sofa.jraft.test.TestUtils;
+import com.alipay.sofa.jraft.util.Bytes;
+import com.alipay.sofa.jraft.util.Utils;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ReadOnlyServiceTest {
+
+ private ReadOnlyServiceImpl readOnlyServiceImpl;
+
+ @Mock
+ private NodeImpl node;
+
+ @Mock
+ private FSMCaller fsmCaller;
+
+ @Before
+ public void setup() {
+ this.readOnlyServiceImpl = new ReadOnlyServiceImpl();
+ final ReadOnlyServiceOptions opts = new ReadOnlyServiceOptions();
+ opts.setFsmCaller(this.fsmCaller);
+ opts.setNode(this.node);
+ opts.setRaftOptions(new RaftOptions());
+ Mockito.when(this.node.getNodeMetrics()).thenReturn(new NodeMetrics(false));
+ Mockito.when(this.node.getGroupId()).thenReturn("test");
+ Mockito.when(this.node.getServerId()).thenReturn(new PeerId("localhost:8081", 0));
+ assertTrue(this.readOnlyServiceImpl.init(opts));
+ }
+
+ @After
+ public void teardown() throws Exception {
+ this.readOnlyServiceImpl.shutdown();
+ this.readOnlyServiceImpl.join();
+ }
+
+ @Test
+ public void testAddRequest() throws Exception {
+ final byte[] requestContext = TestUtils.getRandomBytes();
+ this.readOnlyServiceImpl.addRequest(requestContext, new ReadIndexClosure() {
+
+ @Override
+ public void run(final Status status, final long index, final byte[] reqCtx) {
+
+ }
+ });
+ this.readOnlyServiceImpl.flush();
+ Mockito.verify(this.node).handleReadIndexRequest(Mockito.argThat(new ArgumentMatcher<ReadIndexRequest>() {
+ @Override public boolean matches(ReadIndexRequest argument) {
+ if (argument != null) {
+ final ReadIndexRequest req = (ReadIndexRequest) argument;
+ return req.getGroupId().equals("test") && req.getServerId().equals("localhost:8081:0")
+ && req.getEntriesCount() == 1
+ && Arrays.equals(requestContext, req.getEntries(0).toByteArray());
+ }
+ return false;
+ }
+
+ }), Mockito.any());
+ }
+
+ @Test
+ public void testAddRequestOnResponsePending() throws Exception {
+ final byte[] requestContext = TestUtils.getRandomBytes();
+ final CountDownLatch latch = new CountDownLatch(1);
+ this.readOnlyServiceImpl.addRequest(requestContext, new ReadIndexClosure() {
+
+ @Override
+ public void run(final Status status, final long index, final byte[] reqCtx) {
+ assertTrue(status.isOk());
+ assertEquals(index, 1);
+ assertArrayEquals(reqCtx, requestContext);
+ latch.countDown();
+ }
+ });
+ this.readOnlyServiceImpl.flush();
+
+ final ArgumentCaptor<RpcResponseClosure> closureCaptor = ArgumentCaptor.forClass(RpcResponseClosure.class);
+
+ Mockito.verify(this.node).handleReadIndexRequest(Mockito.argThat(new ArgumentMatcher<ReadIndexRequest>() {
+
+ @Override
+ public boolean matches(final ReadIndexRequest argument) {
+ if (argument != null) {
+ final ReadIndexRequest req = (ReadIndexRequest) argument;
+ return req.getGroupId().equals("test") && req.getServerId().equals("localhost:8081:0")
+ && req.getEntriesCount() == 1
+ && Arrays.equals(requestContext, req.getEntries(0).toByteArray());
+ }
+ return false;
+ }
+
+ }), closureCaptor.capture());
+
+ final RpcResponseClosure closure = closureCaptor.getValue();
+
+ assertNotNull(closure);
+
+ closure.setResponse(ReadIndexResponse.newBuilder().setIndex(1).setSuccess(true).build());
+ assertTrue(this.readOnlyServiceImpl.getPendingNotifyStatus().isEmpty());
+ closure.run(Status.OK());
+ assertEquals(this.readOnlyServiceImpl.getPendingNotifyStatus().size(), 1);
+ this.readOnlyServiceImpl.onApplied(2);
+ latch.await();
+ }
+
+ @Test
+ public void testAddRequestOnResponseFailure() throws Exception {
+ Mockito.when(this.fsmCaller.getLastAppliedIndex()).thenReturn(2L);
+
+ final byte[] requestContext = TestUtils.getRandomBytes();
+ final CountDownLatch latch = new CountDownLatch(1);
+ this.readOnlyServiceImpl.addRequest(requestContext, new ReadIndexClosure() {
+
+ @Override
+ public void run(final Status status, final long index, final byte[] reqCtx) {
+ assertFalse(status.isOk());
+ assertEquals(index, -1);
+ assertArrayEquals(reqCtx, requestContext);
+ latch.countDown();
+ }
+ });
+ this.readOnlyServiceImpl.flush();
+
+ final ArgumentCaptor<RpcResponseClosure> closureCaptor = ArgumentCaptor.forClass(RpcResponseClosure.class);
+
+ Mockito.verify(this.node).handleReadIndexRequest(Mockito.argThat(new ArgumentMatcher<ReadIndexRequest>() {
+
+ @Override
+ public boolean matches(final ReadIndexRequest argument) {
+ if (argument != null) {
+ final ReadIndexRequest req = (ReadIndexRequest) argument;
+ return req.getGroupId().equals("test") && req.getServerId().equals("localhost:8081:0")
+ && req.getEntriesCount() == 1
+ && Arrays.equals(requestContext, req.getEntries(0).toByteArray());
+ }
+ return false;
+ }
+
+ }), closureCaptor.capture());
+
+ final RpcResponseClosure closure = closureCaptor.getValue();
+
+ assertNotNull(closure);
+
+ closure.setResponse(ReadIndexResponse.newBuilder().setIndex(1).setSuccess(true).build());
+ closure.run(new Status(-1, "test"));
+ latch.await();
+ }
+
+ @Test
+ public void testAddRequestOnResponseSuccess() throws Exception {
+
+ Mockito.when(this.fsmCaller.getLastAppliedIndex()).thenReturn(2L);
+
+ final byte[] requestContext = TestUtils.getRandomBytes();
+ final CountDownLatch latch = new CountDownLatch(1);
+ this.readOnlyServiceImpl.addRequest(requestContext, new ReadIndexClosure() {
+
+ @Override
+ public void run(final Status status, final long index, final byte[] reqCtx) {
+ assertTrue(status.isOk());
+ assertEquals(index, 1);
+ assertArrayEquals(reqCtx, requestContext);
+ latch.countDown();
+ }
+ });
+ this.readOnlyServiceImpl.flush();
+
+ final ArgumentCaptor<RpcResponseClosure> closureCaptor = ArgumentCaptor.forClass(RpcResponseClosure.class);
+
+ Mockito.verify(this.node).handleReadIndexRequest(Mockito.argThat(new ArgumentMatcher<ReadIndexRequest>() {
+
+ @Override
+ public boolean matches(final ReadIndexRequest argument) {
+ if (argument != null) {
+ final ReadIndexRequest req = (ReadIndexRequest) argument;
+ return req.getGroupId().equals("test") && req.getServerId().equals("localhost:8081:0")
+ && req.getEntriesCount() == 1
+ && Arrays.equals(requestContext, req.getEntries(0).toByteArray());
+ }
+ return false;
+ }
+
+ }), closureCaptor.capture());
+
+ final RpcResponseClosure closure = closureCaptor.getValue();
+
+ assertNotNull(closure);
+
+ closure.setResponse(ReadIndexResponse.newBuilder().setIndex(1).setSuccess(true).build());
+ closure.run(Status.OK());
+ latch.await();
+ }
+
+ @Test
+ public void testOnApplied() throws Exception {
+ final ArrayList<ReadIndexState> states = new ArrayList<>();
+ final byte[] reqContext = TestUtils.getRandomBytes();
+ final CountDownLatch latch = new CountDownLatch(1);
+ final ReadIndexState state = new ReadIndexState(new Bytes(reqContext), new ReadIndexClosure() {
+
+ @Override
+ public void run(final Status status, final long index, final byte[] reqCtx) {
+ assertTrue(status.isOk());
+ assertEquals(index, 1);
+ assertArrayEquals(reqCtx, reqContext);
+ latch.countDown();
+ }
+ }, Utils.monotonicMs());
+ state.setIndex(1);
+ states.add(state);
+ final ReadIndexStatus readIndexStatus = new ReadIndexStatus(states, null, 1);
+ this.readOnlyServiceImpl.getPendingNotifyStatus().put(1L, Arrays.asList(readIndexStatus));
+
+ this.readOnlyServiceImpl.onApplied(2);
+ latch.await();
+ assertTrue(this.readOnlyServiceImpl.getPendingNotifyStatus().isEmpty());
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/core/ReplicatorGroupTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/ReplicatorGroupTest.java
new file mode 100644
index 0000000..0d92ae2
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/ReplicatorGroupTest.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import com.alipay.sofa.jraft.util.ByteString;
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.runners.MockitoJUnitRunner;
+import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.conf.ConfigurationEntry;
+import com.alipay.sofa.jraft.entity.NodeId;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.option.ReplicatorGroupOptions;
+import com.alipay.sofa.jraft.rpc.RaftClientService;
+import com.alipay.sofa.jraft.rpc.RpcRequests;
+import com.alipay.sofa.jraft.rpc.impl.FutureImpl;
+import com.alipay.sofa.jraft.storage.LogManager;
+import com.alipay.sofa.jraft.storage.SnapshotStorage;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.eq;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class ReplicatorGroupTest {
+
+ static final Logger LOG = LoggerFactory.getLogger(ReplicatorGroupTest.class);
+
+ private TimerManager timerManager;
+ private ReplicatorGroupImpl replicatorGroup;
+ @Mock
+ private BallotBox ballotBox;
+ @Mock
+ private LogManager logManager;
+ @Mock
+ private NodeImpl node;
+ @Mock
+ private RaftClientService rpcService;
+ @Mock
+ private SnapshotStorage snapshotStorage;
+ private final NodeOptions options = new NodeOptions();
+ private final RaftOptions raftOptions = new RaftOptions();
+ private final PeerId peerId1 = new PeerId("localhost", 8082);
+ private final PeerId peerId2 = new PeerId("localhost", 8083);
+ private final PeerId peerId3 = new PeerId("localhost", 8084);
+ private final AtomicInteger errorCounter = new AtomicInteger(0);
+ private final AtomicInteger stoppedCounter = new AtomicInteger(0);
+ private final AtomicInteger startedCounter = new AtomicInteger(0);
+
+ @Before
+ public void setup() {
+ this.timerManager = new TimerManager(5);
+ this.replicatorGroup = new ReplicatorGroupImpl();
+ final ReplicatorGroupOptions rgOpts = new ReplicatorGroupOptions();
+ rgOpts.setHeartbeatTimeoutMs(heartbeatTimeout(this.options.getElectionTimeoutMs()));
+ rgOpts.setElectionTimeoutMs(this.options.getElectionTimeoutMs());
+ rgOpts.setLogManager(this.logManager);
+ rgOpts.setBallotBox(this.ballotBox);
+ rgOpts.setNode(this.node);
+ rgOpts.setRaftRpcClientService(this.rpcService);
+ rgOpts.setSnapshotStorage(this.snapshotStorage);
+ rgOpts.setRaftOptions(this.raftOptions);
+ rgOpts.setTimerManager(this.timerManager);
+ Mockito.when(this.logManager.getLastLogIndex()).thenReturn(10L);
+ Mockito.when(this.logManager.getTerm(10)).thenReturn(1L);
+ Mockito.when(this.node.getNodeMetrics()).thenReturn(new NodeMetrics(false));
+ Mockito.when(this.node.getNodeId()).thenReturn(new NodeId("test", new PeerId("localhost", 8081)));
+ mockSendEmptyEntries();
+ assertTrue(this.replicatorGroup.init(this.node.getNodeId(), rgOpts));
+ }
+
+
+
+ @Test
+ public void testAddReplicatorAndFailed() {
+ this.replicatorGroup.resetTerm(1);
+ assertFalse(this.replicatorGroup.addReplicator(this.peerId1));
+ assertEquals(this.replicatorGroup.getFailureReplicators().get(this.peerId1), ReplicatorType.Follower);
+ }
+
+ @Test
+ public void testAddLearnerFailure() {
+ this.replicatorGroup.resetTerm(1);
+ assertFalse(this.replicatorGroup.addReplicator(this.peerId1, ReplicatorType.Learner));
+ assertEquals(this.replicatorGroup.getFailureReplicators().get(this.peerId1), ReplicatorType.Learner);
+ }
+
+ @Test
+ public void testAddLearnerSuccess() {
+ Mockito.when(this.rpcService.connect(this.peerId1.getEndpoint())).thenReturn(true);
+ this.replicatorGroup.resetTerm(1);
+ assertTrue(this.replicatorGroup.addReplicator(this.peerId1, ReplicatorType.Learner));
+ assertNotNull(this.replicatorGroup.getReplicatorMap().get(this.peerId1));
+ assertNull(this.replicatorGroup.getFailureReplicators().get(this.peerId1));
+ }
+
+ @Test
+ public void testAddReplicatorSuccess() {
+ Mockito.when(this.rpcService.connect(this.peerId1.getEndpoint())).thenReturn(true);
+ this.replicatorGroup.resetTerm(1);
+ assertTrue(this.replicatorGroup.addReplicator(this.peerId1));
+ assertNull(this.replicatorGroup.getFailureReplicators().get(this.peerId1));
+
+ try {
+ Thread.sleep(100000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testStopReplicator() {
+ Mockito.when(this.rpcService.connect(this.peerId1.getEndpoint())).thenReturn(true);
+ this.replicatorGroup.resetTerm(1);
+ this.replicatorGroup.addReplicator(this.peerId1);
+ assertTrue(this.replicatorGroup.stopReplicator(this.peerId1));
+ }
+
+ @Test
+ public void testStopAllReplicator() {
+ Mockito.when(this.rpcService.connect(this.peerId1.getEndpoint())).thenReturn(true);
+ Mockito.when(this.rpcService.connect(this.peerId2.getEndpoint())).thenReturn(true);
+ Mockito.when(this.rpcService.connect(this.peerId3.getEndpoint())).thenReturn(true);
+ this.replicatorGroup.resetTerm(1);
+ this.replicatorGroup.addReplicator(this.peerId1);
+ this.replicatorGroup.addReplicator(this.peerId2);
+ this.replicatorGroup.addReplicator(this.peerId3);
+ assertTrue(this.replicatorGroup.contains(this.peerId1));
+ assertTrue(this.replicatorGroup.contains(this.peerId2));
+ assertTrue(this.replicatorGroup.contains(this.peerId3));
+ assertTrue(this.replicatorGroup.stopAll());
+ }
+
+ @Test
+ public void testReplicatorWithNoRepliactorStateListener() {
+ Mockito.when(this.rpcService.connect(this.peerId1.getEndpoint())).thenReturn(true);
+ Mockito.when(this.rpcService.connect(this.peerId2.getEndpoint())).thenReturn(true);
+ Mockito.when(this.rpcService.connect(this.peerId3.getEndpoint())).thenReturn(true);
+ this.replicatorGroup.resetTerm(1);
+ this.replicatorGroup.addReplicator(this.peerId1);
+ this.replicatorGroup.addReplicator(this.peerId2);
+ this.replicatorGroup.addReplicator(this.peerId3);
+ assertTrue(this.replicatorGroup.stopAll());
+ assertEquals(0, this.startedCounter.get());
+ assertEquals(0, this.errorCounter.get());
+ assertEquals(0, this.stoppedCounter.get());
+
+ }
+
+ class UserReplicatorStateListener implements Replicator.ReplicatorStateListener {
+ @Override
+ public void onCreated(final PeerId peer) {
+ LOG.info("Replicator has created");
+ ReplicatorGroupTest.this.startedCounter.incrementAndGet();
+ }
+
+ @Override
+ public void onError(final PeerId peer, final Status status) {
+ LOG.info("Replicator has errors");
+ ReplicatorGroupTest.this.errorCounter.incrementAndGet();
+ }
+
+ @Override
+ public void onDestroyed(final PeerId peer) {
+ LOG.info("Replicator has been destroyed");
+ ReplicatorGroupTest.this.stoppedCounter.incrementAndGet();
+ }
+ }
+
+ @Test
+ public void testTransferLeadershipToAndStop() {
+ Mockito.when(this.rpcService.connect(this.peerId1.getEndpoint())).thenReturn(true);
+ Mockito.when(this.rpcService.connect(this.peerId2.getEndpoint())).thenReturn(true);
+ Mockito.when(this.rpcService.connect(this.peerId3.getEndpoint())).thenReturn(true);
+ this.replicatorGroup.resetTerm(1);
+ this.replicatorGroup.addReplicator(this.peerId1);
+ this.replicatorGroup.addReplicator(this.peerId2);
+ this.replicatorGroup.addReplicator(this.peerId3);
+ long logIndex = 8;
+ assertTrue(this.replicatorGroup.transferLeadershipTo(this.peerId1, 8));
+ final Replicator r = (Replicator) this.replicatorGroup.getReplicator(this.peerId1).lock();
+ assertEquals(r.getTimeoutNowIndex(), logIndex);
+ this.replicatorGroup.getReplicator(this.peerId1).unlock();
+ assertTrue(this.replicatorGroup.stopTransferLeadership(this.peerId1));
+ assertEquals(r.getTimeoutNowIndex(), 0);
+ }
+
+ @Test
+ public void testFindTheNextCandidateWithPriority1() {
+ final PeerId p1 = new PeerId("localhost", 18881, 0, 60);
+ final PeerId p2 = new PeerId("localhost", 18882, 0, 80);
+ final PeerId p3 = new PeerId("localhost", 18883, 0, 100);
+ Mockito.when(this.rpcService.connect(p1.getEndpoint())).thenReturn(true);
+ Mockito.when(this.rpcService.connect(p2.getEndpoint())).thenReturn(true);
+ Mockito.when(this.rpcService.connect(p3.getEndpoint())).thenReturn(true);
+ this.replicatorGroup.resetTerm(1);
+ this.replicatorGroup.addReplicator(p1);
+ this.replicatorGroup.addReplicator(p2);
+ this.replicatorGroup.addReplicator(p3);
+ final ConfigurationEntry conf = new ConfigurationEntry();
+ conf.setConf(new Configuration(Arrays.asList(p1, p2, p3)));
+ final PeerId p = this.replicatorGroup.findTheNextCandidate(conf);
+ assertEquals(p3, p);
+ }
+
+ @Test
+ public void testFindTheNextCandidateWithPriority2() {
+ final PeerId p1 = new PeerId("localhost", 18881, 0, 0);
+ final PeerId p2 = new PeerId("localhost", 18882, 0, 0);
+ final PeerId p3 = new PeerId("localhost", 18883, 0, -1);
+ Mockito.when(this.rpcService.connect(p1.getEndpoint())).thenReturn(true);
+ Mockito.when(this.rpcService.connect(p2.getEndpoint())).thenReturn(true);
+ Mockito.when(this.rpcService.connect(p3.getEndpoint())).thenReturn(true);
+ this.replicatorGroup.resetTerm(1);
+ this.replicatorGroup.addReplicator(p1);
+ this.replicatorGroup.addReplicator(p2);
+ this.replicatorGroup.addReplicator(p3);
+ final ConfigurationEntry conf = new ConfigurationEntry();
+ conf.setConf(new Configuration(Arrays.asList(p1, p2, p3)));
+ final PeerId p = this.replicatorGroup.findTheNextCandidate(conf);
+ assertEquals(p3, p);
+ }
+
+ @After
+ public void teardown() {
+ this.timerManager.shutdown();
+ this.errorCounter.set(0);
+ this.stoppedCounter.set(0);
+ this.startedCounter.set(0);
+ }
+
+ private int heartbeatTimeout(final int electionTimeout) {
+ return Math.max(electionTimeout / this.raftOptions.getElectionHeartbeatFactor(), 10);
+ }
+
+ private void mockSendEmptyEntries() {
+ final RpcRequests.AppendEntriesRequest request1 = createEmptyEntriesRequestToPeer(this.peerId1);
+ final RpcRequests.AppendEntriesRequest request2 = createEmptyEntriesRequestToPeer(this.peerId2);
+ final RpcRequests.AppendEntriesRequest request3 = createEmptyEntriesRequestToPeer(this.peerId3);
+
+ Mockito
+ .when(this.rpcService.appendEntries(eq(this.peerId1.getEndpoint()), eq(request1), eq(-1), Mockito.any()))
+ .thenAnswer(new Answer<Object>() {
+ @Override public Object answer(InvocationOnMock invocation) throws Throwable {
+ return new FutureImpl<>();
+ }
+ });
+ Mockito
+ .when(this.rpcService.appendEntries(eq(this.peerId2.getEndpoint()), eq(request2), eq(-1), Mockito.any()))
+ .thenReturn(new FutureImpl<>());
+ Mockito
+ .when(this.rpcService.appendEntries(eq(this.peerId3.getEndpoint()), eq(request3), eq(-1), Mockito.any()))
+ .thenReturn(new FutureImpl<>());
+ }
+
+ private RpcRequests.AppendEntriesRequest createEmptyEntriesRequestToPeer(final PeerId peerId) {
+ return RpcRequests.AppendEntriesRequest.newBuilder() //
+ .setGroupId("test") //
+ .setServerId(new PeerId("localhost", 8081).toString()) //
+ .setPeerId(peerId.toString()) //
+ .setTerm(1) //
+ .setPrevLogIndex(10) //
+ .setPrevLogTerm(1) //
+ .setCommittedIndex(0) //
+ .setData(ByteString.EMPTY) //
+ .build();
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/core/ReplicatorTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/ReplicatorTest.java
new file mode 100644
index 0000000..d3fe558
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/ReplicatorTest.java
@@ -0,0 +1,809 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import com.alipay.sofa.jraft.util.ByteString;
+import java.nio.ByteBuffer;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledFuture;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Matchers;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.closure.CatchUpClosure;
+import com.alipay.sofa.jraft.core.Replicator.RequestType;
+import com.alipay.sofa.jraft.entity.EnumOutter;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.entity.LogId;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.entity.RaftOutter;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.error.RaftException;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.option.ReplicatorOptions;
+import com.alipay.sofa.jraft.rpc.RaftClientService;
+import com.alipay.sofa.jraft.rpc.RpcRequests;
+import com.alipay.sofa.jraft.rpc.RpcResponseClosureAdapter;
+import com.alipay.sofa.jraft.rpc.impl.FutureImpl;
+import com.alipay.sofa.jraft.storage.LogManager;
+import com.alipay.sofa.jraft.storage.SnapshotStorage;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.util.ThreadId;
+import com.alipay.sofa.jraft.util.Utils;
+import com.alipay.sofa.jraft.rpc.Message;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.same;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class ReplicatorTest {
+
+ private ThreadId id;
+ private final RaftOptions raftOptions = new RaftOptions();
+ private TimerManager timerManager;
+ @Mock
+ private RaftClientService rpcService;
+ @Mock
+ private NodeImpl node;
+ @Mock
+ private BallotBox ballotBox;
+ @Mock
+ private LogManager logManager;
+ @Mock
+ private SnapshotStorage snapshotStorage;
+ private ReplicatorOptions opts;
+ private final PeerId peerId = new PeerId("localhost", 8081);
+
+ @Before
+ public void setup() {
+ this.timerManager = new TimerManager(5);
+ this.opts = new ReplicatorOptions();
+ this.opts.setRaftRpcService(this.rpcService);
+ this.opts.setPeerId(this.peerId);
+ this.opts.setBallotBox(this.ballotBox);
+ this.opts.setGroupId("test");
+ this.opts.setTerm(1);
+ this.opts.setServerId(new PeerId("localhost", 8082));
+ this.opts.setNode(this.node);
+ this.opts.setSnapshotStorage(this.snapshotStorage);
+ this.opts.setTimerManager(this.timerManager);
+ this.opts.setLogManager(this.logManager);
+ this.opts.setDynamicHeartBeatTimeoutMs(100);
+ this.opts.setElectionTimeoutMs(1000);
+
+ Mockito.when(this.logManager.getLastLogIndex()).thenReturn(10L);
+ Mockito.when(this.logManager.getTerm(10)).thenReturn(1L);
+ Mockito.when(this.rpcService.connect(this.peerId.getEndpoint())).thenReturn(true);
+ Mockito.when(this.node.getNodeMetrics()).thenReturn(new NodeMetrics(true));
+ // mock send empty entries
+ mockSendEmptyEntries();
+
+ this.id = Replicator.start(this.opts, this.raftOptions);
+ }
+
+ private void mockSendEmptyEntries() {
+ this.mockSendEmptyEntries(false);
+ }
+
+ private void mockSendEmptyEntries(final boolean isHeartbeat) {
+ final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest(isHeartbeat);
+ Mockito.when(this.rpcService.appendEntries(eq(this.peerId.getEndpoint()), eq(request), eq(-1), Mockito.any()))
+ .thenReturn(new FutureImpl<>());
+ }
+
+ private RpcRequests.AppendEntriesRequest createEmptyEntriesRequest() {
+ return this.createEmptyEntriesRequest(false);
+ }
+
+ private RpcRequests.AppendEntriesRequest createEmptyEntriesRequest(final boolean isHeartbeat) {
+ RpcRequests.AppendEntriesRequest.Builder rb = RpcRequests.AppendEntriesRequest.newBuilder() //
+ .setGroupId("test") //
+ .setServerId(new PeerId("localhost", 8082).toString()) //
+ .setPeerId(this.peerId.toString()) //
+ .setTerm(1) //
+ .setPrevLogIndex(10) //
+ .setPrevLogTerm(1) //
+ .setCommittedIndex(0);
+ if (!isHeartbeat) {
+ rb.setData(ByteString.EMPTY);
+ }
+ return rb.build();
+ }
+
+ @After
+ public void teardown() {
+ this.timerManager.shutdown();
+ }
+
+ @Test
+ public void testStartDestroyJoin() throws Exception {
+ assertNotNull(this.id);
+ final Replicator r = getReplicator();
+ assertNotNull(r);
+ assertNotNull(r.getRpcInFly());
+ assertEquals(r.statInfo.runningState, Replicator.RunningState.APPENDING_ENTRIES);
+ assertSame(r.getOpts(), this.opts);
+ this.id.unlock();
+ assertEquals(0, Replicator.getNextIndex(this.id));
+ assertNotNull(r.getHeartbeatTimer());
+ r.destroy();
+ Replicator.join(this.id);
+ assertNull(r.id);
+ }
+
+ @Test
+ public void testMetricRemoveOnDestroy() {
+ assertNotNull(this.id);
+ final Replicator r = getReplicator();
+ assertNotNull(r);
+ assertSame(r.getOpts(), this.opts);
+ Set<String> metrics = this.opts.getNode().getNodeMetrics().getMetricRegistry().getNames();
+ assertEquals(6, metrics.size());
+ r.destroy();
+ metrics = this.opts.getNode().getNodeMetrics().getMetricRegistry().getNames();
+ assertEquals(1, metrics.size());
+ }
+
+ private Replicator getReplicator() {
+ return (Replicator) this.id.lock();
+ }
+
+ @Test
+ public void testOnRpcReturnedRpcError() {
+ testRpcReturnedError();
+ }
+
+ private Replicator testRpcReturnedError() {
+ final Replicator r = getReplicator();
+ assertNull(r.getBlockTimer());
+ final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
+ final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder() //
+ .setSuccess(false) //
+ .setLastLogIndex(12) //
+ .setTerm(2) //
+ .build();
+ this.id.unlock();
+
+ Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, new Status(-1, "test error"), request,
+ response, 0, 0, Utils.monotonicMs());
+ assertEquals(r.statInfo.runningState, Replicator.RunningState.BLOCKING);
+ assertNotNull(r.getBlockTimer());
+ return r;
+ }
+
+ @Test
+ public void testOnRpcReturnedRpcContinuousError() throws Exception {
+ Replicator r = testRpcReturnedError();
+ ScheduledFuture<?> timer = r.getBlockTimer();
+ assertNotNull(timer);
+
+ final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
+ final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder() //
+ .setSuccess(false) //
+ .setLastLogIndex(12) //
+ .setTerm(2) //
+ .build();
+ r.getInflights().add(new Replicator.Inflight(RequestType.AppendEntries, r.getNextSendIndex(), 0, 0, 1, null));
+ Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, new Status(-1, "test error"), request,
+ response, 1, 1, Utils.monotonicMs());
+ assertEquals(r.statInfo.runningState, Replicator.RunningState.BLOCKING);
+ assertNotNull(r.getBlockTimer());
+ // the same timer
+ assertSame(timer, r.getBlockTimer());
+
+ Thread.sleep(r.getOpts().getDynamicHeartBeatTimeoutMs() * 2);
+ r.getInflights().add(new Replicator.Inflight(RequestType.AppendEntries, r.getNextSendIndex(), 0, 0, 1, null));
+ Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, new Status(-1, "test error"), request,
+ response, 1, 2, Utils.monotonicMs());
+ assertEquals(r.statInfo.runningState, Replicator.RunningState.BLOCKING);
+ assertNotNull(r.getBlockTimer());
+ // the same timer
+ assertNotSame(timer, r.getBlockTimer());
+ }
+
+ @Test
+ public void testOnRpcReturnedTermMismatch() {
+ final Replicator r = getReplicator();
+ final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
+ final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder() //
+ .setSuccess(false) //
+ .setLastLogIndex(12) //
+ .setTerm(2) //
+ .build();
+ this.id.unlock();
+
+ Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, Status.OK(), request, response, 0, 0,
+ Utils.monotonicMs());
+ Mockito.verify(this.node).increaseTermTo(
+ 2,
+ new Status(RaftError.EHIGHERTERMRESPONSE, "Leader receives higher term heartbeat_response from peer:%s",
+ this.peerId));
+ assertNull(r.id);
+ }
+
+ @Test
+ public void testOnRpcReturnedMoreLogs() {
+ final Replicator r = getReplicator();
+ assertEquals(11, r.getRealNextIndex());
+ final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
+ final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder() //
+ .setSuccess(false) //
+ .setLastLogIndex(12) //
+ .setTerm(1) //
+ .build();
+ this.id.unlock();
+ final Future<Message> rpcInFly = r.getRpcInFly();
+ assertNotNull(rpcInFly);
+
+ Mockito.when(this.logManager.getTerm(9)).thenReturn(1L);
+ final RpcRequests.AppendEntriesRequest newReq = RpcRequests.AppendEntriesRequest.newBuilder(). //
+ setGroupId("test"). //
+ setServerId(new PeerId("localhost", 8082).toString()). //
+ setPeerId(this.peerId.toString()). //
+ setTerm(1). //
+ setPrevLogIndex(9). //
+ setData(ByteString.EMPTY). //
+ setPrevLogTerm(1). //
+ setCommittedIndex(0).build();
+ Mockito.when(this.rpcService.appendEntries(eq(this.peerId.getEndpoint()), eq(newReq), eq(-1), Mockito.any()))
+ .thenReturn(new FutureImpl<>());
+
+ Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, Status.OK(), request, response, 0, 0,
+ Utils.monotonicMs());
+
+ assertNotNull(r.getRpcInFly());
+ assertNotSame(r.getRpcInFly(), rpcInFly);
+ assertEquals(r.statInfo.runningState, Replicator.RunningState.APPENDING_ENTRIES);
+ this.id.unlock();
+ assertEquals(0, Replicator.getNextIndex(this.id));
+ assertEquals(10, r.getRealNextIndex());
+ }
+
+ @Test
+ public void testOnRpcReturnedLessLogs() {
+ final Replicator r = getReplicator();
+ assertEquals(11, r.getRealNextIndex());
+ final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
+ final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder() //
+ .setSuccess(false) //
+ .setLastLogIndex(8) //
+ .setTerm(1) //
+ .build();
+ this.id.unlock();
+ final Future<Message> rpcInFly = r.getRpcInFly();
+ assertNotNull(rpcInFly);
+
+ Mockito.when(this.logManager.getTerm(8)).thenReturn(1L);
+ final RpcRequests.AppendEntriesRequest newReq = RpcRequests.AppendEntriesRequest.newBuilder() //
+ .setGroupId("test") //
+ .setServerId(new PeerId("localhost", 8082).toString()) //
+ .setPeerId(this.peerId.toString()) //
+ .setTerm(1) //
+ .setPrevLogIndex(8) //
+ .setPrevLogTerm(1) //
+ .setData(ByteString.EMPTY) //
+ .setCommittedIndex(0) //
+ .build();
+ Mockito.when(this.rpcService.appendEntries(eq(this.peerId.getEndpoint()), eq(newReq), eq(-1), Mockito.any()))
+ .thenReturn(new FutureImpl<>());
+
+ Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, Status.OK(), request, response, 0, 0,
+ Utils.monotonicMs());
+
+ assertNotNull(r.getRpcInFly());
+ assertNotSame(r.getRpcInFly(), rpcInFly);
+ assertEquals(r.statInfo.runningState, Replicator.RunningState.APPENDING_ENTRIES);
+ this.id.unlock();
+ assertEquals(0, Replicator.getNextIndex(this.id));
+ assertEquals(9, r.getRealNextIndex());
+ }
+
+ @Test
+ public void testOnRpcReturnedWaitMoreEntries() throws Exception {
+ final Replicator r = getReplicator();
+ assertEquals(-1, r.getWaitId());
+
+ final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
+ final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder() //
+ .setSuccess(true) //
+ .setLastLogIndex(10) //
+ .setTerm(1) //
+ .build();
+ this.id.unlock();
+ Mockito.when(this.logManager.wait(eq(10L), Mockito.any(), same(this.id))).thenReturn(99L);
+
+ final CountDownLatch latch = new CountDownLatch(1);
+ Replicator.waitForCaughtUp(this.id, 1, System.currentTimeMillis() + 5000, new CatchUpClosure() {
+
+ @Override
+ public void run(final Status status) {
+ assertTrue(status.isOk());
+ latch.countDown();
+ }
+ });
+
+ Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, Status.OK(), request, response, 0, 0,
+ Utils.monotonicMs());
+
+ assertEquals(r.statInfo.runningState, Replicator.RunningState.IDLE);
+ this.id.unlock();
+ assertEquals(11, Replicator.getNextIndex(this.id));
+ assertEquals(99, r.getWaitId());
+ latch.await(); //make sure catch up closure is invoked.
+ }
+
+ @Test
+ public void testStop() {
+ final Replicator r = getReplicator();
+ this.id.unlock();
+ assertNotNull(r.getHeartbeatTimer());
+ assertNotNull(r.getRpcInFly());
+ Replicator.stop(this.id);
+ assertNull(r.id);
+ assertNull(r.getHeartbeatTimer());
+ assertNull(r.getRpcInFly());
+ }
+
+ @Test
+ public void testSetErrorStop() {
+ final Replicator r = getReplicator();
+ this.id.unlock();
+ assertNotNull(r.getHeartbeatTimer());
+ assertNotNull(r.getRpcInFly());
+ this.id.setError(RaftError.ESTOP.getNumber());
+ this.id.unlock();
+ assertNull(r.id);
+ assertNull(r.getHeartbeatTimer());
+ assertNull(r.getRpcInFly());
+ }
+
+ @Test
+ public void testContinueSendingTimeout() throws Exception {
+ testOnRpcReturnedWaitMoreEntries();
+ final Replicator r = getReplicator();
+ this.id.unlock();
+ mockSendEmptyEntries();
+ final Future<Message> rpcInFly = r.getRpcInFly();
+ assertNotNull(rpcInFly);
+ assertTrue(Replicator.continueSending(this.id, RaftError.ETIMEDOUT.getNumber()));
+ assertNotNull(r.getRpcInFly());
+ assertNotSame(rpcInFly, r.getRpcInFly());
+ }
+
+ @Test
+ public void testContinueSendingEntries() throws Exception {
+ testOnRpcReturnedWaitMoreEntries();
+ final Replicator r = getReplicator();
+ this.id.unlock();
+ mockSendEmptyEntries();
+ final Future<Message> rpcInFly = r.getRpcInFly();
+ assertNotNull(rpcInFly);
+
+ final RpcRequests.AppendEntriesRequest.Builder rb = RpcRequests.AppendEntriesRequest.newBuilder() //
+ .setGroupId("test") //
+ .setServerId(new PeerId("localhost", 8082).toString()) //
+ .setPeerId(this.peerId.toString()) //
+ .setTerm(1) //
+ .setPrevLogIndex(10) //
+ .setPrevLogTerm(1) //
+ .setCommittedIndex(0);
+
+ int totalDataLen = 0;
+ for (int i = 0; i < 10; i++) {
+ totalDataLen += i;
+ final LogEntry value = new LogEntry();
+ value.setData(ByteBuffer.allocate(i));
+ value.setType(EnumOutter.EntryType.ENTRY_TYPE_DATA);
+ value.setId(new LogId(11 + i, 1));
+ Mockito.when(this.logManager.getEntry(11 + i)).thenReturn(value);
+ rb.addEntries(RaftOutter.EntryMeta.newBuilder().setTerm(1).setType(EnumOutter.EntryType.ENTRY_TYPE_DATA)
+ .setDataLen(i).build());
+ }
+ rb.setData(new ByteString(new byte[totalDataLen]));
+
+ final RpcRequests.AppendEntriesRequest request = rb.build();
+ Mockito.when(this.rpcService.appendEntries(eq(this.peerId.getEndpoint()), eq(request), eq(-1), Mockito.any()))
+ .thenReturn(new FutureImpl<>());
+
+ assertEquals(11, r.statInfo.firstLogIndex);
+ assertEquals(10, r.statInfo.lastLogIndex);
+ Mockito.when(this.logManager.getTerm(20)).thenReturn(1L);
+ assertTrue(Replicator.continueSending(this.id, 0));
+ assertNotNull(r.getRpcInFly());
+ assertNotSame(rpcInFly, r.getRpcInFly());
+ assertEquals(11, r.statInfo.firstLogIndex);
+ assertEquals(20, r.statInfo.lastLogIndex);
+ assertEquals(0, r.getWaitId());
+ assertEquals(r.statInfo.runningState, Replicator.RunningState.IDLE);
+ }
+
+ @Test
+ public void testSetErrorTimeout() throws Exception {
+ final Replicator r = getReplicator();
+ this.id.unlock();
+ assertNull(r.getHeartbeatInFly());
+ final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest(true);
+ Mockito.when(
+ this.rpcService.appendEntries(eq(this.peerId.getEndpoint()), eq(request),
+ eq(this.opts.getElectionTimeoutMs() / 2), Mockito.any())).thenReturn(new FutureImpl<>());
+ this.id.setError(RaftError.ETIMEDOUT.getNumber());
+ Thread.sleep(this.opts.getElectionTimeoutMs() + 1000);
+ assertNotNull(r.getHeartbeatInFly());
+ }
+
+ @Test
+ public void testOnHeartbeatReturnedRpcError() {
+ final Replicator r = getReplicator();
+ this.id.unlock();
+ final ScheduledFuture<?> timer = r.getHeartbeatTimer();
+ assertNotNull(timer);
+ Replicator.onHeartbeatReturned(this.id, new Status(-1, "test"), createEmptyEntriesRequest(), null,
+ Utils.monotonicMs());
+ assertNotNull(r.getHeartbeatTimer());
+ assertNotSame(timer, r.getHeartbeatTimer());
+ }
+
+ @Test
+ public void testOnHeartbeatReturnedOK() {
+ final Replicator r = getReplicator();
+ this.id.unlock();
+ final ScheduledFuture<?> timer = r.getHeartbeatTimer();
+ assertNotNull(timer);
+ final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder(). //
+ setSuccess(false). //
+ setLastLogIndex(10).setTerm(1).build();
+ Replicator
+ .onHeartbeatReturned(this.id, Status.OK(), createEmptyEntriesRequest(), response, Utils.monotonicMs());
+ assertNotNull(r.getHeartbeatTimer());
+ assertNotSame(timer, r.getHeartbeatTimer());
+ }
+
+ @Test
+ public void testOnHeartbeatReturnedTermMismatch() {
+ final Replicator r = getReplicator();
+ final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
+ final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder() //
+ .setSuccess(false) //
+ .setLastLogIndex(12) //
+ .setTerm(2) //
+ .build();
+ this.id.unlock();
+
+ Replicator.onHeartbeatReturned(this.id, Status.OK(), request, response, Utils.monotonicMs());
+ Mockito.verify(this.node).increaseTermTo(
+ 2,
+ new Status(RaftError.EHIGHERTERMRESPONSE, "Leader receives higher term heartbeat_response from peer:%s",
+ this.peerId));
+ assertNull(r.id);
+ }
+
+ @Test
+ public void testTransferLeadership() {
+ final Replicator r = getReplicator();
+ this.id.unlock();
+ assertEquals(0, r.getTimeoutNowIndex());
+ assertTrue(Replicator.transferLeadership(this.id, 11));
+ assertEquals(11, r.getTimeoutNowIndex());
+ assertNull(r.getTimeoutNowInFly());
+ }
+
+ @Test
+ public void testStopTransferLeadership() {
+ testTransferLeadership();
+ Replicator.stopTransferLeadership(this.id);
+ final Replicator r = getReplicator();
+ this.id.unlock();
+ assertEquals(0, r.getTimeoutNowIndex());
+ assertNull(r.getTimeoutNowInFly());
+ }
+
+ @Test
+ public void testTransferLeadershipSendTimeoutNow() {
+ final Replicator r = getReplicator();
+ this.id.unlock();
+ r.setHasSucceeded();
+ assertEquals(0, r.getTimeoutNowIndex());
+ assertNull(r.getTimeoutNowInFly());
+
+ final RpcRequests.TimeoutNowRequest request = createTimeoutnowRequest();
+ Mockito.when(
+ this.rpcService.timeoutNow(Matchers.eq(this.opts.getPeerId().getEndpoint()), eq(request), eq(-1),
+ Mockito.any())).thenReturn(new FutureImpl<>());
+
+ assertTrue(Replicator.transferLeadership(this.id, 10));
+ assertEquals(0, r.getTimeoutNowIndex());
+ assertNotNull(r.getTimeoutNowInFly());
+ }
+
+ @Test
+ public void testSendHeartbeat() {
+ final Replicator r = getReplicator();
+ this.id.unlock();
+
+ assertNull(r.getHeartbeatInFly());
+ final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest(true);
+ Mockito.when(
+ this.rpcService.appendEntries(eq(this.peerId.getEndpoint()), eq(request),
+ eq(this.opts.getElectionTimeoutMs() / 2), Mockito.any())).thenReturn(new FutureImpl<>());
+ Replicator.sendHeartbeat(this.id, new RpcResponseClosureAdapter<RpcRequests.AppendEntriesResponse>() {
+
+ @Override
+ public void run(final Status status) {
+ assertTrue(status.isOk());
+
+ }
+ });
+
+ assertNotNull(r.getHeartbeatInFly());
+
+ assertSame(r, this.id.lock());
+ this.id.unlock();
+ }
+
+ @Test
+ public void testSendTimeoutNowAndStop() {
+ final Replicator r = getReplicator();
+ this.id.unlock();
+ r.setHasSucceeded();
+ assertEquals(0, r.getTimeoutNowIndex());
+ assertNull(r.getTimeoutNowInFly());
+ assertTrue(Replicator.sendTimeoutNowAndStop(this.id, 10));
+ assertEquals(0, r.getTimeoutNowIndex());
+ assertNull(r.getTimeoutNowInFly());
+ final RpcRequests.TimeoutNowRequest request = createTimeoutnowRequest();
+ Mockito.verify(this.rpcService).timeoutNow(Matchers.eq(this.opts.getPeerId().getEndpoint()), eq(request),
+ eq(10), Mockito.any());
+ }
+
+ private RpcRequests.TimeoutNowRequest createTimeoutnowRequest() {
+ final RpcRequests.TimeoutNowRequest.Builder rb = RpcRequests.TimeoutNowRequest.newBuilder();
+ rb.setTerm(this.opts.getTerm());
+ rb.setGroupId(this.opts.getGroupId());
+ rb.setServerId(this.opts.getServerId().toString());
+ rb.setPeerId(this.opts.getPeerId().toString());
+ return rb.build();
+ }
+
+ @Test
+ public void testOnTimeoutNowReturnedRpcErrorAndStop() {
+ final Replicator r = getReplicator();
+ final RpcRequests.TimeoutNowRequest request = createTimeoutnowRequest();
+ this.id.unlock();
+
+ Replicator.onTimeoutNowReturned(this.id, new Status(-1, "test"), request, null, true);
+ assertNull(r.id);
+ }
+
+ @Test
+ public void testInstallSnapshotNoReader() {
+ final Replicator r = getReplicator();
+ this.id.unlock();
+
+ final Future<Message> rpcInFly = r.getRpcInFly();
+ assertNotNull(rpcInFly);
+ r.installSnapshot();
+ final ArgumentCaptor<RaftException> errArg = ArgumentCaptor.forClass(RaftException.class);
+ Mockito.verify(this.node).onError(errArg.capture());
+ Assert.assertEquals(RaftError.EIO, errArg.getValue().getStatus().getRaftError());
+ Assert.assertEquals("Fail to open snapshot", errArg.getValue().getStatus().getErrorMsg());
+ }
+
+ @Test
+ public void testInstallSnapshot() {
+ final Replicator r = getReplicator();
+ this.id.unlock();
+
+ final Future<Message> rpcInFly = r.getRpcInFly();
+ assertNotNull(rpcInFly);
+ final SnapshotReader reader = Mockito.mock(SnapshotReader.class);
+ Mockito.when(this.snapshotStorage.open()).thenReturn(reader);
+ final String uri = "remote://localhost:8081/99";
+ Mockito.when(reader.generateURIForCopy()).thenReturn(uri);
+ final RaftOutter.SnapshotMeta meta = RaftOutter.SnapshotMeta.newBuilder() //
+ .setLastIncludedIndex(11) //
+ .setLastIncludedTerm(1) //
+ .build();
+ Mockito.when(reader.load()).thenReturn(meta);
+
+ assertEquals(0, r.statInfo.lastLogIncluded);
+ assertEquals(0, r.statInfo.lastTermIncluded);
+
+ final RpcRequests.InstallSnapshotRequest.Builder rb = RpcRequests.InstallSnapshotRequest.newBuilder();
+ rb.setTerm(this.opts.getTerm());
+ rb.setGroupId(this.opts.getGroupId());
+ rb.setServerId(this.opts.getServerId().toString());
+ rb.setPeerId(this.opts.getPeerId().toString());
+ rb.setMeta(meta);
+ rb.setUri(uri);
+
+ Mockito.when(
+ this.rpcService.installSnapshot(Matchers.eq(this.opts.getPeerId().getEndpoint()), eq(rb.build()),
+ Mockito.any())).thenReturn(new FutureImpl<>());
+
+ r.installSnapshot();
+ assertNotNull(r.getRpcInFly());
+ assertNotSame(r.getRpcInFly(), rpcInFly);
+ Assert.assertEquals(Replicator.RunningState.INSTALLING_SNAPSHOT, r.statInfo.runningState);
+ assertEquals(11, r.statInfo.lastLogIncluded);
+ assertEquals(1, r.statInfo.lastTermIncluded);
+ }
+
+ @Test
+ public void testOnTimeoutNowReturnedTermMismatch() {
+ final Replicator r = getReplicator();
+ this.id.unlock();
+ final RpcRequests.TimeoutNowRequest request = createTimeoutnowRequest();
+ final RpcRequests.TimeoutNowResponse response = RpcRequests.TimeoutNowResponse.newBuilder() //
+ .setSuccess(false) //
+ .setTerm(12) //
+ .build();
+ this.id.unlock();
+
+ Replicator.onTimeoutNowReturned(this.id, Status.OK(), request, response, false);
+ Mockito.verify(this.node).increaseTermTo(
+ 12,
+ new Status(RaftError.EHIGHERTERMRESPONSE, "Leader receives higher term timeout_now_response from peer:%s",
+ this.peerId));
+ assertNull(r.id);
+ }
+
+ @Test
+ public void testOnInstallSnapshotReturned() {
+ final Replicator r = getReplicator();
+ this.id.unlock();
+ assertNull(r.getBlockTimer());
+
+ final RpcRequests.InstallSnapshotRequest request = createInstallSnapshotRequest();
+ final RpcRequests.InstallSnapshotResponse response = RpcRequests.InstallSnapshotResponse.newBuilder()
+ .setSuccess(true).setTerm(1).build();
+ assertEquals(-1, r.getWaitId());
+ Mockito.when(this.logManager.getTerm(11)).thenReturn(1L);
+ Replicator.onRpcReturned(this.id, Replicator.RequestType.Snapshot, Status.OK(), request, response, 0, 0, -1);
+ assertNull(r.getBlockTimer());
+ assertEquals(0, r.getWaitId());
+ }
+
+ @Test
+ public void testOnInstallSnapshotReturnedRpcError() {
+ final Replicator r = getReplicator();
+ this.id.unlock();
+ assertNull(r.getBlockTimer());
+
+ final RpcRequests.InstallSnapshotRequest request = createInstallSnapshotRequest();
+ final RpcRequests.InstallSnapshotResponse response = RpcRequests.InstallSnapshotResponse.newBuilder()
+ .setSuccess(true).setTerm(1).build();
+ assertEquals(-1, r.getWaitId());
+ Mockito.when(this.logManager.getTerm(11)).thenReturn(1L);
+ Replicator.onRpcReturned(this.id, Replicator.RequestType.Snapshot, new Status(-1, "test"), request, response,
+ 0, 0, -1);
+ assertNotNull(r.getBlockTimer());
+ assertEquals(-1, r.getWaitId());
+ }
+
+ @Test
+ public void testOnInstallSnapshotReturnedFailure() {
+ final Replicator r = getReplicator();
+ this.id.unlock();
+ assertNull(r.getBlockTimer());
+
+ final RpcRequests.InstallSnapshotRequest request = createInstallSnapshotRequest();
+ final RpcRequests.InstallSnapshotResponse response = RpcRequests.InstallSnapshotResponse.newBuilder()
+ .setSuccess(false).setTerm(1).build();
+ assertEquals(-1, r.getWaitId());
+ Mockito.when(this.logManager.getTerm(11)).thenReturn(1L);
+ Replicator.onRpcReturned(this.id, Replicator.RequestType.Snapshot, Status.OK(), request, response, 0, 0, -1);
+ assertNotNull(r.getBlockTimer());
+ assertEquals(-1, r.getWaitId());
+ }
+
+ @Test
+ public void testOnRpcReturnedOutOfOrder() {
+ final Replicator r = getReplicator();
+ assertEquals(-1, r.getWaitId());
+
+ final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
+ final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder(). //
+ setSuccess(true). //
+ setLastLogIndex(10).setTerm(1).build();
+ assertNull(r.getBlockTimer());
+ this.id.unlock();
+
+ assertTrue(r.getPendingResponses().isEmpty());
+ Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, Status.OK(), request, response, 1, 0,
+ Utils.monotonicMs());
+ assertEquals(1, r.getPendingResponses().size());
+ Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, Status.OK(), request, response, 0, 0,
+ Utils.monotonicMs());
+ assertTrue(r.getPendingResponses().isEmpty());
+ assertEquals(0, r.getWaitId());
+ assertEquals(11, r.getRealNextIndex());
+ assertEquals(1, r.getRequiredNextSeq());
+ }
+
+ private void mockSendEntries(@SuppressWarnings("SameParameterValue") final int n) {
+ final RpcRequests.AppendEntriesRequest request = createEntriesRequest(n);
+ Mockito.when(this.rpcService.appendEntries(eq(this.peerId.getEndpoint()), eq(request), eq(-1), Mockito.any()))
+ .thenReturn(new FutureImpl<>());
+ }
+
+ private RpcRequests.AppendEntriesRequest createEntriesRequest(final int n) {
+ final RpcRequests.AppendEntriesRequest.Builder rb = RpcRequests.AppendEntriesRequest.newBuilder() //
+ .setGroupId("test") //
+ .setServerId(new PeerId("localhost", 8082).toString()) //
+ .setPeerId(this.peerId.toString()) //
+ .setTerm(1) //
+ .setPrevLogIndex(10) //
+ .setPrevLogTerm(1) //
+ .setCommittedIndex(0);
+
+ for (int i = 0; i < n; i++) {
+ final LogEntry log = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_DATA);
+ log.setData(ByteBuffer.wrap(new byte[i]));
+ log.setId(new LogId(i + 11, 1));
+ Mockito.when(this.logManager.getEntry(i + 11)).thenReturn(log);
+ Mockito.when(this.logManager.getTerm(i + 11)).thenReturn(1L);
+ rb.addEntries(RaftOutter.EntryMeta.newBuilder().setDataLen(i).setTerm(1)
+ .setType(EnumOutter.EntryType.ENTRY_TYPE_DATA).build());
+ }
+
+ return rb.build();
+ }
+
+ @Test
+ public void testGetNextSendIndex() {
+ final Replicator r = getReplicator();
+ assertEquals(-1, r.getNextSendIndex());
+ r.resetInflights();
+ assertEquals(11, r.getNextSendIndex());
+ mockSendEntries(3);
+ r.sendEntries();
+ assertEquals(14, r.getNextSendIndex());
+ }
+
+ private RpcRequests.InstallSnapshotRequest createInstallSnapshotRequest() {
+ final String uri = "remote://localhost:8081/99";
+ final RaftOutter.SnapshotMeta meta = RaftOutter.SnapshotMeta.newBuilder() //
+ .setLastIncludedIndex(11) //
+ .setLastIncludedTerm(1) //
+ .build();
+ final RpcRequests.InstallSnapshotRequest.Builder rb = RpcRequests.InstallSnapshotRequest.newBuilder();
+ rb.setTerm(this.opts.getTerm());
+ rb.setGroupId(this.opts.getGroupId());
+ rb.setServerId(this.opts.getServerId().toString());
+ rb.setPeerId(this.opts.getPeerId().toString());
+ rb.setMeta(meta);
+ rb.setUri(uri);
+ return rb.build();
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/core/TestCluster.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/TestCluster.java
new file mode 100644
index 0000000..aa1bf5e
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/TestCluster.java
@@ -0,0 +1,494 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.IdentityHashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.FileUtils;
+
+import com.alipay.sofa.jraft.JRaftServiceFactory;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.RaftGroupService;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import com.alipay.sofa.jraft.storage.SnapshotThrottle;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ * Test cluster for NodeTest
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-20 1:41:17 PM
+ */
+public class TestCluster {
+
+ static class Clusters {
+
+ public final IdentityHashMap<TestCluster, Object> needCloses = new IdentityHashMap<>();
+ private final Object EXIST = new Object();
+
+ public synchronized void add(final TestCluster cluster) {
+ this.needCloses.put(cluster, EXIST);
+ }
+
+ public synchronized boolean remove(final TestCluster cluster) {
+ return this.needCloses.remove(cluster) != null;
+ }
+
+ public synchronized boolean isEmpty() {
+ return this.needCloses.isEmpty();
+ }
+
+ public synchronized List<TestCluster> removeAll() {
+ final List<TestCluster> clusters = new ArrayList<>(this.needCloses.keySet());
+ this.needCloses.clear();
+ return clusters;
+ }
+ }
+
+ public static final Clusters CLUSTERS = new Clusters();
+
+ private final String dataPath;
+ private final String name; // groupId
+ private final List<PeerId> peers;
+ private final List<NodeImpl> nodes;
+ private final LinkedHashMap<PeerId, MockStateMachine> fsms;
+ private final ConcurrentMap<String, RaftGroupService> serverMap = new ConcurrentHashMap<>();
+ private final int electionTimeoutMs;
+ private final Lock lock = new ReentrantLock();
+
+ private JRaftServiceFactory raftServiceFactory = new TestJRaftServiceFactory();
+
+ private LinkedHashSet<PeerId> learners;
+
+ public JRaftServiceFactory getRaftServiceFactory() {
+ return this.raftServiceFactory;
+ }
+
+ public void setRaftServiceFactory(final JRaftServiceFactory raftServiceFactory) {
+ this.raftServiceFactory = raftServiceFactory;
+ }
+
+ public LinkedHashSet<PeerId> getLearners() {
+ return this.learners;
+ }
+
+ public void setLearners(final LinkedHashSet<PeerId> learners) {
+ this.learners = learners;
+ }
+
+ public List<PeerId> getPeers() {
+ return this.peers;
+ }
+
+ public TestCluster(final String name, final String dataPath, final List<PeerId> peers) {
+ this(name, dataPath, peers, 300);
+ }
+
+ public TestCluster(final String name, final String dataPath, final List<PeerId> peers, final int electionTimeoutMs) {
+ this(name, dataPath, peers, new LinkedHashSet<>(), 300);
+ }
+
+ public TestCluster(final String name, final String dataPath, final List<PeerId> peers,
+ final LinkedHashSet<PeerId> learners, final int electionTimeoutMs) {
+ super();
+ this.name = name;
+ this.dataPath = dataPath;
+ this.peers = peers;
+ this.nodes = new ArrayList<>(this.peers.size());
+ this.fsms = new LinkedHashMap<>(this.peers.size());
+ this.electionTimeoutMs = electionTimeoutMs;
+ this.learners = learners;
+ CLUSTERS.add(this);
+ }
+
+ public boolean start(final Endpoint addr) throws Exception {
+ return this.start(addr, false, 300);
+ }
+
+ public boolean start(final Endpoint addr, final int priority) throws Exception {
+ return this.start(addr, false, 300, false, null, null, priority);
+ }
+
+ public boolean startLearner(final PeerId peer) throws Exception {
+ this.learners.add(peer);
+ return this.start(peer.getEndpoint(), false, 300);
+ }
+
+ public boolean start(final Endpoint listenAddr, final boolean emptyPeers, final int snapshotIntervalSecs)
+ throws IOException {
+ return this.start(listenAddr, emptyPeers, snapshotIntervalSecs, false);
+ }
+
+ public boolean start(final Endpoint listenAddr, final boolean emptyPeers, final int snapshotIntervalSecs,
+ final boolean enableMetrics) throws IOException {
+ return this.start(listenAddr, emptyPeers, snapshotIntervalSecs, enableMetrics, null, null);
+ }
+
+ public boolean start(final Endpoint listenAddr, final boolean emptyPeers, final int snapshotIntervalSecs,
+ final boolean enableMetrics, final SnapshotThrottle snapshotThrottle) throws IOException {
+ return this.start(listenAddr, emptyPeers, snapshotIntervalSecs, enableMetrics, snapshotThrottle, null);
+ }
+
+ public boolean start(final Endpoint listenAddr, final boolean emptyPeers, final int snapshotIntervalSecs,
+ final boolean enableMetrics, final SnapshotThrottle snapshotThrottle,
+ final RaftOptions raftOptions, final int priority) throws IOException {
+
+ if (this.serverMap.get(listenAddr.toString()) != null) {
+ return true;
+ }
+
+ final NodeOptions nodeOptions = new NodeOptions();
+ nodeOptions.setElectionTimeoutMs(this.electionTimeoutMs);
+ nodeOptions.setEnableMetrics(enableMetrics);
+ nodeOptions.setSnapshotThrottle(snapshotThrottle);
+ nodeOptions.setSnapshotIntervalSecs(snapshotIntervalSecs);
+ nodeOptions.setServiceFactory(this.raftServiceFactory);
+ if (raftOptions != null) {
+ nodeOptions.setRaftOptions(raftOptions);
+ }
+ final String serverDataPath = this.dataPath + File.separator + listenAddr.toString().replace(':', '_');
+ FileUtils.forceMkdir(new File(serverDataPath));
+ nodeOptions.setLogUri(serverDataPath + File.separator + "logs");
+ nodeOptions.setRaftMetaUri(serverDataPath + File.separator + "meta");
+ nodeOptions.setSnapshotUri(serverDataPath + File.separator + "snapshot");
+ nodeOptions.setElectionPriority(priority);
+
+ final MockStateMachine fsm = new MockStateMachine(listenAddr);
+ nodeOptions.setFsm(fsm);
+
+ if (!emptyPeers) {
+ nodeOptions.setInitialConf(new Configuration(this.peers, this.learners));
+ }
+
+ final RpcServer rpcServer = RaftRpcServerFactory.createRaftRpcServer(listenAddr);
+ final RaftGroupService server = new RaftGroupService(this.name, new PeerId(listenAddr, 0, priority),
+ nodeOptions, rpcServer);
+
+ this.lock.lock();
+ try {
+ if (this.serverMap.put(listenAddr.toString(), server) == null) {
+ final Node node = server.start();
+
+ this.fsms.put(new PeerId(listenAddr, 0), fsm);
+ this.nodes.add((NodeImpl) node);
+ return true;
+ }
+ } finally {
+ this.lock.unlock();
+ }
+ return false;
+ }
+
+ public boolean start(final Endpoint listenAddr, final boolean emptyPeers, final int snapshotIntervalSecs,
+ final boolean enableMetrics, final SnapshotThrottle snapshotThrottle,
+ final RaftOptions raftOptions) throws IOException {
+
+ if (this.serverMap.get(listenAddr.toString()) != null) {
+ return true;
+ }
+
+ final NodeOptions nodeOptions = new NodeOptions();
+ nodeOptions.setElectionTimeoutMs(this.electionTimeoutMs);
+ nodeOptions.setEnableMetrics(enableMetrics);
+ nodeOptions.setSnapshotThrottle(snapshotThrottle);
+ nodeOptions.setSnapshotIntervalSecs(snapshotIntervalSecs);
+ nodeOptions.setServiceFactory(this.raftServiceFactory);
+ if (raftOptions != null) {
+ nodeOptions.setRaftOptions(raftOptions);
+ }
+ final String serverDataPath = this.dataPath + File.separator + listenAddr.toString().replace(':', '_');
+ FileUtils.forceMkdir(new File(serverDataPath));
+ nodeOptions.setLogUri(serverDataPath + File.separator + "logs");
+ nodeOptions.setRaftMetaUri(serverDataPath + File.separator + "meta");
+ nodeOptions.setSnapshotUri(serverDataPath + File.separator + "snapshot");
+ final MockStateMachine fsm = new MockStateMachine(listenAddr);
+ nodeOptions.setFsm(fsm);
+
+ if (!emptyPeers) {
+ nodeOptions.setInitialConf(new Configuration(this.peers, this.learners));
+ }
+
+ final RpcServer rpcServer = RaftRpcServerFactory.createRaftRpcServer(listenAddr);
+ final RaftGroupService server = new RaftGroupService(this.name, new PeerId(listenAddr, 0), nodeOptions,
+ rpcServer);
+
+ this.lock.lock();
+ try {
+ if (this.serverMap.put(listenAddr.toString(), server) == null) {
+ final Node node = server.start();
+
+ this.fsms.put(new PeerId(listenAddr, 0), fsm);
+ this.nodes.add((NodeImpl) node);
+ return true;
+ }
+ } finally {
+ this.lock.unlock();
+ }
+ return false;
+ }
+
+ public MockStateMachine getFsmByPeer(final PeerId peer) {
+ this.lock.lock();
+ try {
+ return this.fsms.get(peer);
+ } finally {
+ this.lock.unlock();
+ }
+ }
+
+ public List<MockStateMachine> getFsms() {
+ this.lock.lock();
+ try {
+ return new ArrayList<>(this.fsms.values());
+ } finally {
+ this.lock.unlock();
+ }
+ }
+
+ public boolean stop(final Endpoint listenAddr) throws InterruptedException {
+ final Node node = removeNode(listenAddr);
+ final CountDownLatch latch = new CountDownLatch(1);
+ if (node != null) {
+ node.shutdown(new ExpectClosure(latch));
+ node.join();
+ latch.await();
+ }
+ final RaftGroupService raftGroupService = this.serverMap.remove(listenAddr.toString());
+ raftGroupService.shutdown();
+ raftGroupService.join();
+ return node != null;
+ }
+
+ public void stopAll() throws InterruptedException {
+ final List<Endpoint> addrs = getAllNodes();
+ final List<Node> nodes = new ArrayList<>();
+ for (final Endpoint addr : addrs) {
+ final Node node = removeNode(addr);
+ node.shutdown();
+ nodes.add(node);
+ this.serverMap.remove(addr.toString()).shutdown();
+ }
+ for (final Node node : nodes) {
+ node.join();
+ }
+ CLUSTERS.remove(this);
+ }
+
+ public void clean(final Endpoint listenAddr) throws IOException {
+ final String path = this.dataPath + File.separator + listenAddr.toString().replace(':', '_');
+ System.out.println("Clean dir:" + path);
+ FileUtils.deleteDirectory(new File(path));
+ }
+
+ public Node getLeader() {
+ this.lock.lock();
+ try {
+ for (int i = 0; i < this.nodes.size(); i++) {
+ final NodeImpl node = this.nodes.get(i);
+ if (node.isLeader() && this.fsms.get(node.getServerId()).getLeaderTerm() == node.getCurrentTerm()) {
+ return node;
+ }
+ }
+ return null;
+ } finally {
+ this.lock.unlock();
+ }
+ }
+
+ public MockStateMachine getLeaderFsm() {
+ final Node leader = getLeader();
+ if (leader != null) {
+ return (MockStateMachine) leader.getOptions().getFsm();
+ }
+ return null;
+ }
+
+ public void waitLeader() throws InterruptedException {
+ while (true) {
+ final Node node = getLeader();
+ if (node != null) {
+ return;
+ } else {
+ Thread.sleep(10);
+ }
+ }
+ }
+
+ public List<Node> getFollowers() {
+ final List<Node> ret = new ArrayList<>();
+ this.lock.lock();
+ try {
+ for (final NodeImpl node : this.nodes) {
+ if (!node.isLeader() && !this.learners.contains(node.getServerId())) {
+ ret.add(node);
+ }
+ }
+ } finally {
+ this.lock.unlock();
+ }
+ return ret;
+ }
+
+ /**
+ * Ensure all peers leader is expectAddr
+ * @param expectAddr expected address
+ * @throws InterruptedException if interrupted
+ */
+ public void ensureLeader(final Endpoint expectAddr) throws InterruptedException {
+ while (true) {
+ this.lock.lock();
+ for (final Node node : this.nodes) {
+ final PeerId leaderId = node.getLeaderId();
+ if (!leaderId.getEndpoint().equals(expectAddr)) {
+ this.lock.unlock();
+ Thread.sleep(10);
+ continue;
+ }
+ }
+ // all is ready
+ this.lock.unlock();
+ return;
+ }
+ }
+
+ public List<NodeImpl> getNodes() {
+ this.lock.lock();
+ try {
+ return new ArrayList<>(this.nodes);
+ } finally {
+ this.lock.unlock();
+ }
+ }
+
+ public List<Endpoint> getAllNodes() {
+ this.lock.lock();
+ try {
+ return this.nodes.stream().map(node -> node.getNodeId().getPeerId().getEndpoint())
+ .collect(Collectors.toList());
+ } finally {
+ this.lock.unlock();
+ }
+ }
+
+ public Node removeNode(final Endpoint addr) {
+ Node ret = null;
+ this.lock.lock();
+ try {
+ for (int i = 0; i < this.nodes.size(); i++) {
+ if (this.nodes.get(i).getNodeId().getPeerId().getEndpoint().equals(addr)) {
+ ret = this.nodes.remove(i);
+ this.fsms.remove(ret.getNodeId().getPeerId());
+ break;
+ }
+ }
+ } finally {
+ this.lock.unlock();
+ }
+ return ret;
+ }
+
+ public boolean ensureSame() throws InterruptedException {
+ return this.ensureSame(-1);
+ }
+
+ /**
+ * Ensure all logs is the same in all nodes.
+ * @param waitTimes
+ * @return
+ * @throws InterruptedException
+ */
+ public boolean ensureSame(final int waitTimes) throws InterruptedException {
+ this.lock.lock();
+ List<MockStateMachine> fsmList = new ArrayList<>(this.fsms.values());
+ if (fsmList.size() <= 1) {
+ this.lock.unlock();
+ return true;
+ }
+ System.out.println("Start ensureSame, waitTimes=" + waitTimes);
+ try {
+ int nround = 0;
+ final MockStateMachine first = fsmList.get(0);
+ CHECK: while (true) {
+ first.lock();
+ if (first.getLogs().isEmpty()) {
+ first.unlock();
+ Thread.sleep(10);
+ nround++;
+ if (waitTimes > 0 && nround > waitTimes) {
+ return false;
+ }
+ continue CHECK;
+ }
+
+ for (int i = 1; i < fsmList.size(); i++) {
+ final MockStateMachine fsm = fsmList.get(i);
+ fsm.lock();
+ if (fsm.getLogs().size() != first.getLogs().size()) {
+ fsm.unlock();
+ first.unlock();
+ Thread.sleep(10);
+ nround++;
+ if (waitTimes > 0 && nround > waitTimes) {
+ return false;
+ }
+ continue CHECK;
+ }
+
+ for (int j = 0; j < first.getLogs().size(); j++) {
+ final ByteBuffer firstData = first.getLogs().get(j);
+ final ByteBuffer fsmData = fsm.getLogs().get(j);
+ if (!firstData.equals(fsmData)) {
+ fsm.unlock();
+ first.unlock();
+ Thread.sleep(10);
+ nround++;
+ if (waitTimes > 0 && nround > waitTimes) {
+ return false;
+ }
+ continue CHECK;
+ }
+ }
+ fsm.unlock();
+ }
+ first.unlock();
+ break;
+ }
+ return true;
+ } finally {
+ this.lock.unlock();
+ System.out.println("End ensureSame, waitTimes=" + waitTimes);
+ }
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/core/TestJRaftServiceFactory.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/TestJRaftServiceFactory.java
new file mode 100644
index 0000000..a7ef620
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/TestJRaftServiceFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.storage.LogStorage;
+import com.alipay.sofa.jraft.storage.impl.LocalLogStorage;
+//import com.alipay.sofa.jraft.storage.log.RocksDBSegmentLogStorage;
+
+public class TestJRaftServiceFactory extends DefaultJRaftServiceFactory {
+
+ @Override
+ public LogStorage createLogStorage(final String uri, final RaftOptions raftOptions) {
+// return RocksDBSegmentLogStorage.builder(uri, raftOptions) //
+// .setPreAllocateSegmentCount(1) //
+// .setKeepInMemorySegmentCount(2) //
+// .setMaxSegmentFileSize(512 * 1024) //
+// .setValueSizeThreshold(0) //
+// .build();
+
+ return new LocalLogStorage(uri, raftOptions);
+ }
+
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/core/V1JRaftServiceFactory.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/V1JRaftServiceFactory.java
new file mode 100644
index 0000000..9a9209d
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/core/V1JRaftServiceFactory.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import com.alipay.sofa.jraft.entity.codec.LogEntryCodecFactory;
+import com.alipay.sofa.jraft.entity.codec.v1.LogEntryV1CodecFactory;
+
+public class V1JRaftServiceFactory extends DefaultJRaftServiceFactory {
+
+ @Override
+ public LogEntryCodecFactory createLogEntryCodecFactory() {
+ return LogEntryV1CodecFactory.getInstance();
+ }
+
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/BallotTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/BallotTest.java
new file mode 100644
index 0000000..f82388b
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/BallotTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.entity;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.JRaftUtils;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class BallotTest {
+
+ private Ballot ballot;
+
+ @Before
+ public void setup() {
+ this.ballot = new Ballot();
+ this.ballot.init(JRaftUtils.getConfiguration("localhost:8081,localhost:8082,localhost:8083"), null);
+ }
+
+ @Test
+ public void testGrant() {
+ PeerId peer1 = new PeerId("localhost", 8081);
+ this.ballot.grant(peer1);
+ assertFalse(this.ballot.isGranted());
+
+ PeerId unfoundPeer = new PeerId("localhost", 8084);
+ this.ballot.grant(unfoundPeer);
+ assertFalse(this.ballot.isGranted());
+
+ PeerId peer2 = new PeerId("localhost", 8082);
+ this.ballot.grant(peer2);
+ assertTrue(this.ballot.isGranted());
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/LogEntryTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/LogEntryTest.java
new file mode 100644
index 0000000..5f1a33d
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/LogEntryTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.entity;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.entity.codec.v1.LogEntryV1CodecFactory;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class LogEntryTest {
+
+ @SuppressWarnings("deprecation")
+ @Test
+ public void testEncodeDecodeWithoutData() {
+ LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+ entry.setId(new LogId(100, 3));
+ entry.setPeers(Arrays.asList(new PeerId("localhost", 99, 1), new PeerId("localhost", 100, 2)));
+ assertNull(entry.getData());
+ assertNull(entry.getOldPeers());
+
+ byte[] content = entry.encode();
+
+ assertNotNull(content);
+ assertTrue(content.length > 0);
+ assertEquals(LogEntryV1CodecFactory.MAGIC, content[0]);
+
+ LogEntry nentry = new LogEntry();
+ assertTrue(nentry.decode(content));
+
+ assertEquals(100, nentry.getId().getIndex());
+ assertEquals(3, nentry.getId().getTerm());
+ Assert.assertEquals(EnumOutter.EntryType.ENTRY_TYPE_NO_OP, nentry.getType());
+ assertEquals(2, nentry.getPeers().size());
+ assertEquals("localhost:99:1", nentry.getPeers().get(0).toString());
+ assertEquals("localhost:100:2", nentry.getPeers().get(1).toString());
+ assertNull(nentry.getData());
+ assertNull(nentry.getOldPeers());
+ }
+
+ @SuppressWarnings("deprecation")
+ @Test
+ public void testEncodeDecodeWithData() {
+ ByteBuffer buf = ByteBuffer.wrap("hello".getBytes());
+ LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+ entry.setId(new LogId(100, 3));
+ entry.setData(buf);
+ entry.setPeers(Arrays.asList(new PeerId("localhost", 99, 1), new PeerId("localhost", 100, 2)));
+ assertEquals(buf, entry.getData());
+
+ byte[] content = entry.encode();
+
+ assertNotNull(content);
+ assertTrue(content.length > 0);
+ assertEquals(LogEntryV1CodecFactory.MAGIC, content[0]);
+
+ LogEntry nentry = new LogEntry();
+ assertTrue(nentry.decode(content));
+
+ assertEquals(100, nentry.getId().getIndex());
+ assertEquals(3, nentry.getId().getTerm());
+
+ assertEquals(2, nentry.getPeers().size());
+ assertEquals("localhost:99:1", nentry.getPeers().get(0).toString());
+ assertEquals("localhost:100:2", nentry.getPeers().get(1).toString());
+ assertEquals(buf, nentry.getData());
+ assertEquals(0, nentry.getData().position());
+ assertEquals(5, nentry.getData().remaining());
+ assertNull(nentry.getOldPeers());
+ }
+
+ @Test
+ public void testChecksum() {
+ ByteBuffer buf = ByteBuffer.wrap("hello".getBytes());
+ LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+ entry.setId(new LogId(100, 3));
+ entry.setData(buf);
+ entry.setPeers(Arrays.asList(new PeerId("localhost", 99, 1), new PeerId("localhost", 100, 2)));
+
+ long c = entry.checksum();
+ assertTrue(c != 0);
+ assertEquals(c, entry.checksum());
+ assertFalse(entry.isCorrupted());
+
+ assertFalse(entry.hasChecksum());
+ entry.setChecksum(c);
+ assertTrue(entry.hasChecksum());
+ assertFalse(entry.isCorrupted());
+
+ // modify index, detect corrupted.
+ entry.getId().setIndex(1);
+ assertNotEquals(c, entry.checksum());
+ assertTrue(entry.isCorrupted());
+ // fix index
+ entry.getId().setIndex(100);
+ assertFalse(entry.isCorrupted());
+
+ // modify data, detect corrupted
+ entry.setData(ByteBuffer.wrap("hEllo".getBytes()));
+ assertNotEquals(c, entry.checksum());
+ assertTrue(entry.isCorrupted());
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/LogIdTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/LogIdTest.java
new file mode 100644
index 0000000..fcee0a2
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/LogIdTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.entity;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class LogIdTest {
+
+ @Test
+ public void testCompareTo() {
+ LogId logId = new LogId();
+ assertEquals(0, logId.getIndex());
+ assertEquals(0, logId.getTerm());
+
+ assertTrue(new LogId(1, 0).compareTo(logId) > 0);
+ assertTrue(new LogId(0, 1).compareTo(logId) > 0);
+
+ logId = new LogId(1, 2);
+ assertTrue(new LogId(0, 1).compareTo(logId) < 0);
+ assertTrue(new LogId(0, 2).compareTo(logId) < 0);
+ assertTrue(new LogId(3, 1).compareTo(logId) < 0);
+ assertTrue(new LogId(1, 2).compareTo(logId) == 0);
+ }
+
+ @Test
+ public void testChecksum() {
+ LogId logId = new LogId();
+ logId.setIndex(1);
+ logId.setTerm(2);
+ long c = logId.checksum();
+ assertTrue(c != 0);
+ assertEquals(c, logId.checksum());
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/PeerIdTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/PeerIdTest.java
new file mode 100644
index 0000000..a83910a
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/PeerIdTest.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.entity;
+
+import com.alipay.sofa.jraft.util.Endpoint;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class PeerIdTest {
+
+ @Test
+ public void testToStringParse() {
+ final PeerId peer = new PeerId("192.168.1.1", 8081, 0);
+ assertEquals("192.168.1.1:8081", peer.toString());
+
+ final PeerId pp = new PeerId();
+ assertTrue(pp.parse(peer.toString()));
+ assertEquals(8081, pp.getPort());
+ assertEquals("192.168.1.1", pp.getIp());
+ assertEquals(0, pp.getIdx());
+ assertEquals(pp, peer);
+ assertEquals(pp.hashCode(), peer.hashCode());
+ }
+
+ @Test
+ public void testIsPriorityNotElected() {
+
+ final Endpoint endpoint1 = new Endpoint("192.168.1.1", 8081);
+ final PeerId peer1 = new PeerId(endpoint1, 0, 0);
+ assertEquals("192.168.1.1:8081::0", peer1.toString());
+ assertTrue(peer1.isPriorityNotElected());
+ }
+
+ @Test
+ public void testIsPriorityDisabled() {
+
+ final Endpoint endpoint1 = new Endpoint("192.168.1.1", 8081);
+ final PeerId peer1 = new PeerId(endpoint1, 0);
+ assertEquals("192.168.1.1:8081", peer1.toString());
+ assertTrue(peer1.isPriorityDisabled());
+ }
+
+ @Test
+ public void testToStringParseWithIdxAndPriority() {
+
+ // 1.String format is, ip:port::priority
+ final Endpoint endpoint1 = new Endpoint("192.168.1.1", 8081);
+ final PeerId peer1 = new PeerId(endpoint1, 0, 100);
+ assertEquals("192.168.1.1:8081::100", peer1.toString());
+
+ final PeerId p1 = new PeerId();
+ final String str1 = "192.168.1.1:8081::100";
+ assertTrue(p1.parse(str1));
+ assertEquals(8081, p1.getPort());
+ assertEquals("192.168.1.1", p1.getIp());
+ assertEquals(0, p1.getIdx());
+ assertEquals(100, p1.getPriority());
+
+ assertEquals(p1, peer1);
+ assertEquals(p1.hashCode(), peer1.hashCode());
+
+ // 2.String format is, ip:port:idx:priority
+ final Endpoint endpoint2 = new Endpoint("192.168.1.1", 8081);
+ final PeerId peer2 = new PeerId(endpoint2, 100, 200);
+ assertEquals("192.168.1.1:8081:100:200", peer2.toString());
+
+ final PeerId p2 = new PeerId();
+ final String str2 = "192.168.1.1:8081:100:200";
+ assertTrue(p2.parse(str2));
+ assertEquals(8081, p2.getPort());
+ assertEquals("192.168.1.1", p2.getIp());
+ assertEquals(100, p2.getIdx());
+ assertEquals(200, p2.getPriority());
+
+ assertEquals(p2, peer2);
+ assertEquals(p2.hashCode(), peer2.hashCode());
+ }
+
+ @Test
+ public void testIdx() {
+ final PeerId peer = new PeerId("192.168.1.1", 8081, 1);
+ assertEquals("192.168.1.1:8081:1", peer.toString());
+ assertFalse(peer.isEmpty());
+
+ final PeerId pp = new PeerId();
+ assertTrue(pp.parse(peer.toString()));
+ assertEquals(8081, pp.getPort());
+ assertEquals("192.168.1.1", pp.getIp());
+ assertEquals(1, pp.getIdx());
+ assertEquals(pp, peer);
+ assertEquals(pp.hashCode(), peer.hashCode());
+ }
+
+ @Test
+ public void testParseFail() {
+ final PeerId peer = new PeerId();
+ assertTrue(peer.isEmpty());
+ assertFalse(peer.parse("localhsot:2:3:4:5"));
+ assertTrue(peer.isEmpty());
+ }
+
+ @Test
+ public void testEmptyPeer() {
+ PeerId peer = new PeerId("192.168.1.1", 8081, 1);
+ assertFalse(peer.isEmpty());
+ peer = PeerId.emptyPeer();
+ assertTrue(peer.isEmpty());
+ }
+
+ @Test
+ public void testChecksum() {
+ PeerId peer = new PeerId("192.168.1.1", 8081, 1);
+ long c = peer.checksum();
+ assertTrue(c != 0);
+ assertEquals(c, peer.checksum());
+ }
+
+ @Test
+ public void testToStringParseFailed() {
+ final PeerId pp = new PeerId();
+ final String str1 = "";
+ final String str2 = "192.168.1.1";
+ final String str3 = "92.168.1.1:8081::1:2";
+ assertFalse(pp.parse(str1));
+ assertFalse(pp.parse(str2));
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/codec/BaseLogEntryCodecFactoryTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/codec/BaseLogEntryCodecFactoryTest.java
new file mode 100644
index 0000000..8aec68c
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/codec/BaseLogEntryCodecFactoryTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.entity.codec;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.entity.EnumOutter;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.entity.LogId;
+import com.alipay.sofa.jraft.entity.PeerId;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public abstract class BaseLogEntryCodecFactoryTest {
+
+ protected LogEntryEncoder encoder;
+ protected LogEntryDecoder decoder;
+
+ @Before
+ public void setup() {
+ LogEntryCodecFactory factory = newFactory();
+ this.encoder = factory.encoder();
+ this.decoder = factory.decoder();
+ }
+
+ protected abstract LogEntryCodecFactory newFactory();
+
+ @Test
+ public void testEncodeDecodeEmpty() {
+ try {
+ assertNull(this.encoder.encode(null));
+ fail();
+ } catch (NullPointerException e) {
+ assertTrue(true);
+ }
+ assertNull(this.decoder.decode(null));
+ assertNull(this.decoder.decode(new byte[0]));
+ }
+
+ @Test
+ public void testEncodeDecodeWithoutData() {
+ LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+ entry.setId(new LogId(100, 3));
+ entry.setPeers(Arrays.asList(new PeerId("localhost", 99, 1), new PeerId("localhost", 100, 2)));
+ assertNull(entry.getData());
+ assertNull(entry.getOldPeers());
+
+ byte[] content = this.encoder.encode(entry);
+
+ assertNotNull(content);
+ assertTrue(content.length > 0);
+
+ LogEntry nentry = this.decoder.decode(content);
+ assertNotNull(nentry);
+
+ assertEquals(100, nentry.getId().getIndex());
+ assertEquals(3, nentry.getId().getTerm());
+ Assert.assertEquals(EnumOutter.EntryType.ENTRY_TYPE_NO_OP, nentry.getType());
+ assertEquals(2, nentry.getPeers().size());
+ assertEquals("localhost:99:1", nentry.getPeers().get(0).toString());
+ assertEquals("localhost:100:2", nentry.getPeers().get(1).toString());
+ assertNull(nentry.getData());
+ assertNull(nentry.getOldPeers());
+ }
+
+ @Test
+ public void testEncodeDecodeWithData() {
+ ByteBuffer buf = ByteBuffer.wrap("hello".getBytes());
+ LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+ entry.setId(new LogId(100, 3));
+ entry.setData(buf);
+ entry.setPeers(Arrays.asList(new PeerId("localhost", 99, 1), new PeerId("localhost", 100, 2)));
+ assertEquals(buf, entry.getData());
+
+ byte[] content = this.encoder.encode(entry);
+
+ assertNotNull(content);
+ assertTrue(content.length > 0);
+
+ LogEntry nentry = this.decoder.decode(content);
+ assertNotNull(nentry);
+
+ assertEquals(100, nentry.getId().getIndex());
+ assertEquals(3, nentry.getId().getTerm());
+
+ assertEquals(2, nentry.getPeers().size());
+ assertEquals("localhost:99:1", nentry.getPeers().get(0).toString());
+ assertEquals("localhost:100:2", nentry.getPeers().get(1).toString());
+ assertEquals(buf, nentry.getData());
+ assertEquals(0, nentry.getData().position());
+ assertEquals(5, nentry.getData().remaining());
+ assertNull(nentry.getOldPeers());
+ }
+
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/codec/LogEntryCodecPerfTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/codec/LogEntryCodecPerfTest.java
new file mode 100644
index 0000000..39933ff
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/codec/LogEntryCodecPerfTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.entity.codec;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.entity.EnumOutter;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.entity.LogId;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.entity.codec.v1.V1Decoder;
+import com.alipay.sofa.jraft.entity.codec.v1.V1Encoder;
+//import com.alipay.sofa.jraft.entity.codec.v2.V2Encoder;
+import com.alipay.sofa.jraft.util.Utils;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class LogEntryCodecPerfTest {
+
+ static byte[] DATA = new byte[512];
+
+ static {
+ ThreadLocalRandom.current().nextBytes(DATA);
+ }
+
+ static final int TIMES = 100000;
+
+ static final int THREADS = 20;
+
+ private final AtomicLong logSize = new AtomicLong(0);
+
+ @Before
+ public void setup() throws Exception {
+ this.logSize.set(0);
+ System.gc();
+ }
+
+ private void testEncodeDecode(final LogEntryEncoder encoder, final LogEntryDecoder decoder,
+ final CyclicBarrier barrier) throws Exception {
+ ByteBuffer buf = ByteBuffer.wrap(DATA);
+ LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+ entry.setData(buf);
+ entry.setPeers(Arrays.asList(new PeerId("localhost", 99, 1), new PeerId("localhost", 100, 2)));
+
+ if (barrier != null) {
+ barrier.await();
+ }
+
+ for (int i = 0; i < TIMES; i++) {
+ entry.setId(new LogId(i, i));
+ byte[] content = encoder.encode(entry);
+ assert (content.length > 0);
+ this.logSize.addAndGet(content.length);
+ LogEntry nLog = decoder.decode(content);
+ assertEquals(2, nLog.getPeers().size());
+ assertArrayEquals(DATA, nLog.getData().array());
+ assertEquals(i, nLog.getId().getIndex());
+ assertEquals(i, nLog.getId().getTerm());
+ }
+
+ if (barrier != null) {
+ barrier.await();
+ }
+
+ }
+
+ @Test
+ public void testV1Codec() throws Exception {
+ LogEntryEncoder encoder = V1Encoder.INSTANCE;
+ LogEntryDecoder decoder = V1Decoder.INSTANCE;
+ testEncodeDecode(encoder, decoder, null);
+ concurrentTest("V1", encoder, decoder);
+ }
+
+// @Test
+// public void testV2Codec() throws Exception {
+// LogEntryEncoder encoder = V2Encoder.INSTANCE;
+// LogEntryDecoder decoder = AutoDetectDecoder.INSTANCE;
+// testEncodeDecode(encoder, decoder, null);
+// concurrentTest("V2", encoder, decoder);
+// }
+
+ private void concurrentTest(final String version, final LogEntryEncoder encoder, final LogEntryDecoder decoder)
+ throws InterruptedException,
+ BrokenBarrierException {
+ final CyclicBarrier barrier = new CyclicBarrier(THREADS + 1);
+ for (int i = 0; i < THREADS; i++) {
+ new Thread(() -> {
+ try {
+ testEncodeDecode(encoder, decoder, barrier);
+ } catch (Exception e) {
+ e.printStackTrace(); // NOPMD
+ fail();
+ }
+ }).start();
+ }
+ long start = Utils.monotonicMs();
+ barrier.await();
+ barrier.await();
+ System.out.println(version + " codec cost:" + (Utils.monotonicMs() - start) + " ms.");
+ System.out.println("Total log size:" + this.logSize.get() + " bytes.");
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/codec/v1/LogEntryV1CodecFactoryTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/codec/v1/LogEntryV1CodecFactoryTest.java
new file mode 100644
index 0000000..09e985c
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/entity/codec/v1/LogEntryV1CodecFactoryTest.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.entity.codec.v1;
+
+import com.alipay.sofa.jraft.entity.codec.BaseLogEntryCodecFactoryTest;
+import com.alipay.sofa.jraft.entity.codec.LogEntryCodecFactory;
+
+public class LogEntryV1CodecFactoryTest extends BaseLogEntryCodecFactoryTest {
+
+ @Override
+ protected LogEntryCodecFactory newFactory() {
+ LogEntryCodecFactory factory = LogEntryV1CodecFactory.getInstance();
+ return factory;
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/AbstractClientServiceTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/AbstractClientServiceTest.java
new file mode 100644
index 0000000..ec6c656
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/AbstractClientServiceTest.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.error.InvokeTimeoutException;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.error.RemotingException;
+import com.alipay.sofa.jraft.option.RpcOptions;
+import com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse;
+import com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest;
+import com.alipay.sofa.jraft.rpc.impl.AbstractClientService;
+import com.alipay.sofa.jraft.test.TestUtils;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.RpcFactoryHelper;
+import com.alipay.sofa.jraft.rpc.Message;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.eq;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class AbstractClientServiceTest {
+ static class MockClientService extends AbstractClientService {
+ public void setRpcClient(final RpcClient rpcClient) {
+ this.rpcClient = rpcClient;
+ }
+ }
+
+ private RpcOptions rpcOptions;
+ private MockClientService clientService;
+ @Mock
+ private RpcClient rpcClient;
+ private RpcResponseFactory rpcResponseFactory = RpcFactoryHelper.responseFactory();
+ private final Endpoint endpoint = new Endpoint("localhost", 8081);
+
+ @Before
+ public void setup() {
+ this.rpcOptions = new RpcOptions();
+ this.clientService = new MockClientService();
+ assertTrue(this.clientService.init(this.rpcOptions));
+ this.clientService.setRpcClient(this.rpcClient);
+
+ }
+
+ @Test
+ public void testConnect() throws Exception {
+ Mockito.when(
+ this.rpcClient.invokeSync(eq(this.endpoint), Mockito.any(),
+ eq((long) this.rpcOptions.getRpcConnectTimeoutMs()))) //
+ .thenReturn(this.rpcResponseFactory.newResponse(null, Status.OK()));
+ assertTrue(this.clientService.connect(this.endpoint));
+ }
+
+ @Test
+ public void testConnectFailure() throws Exception {
+ Mockito.when(
+ this.rpcClient.invokeSync(eq(this.endpoint), Mockito.any(),
+ eq((long) this.rpcOptions.getRpcConnectTimeoutMs()))) //
+ .thenReturn(this.rpcResponseFactory.newResponse(null, new Status(-1, "test")));
+ assertFalse(this.clientService.connect(this.endpoint));
+ }
+
+ @Test
+ public void testConnectException() throws Exception {
+ Mockito.when(
+ this.rpcClient.invokeSync(eq(this.endpoint), Mockito.any(),
+ eq((long) this.rpcOptions.getRpcConnectTimeoutMs()))) //
+ .thenThrow(new RemotingException("test"));
+ assertFalse(this.clientService.connect(this.endpoint));
+ }
+
+ @Test
+ public void testDisconnect() {
+ this.clientService.disconnect(this.endpoint);
+ Mockito.verify(this.rpcClient).closeConnection(this.endpoint);
+ }
+
+ static class MockRpcResponseClosure<T extends Message> extends RpcResponseClosureAdapter<T> {
+
+ CountDownLatch latch = new CountDownLatch(1);
+
+ Status status;
+
+ @Override
+ public void run(final Status status) {
+ this.status = status;
+ this.latch.countDown();
+ }
+
+ }
+
+ @Test
+ public void testCancel() throws Exception {
+ ArgumentCaptor<InvokeCallback> callbackArg = ArgumentCaptor.forClass(InvokeCallback.class);
+ PingRequest request = TestUtils.createPingRequest();
+
+ MockRpcResponseClosure<ErrorResponse> done = new MockRpcResponseClosure<>();
+ Future<Message> future = this.clientService.invokeWithDone(this.endpoint, request, done, -1);
+ Mockito.verify(this.rpcClient).invokeAsync(eq(this.endpoint), eq(request), Mockito.any(),
+ callbackArg.capture(), eq((long) this.rpcOptions.getRpcDefaultTimeout()));
+ InvokeCallback cb = callbackArg.getValue();
+ assertNotNull(cb);
+ assertNotNull(future);
+
+ assertNull(done.getResponse());
+ assertNull(done.status);
+ assertFalse(future.isDone());
+
+ future.cancel(true);
+ ErrorResponse response = (ErrorResponse) this.rpcResponseFactory.newResponse(null, Status.OK());
+ cb.complete(response, null);
+
+ // The closure should be notified with ECANCELED error code.
+ done.latch.await();
+ assertNotNull(done.status);
+ assertEquals(RaftError.ECANCELED.getNumber(), done.status.getCode());
+ }
+
+ @Test
+ public void testInvokeWithDoneOK() throws Exception {
+ ArgumentCaptor<InvokeCallback> callbackArg = ArgumentCaptor.forClass(InvokeCallback.class);
+ PingRequest request = TestUtils.createPingRequest();
+
+ MockRpcResponseClosure<ErrorResponse> done = new MockRpcResponseClosure<>();
+ Future<Message> future = this.clientService.invokeWithDone(this.endpoint, request, done, -1);
+ Mockito.verify(this.rpcClient).invokeAsync(eq(this.endpoint), eq(request), Mockito.any(),
+ callbackArg.capture(), eq((long) this.rpcOptions.getRpcDefaultTimeout()));
+ InvokeCallback cb = callbackArg.getValue();
+ assertNotNull(cb);
+ assertNotNull(future);
+
+ assertNull(done.getResponse());
+ assertNull(done.status);
+ assertFalse(future.isDone());
+
+ ErrorResponse response = (ErrorResponse) this.rpcResponseFactory.newResponse(null, Status.OK());
+ cb.complete(response, null);
+
+ Message msg = future.get();
+ assertNotNull(msg);
+ assertTrue(msg instanceof ErrorResponse);
+ assertSame(msg, response);
+
+ done.latch.await();
+ assertNotNull(done.status);
+ assertEquals(0, done.status.getCode());
+ }
+
+ @Test
+ public void testInvokeWithDoneException() throws Exception {
+ InvokeContext invokeCtx = new InvokeContext();
+ invokeCtx.put(InvokeContext.CRC_SWITCH, false);
+ ArgumentCaptor<InvokeCallback> callbackArg = ArgumentCaptor.forClass(InvokeCallback.class);
+ PingRequest request = TestUtils.createPingRequest();
+
+ Mockito
+ .doThrow(new RemotingException())
+ .when(this.rpcClient)
+ .invokeAsync(eq(this.endpoint), eq(request), eq(invokeCtx), callbackArg.capture(),
+ eq((long) this.rpcOptions.getRpcDefaultTimeout()));
+
+ MockRpcResponseClosure<ErrorResponse> done = new MockRpcResponseClosure<>();
+ Future<Message> future = this.clientService.invokeWithDone(this.endpoint, request, invokeCtx, done, -1);
+ InvokeCallback cb = callbackArg.getValue();
+ assertNotNull(cb);
+ assertNotNull(future);
+
+ assertTrue(future.isDone());
+
+ try {
+ future.get();
+ fail();
+ } catch (ExecutionException e) {
+ assertTrue(e.getCause() instanceof RemotingException);
+ }
+
+ done.latch.await();
+ assertNotNull(done.status);
+ assertEquals(RaftError.EINTERNAL.getNumber(), done.status.getCode());
+ }
+
+ @Test
+ public void testInvokeWithDoneOnException() throws Exception {
+ InvokeContext invokeCtx = new InvokeContext();
+ invokeCtx.put(InvokeContext.CRC_SWITCH, false);
+ ArgumentCaptor<InvokeCallback> callbackArg = ArgumentCaptor.forClass(InvokeCallback.class);
+ PingRequest request = TestUtils.createPingRequest();
+
+ MockRpcResponseClosure<ErrorResponse> done = new MockRpcResponseClosure<>();
+ Future<Message> future = this.clientService.invokeWithDone(this.endpoint, request, invokeCtx, done, -1);
+ Mockito.verify(this.rpcClient).invokeAsync(eq(this.endpoint), eq(request), eq(invokeCtx),
+ callbackArg.capture(), eq((long) this.rpcOptions.getRpcDefaultTimeout()));
+ InvokeCallback cb = callbackArg.getValue();
+ assertNotNull(cb);
+ assertNotNull(future);
+
+ assertNull(done.getResponse());
+ assertNull(done.status);
+ assertFalse(future.isDone());
+
+ cb.complete(null, new InvokeTimeoutException());
+
+ try {
+ future.get();
+ fail();
+ } catch (ExecutionException e) {
+ assertTrue(e.getCause() instanceof InvokeTimeoutException);
+ }
+
+ done.latch.await();
+ assertNotNull(done.status);
+ assertEquals(RaftError.ETIMEDOUT.getNumber(), done.status.getCode());
+ }
+
+ @Test
+ public void testInvokeWithDOneOnErrorResponse() throws Exception {
+ final InvokeContext invokeCtx = new InvokeContext();
+ invokeCtx.put(InvokeContext.CRC_SWITCH, false);
+ final ArgumentCaptor<InvokeCallback> callbackArg = ArgumentCaptor.forClass(InvokeCallback.class);
+ final CliRequests.GetPeersRequest request = CliRequests.GetPeersRequest.newBuilder() //
+ .setGroupId("id") //
+ .setLeaderId("127.0.0.1:8001") //
+ .build();
+
+ MockRpcResponseClosure<ErrorResponse> done = new MockRpcResponseClosure<>();
+ Future<Message> future = this.clientService.invokeWithDone(this.endpoint, request, invokeCtx, done, -1);
+ Mockito.verify(this.rpcClient).invokeAsync(eq(this.endpoint), eq(request), eq(invokeCtx),
+ callbackArg.capture(), eq((long) this.rpcOptions.getRpcDefaultTimeout()));
+ InvokeCallback cb = callbackArg.getValue();
+ assertNotNull(cb);
+ assertNotNull(future);
+
+ assertNull(done.getResponse());
+ assertNull(done.status);
+ assertFalse(future.isDone());
+
+ final Message resp = this.rpcResponseFactory.newResponse(CliRequests.GetPeersResponse.getDefaultInstance(),
+ new Status(-1, "failed"));
+ cb.complete(resp, null);
+
+ final Message msg = future.get();
+
+ assertTrue(msg instanceof ErrorResponse);
+ assertEquals(((ErrorResponse) msg).getErrorMsg(), "failed");
+
+ done.latch.await();
+ assertNotNull(done.status);
+ assertTrue(!done.status.isOk());
+ assertEquals(done.status.getErrorMsg(), "failed");
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/AppendEntriesBenchmark.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/AppendEntriesBenchmark.java
new file mode 100644
index 0000000..910f89d
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/AppendEntriesBenchmark.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc;
+
+import com.alipay.sofa.jraft.util.ByteString;
+import java.nio.ByteBuffer;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.options.TimeValue;
+
+import com.alipay.sofa.jraft.util.AdaptiveBufAllocator;
+import com.alipay.sofa.jraft.util.ByteBufferCollector;
+import com.alipay.sofa.jraft.util.RecyclableByteBufferList;
+import com.alipay.sofa.jraft.util.RecycleUtil;
+
+import static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+@State(Scope.Benchmark)
+public class AppendEntriesBenchmark {
+
+ /**
+ * entryCount=256, sizeOfEntry=2048
+ * ---------------------------------------------------------------------------
+ * Benchmark Mode Cnt Score Error Units
+ * AppendEntriesBenchmark.adaptiveAndPooled thrpt 3 4.139 ± 2.662 ops/ms
+ * AppendEntriesBenchmark.copy thrpt 3 0.148 ± 0.027 ops/ms
+ * AppendEntriesBenchmark.pooled thrpt 3 3.730 ± 0.355 ops/ms
+ * AppendEntriesBenchmark.zeroCopy thrpt 3 3.069 ± 3.563 ops/ms
+ *
+ *
+ * entryCount=256, sizeOfEntry=1024
+ * ---------------------------------------------------------------------------
+ * Benchmark Mode Cnt Score Error Units
+ * AppendEntriesBenchmark.adaptiveAndPooled thrpt 3 8.290 ± 5.438 ops/ms
+ * AppendEntriesBenchmark.copy thrpt 3 0.326 ± 0.137 ops/ms
+ * AppendEntriesBenchmark.pooled thrpt 3 7.559 ± 1.245 ops/ms
+ * AppendEntriesBenchmark.zeroCopy thrpt 3 6.602 ± 0.859 ops/ms
+ *
+ * entryCount=256, sizeOfEntry=512
+ * ---------------------------------------------------------------------------
+ *
+ * Benchmark Mode Cnt Score Error Units
+ * AppendEntriesBenchmark.adaptiveAndPooled thrpt 3 14.358 ± 8.622 ops/ms
+ * AppendEntriesBenchmark.copy thrpt 3 1.625 ± 0.058 ops/ms
+ * AppendEntriesBenchmark.pooled thrpt 3 15.332 ± 1.531 ops/ms
+ * AppendEntriesBenchmark.zeroCopy thrpt 3 12.614 ± 5.904 ops/ms
+ *
+ * entryCount=256, sizeOfEntry=256
+ * ---------------------------------------------------------------------------
+ * Benchmark Mode Cnt Score Error Units
+ * AppendEntriesBenchmark.adaptiveAndPooled thrpt 3 32.506 ± 21.961 ops/ms
+ * AppendEntriesBenchmark.copy thrpt 3 6.595 ± 5.772 ops/ms
+ * AppendEntriesBenchmark.pooled thrpt 3 27.847 ± 14.010 ops/ms
+ * AppendEntriesBenchmark.zeroCopy thrpt 3 26.427 ± 5.187 ops/ms
+ *
+ * entryCount=256, sizeOfEntry=128
+ * ---------------------------------------------------------------------------
+ * Benchmark Mode Cnt Score Error Units
+ * AppendEntriesBenchmark.adaptiveAndPooled thrpt 3 60.014 ± 47.206 ops/ms
+ * AppendEntriesBenchmark.copy thrpt 3 22.884 ± 3.286 ops/ms
+ * AppendEntriesBenchmark.pooled thrpt 3 57.373 ± 8.201 ops/ms
+ * AppendEntriesBenchmark.zeroCopy thrpt 3 43.923 ± 7.133 ops/ms
+ *
+ * entryCount=256, sizeOfEntry=64
+ * ---------------------------------------------------------------------------
+ * Benchmark Mode Cnt Score Error Units
+ * AppendEntriesBenchmark.adaptiveAndPooled thrpt 3 114.016 ± 84.874 ops/ms
+ * AppendEntriesBenchmark.copy thrpt 3 71.699 ± 19.016 ops/ms
+ * AppendEntriesBenchmark.pooled thrpt 3 107.714 ± 7.944 ops/ms
+ * AppendEntriesBenchmark.zeroCopy thrpt 3 71.767 ± 14.510 ops/ms
+ *
+ * entryCount=256, sizeOfEntry=16
+ * ---------------------------------------------------------------------------
+ * Benchmark Mode Cnt Score Error Units
+ * AppendEntriesBenchmark.adaptiveAndPooled thrpt 3 285.386 ± 114.361 ops/ms
+ * AppendEntriesBenchmark.copy thrpt 3 243.805 ± 31.725 ops/ms
+ * AppendEntriesBenchmark.pooled thrpt 3 293.779 ± 76.557 ops/ms
+ * AppendEntriesBenchmark.zeroCopy thrpt 3 124.669 ± 32.460 ops/ms
+ */
+
+ private static final ThreadLocal<AdaptiveBufAllocator.Handle> handleThreadLocal = ThreadLocal
+ .withInitial(AdaptiveBufAllocator.DEFAULT::newHandle);
+
+ private int entryCount;
+ private int sizeOfEntry;
+
+ @Setup
+ public void setup() {
+ this.entryCount = 256;
+ this.sizeOfEntry = 2048;
+ }
+
+ public static void main(String[] args) throws RunnerException {
+ final int size = ThreadLocalRandom.current().nextInt(100, 1000);
+ System.out.println(sendEntries1(256, size).length);
+ System.out.println(sendEntries2(256, size).length);
+ System.out.println(sendEntries3(256, size, AdaptiveBufAllocator.DEFAULT.newHandle()).length);
+ System.out.println(sendEntries4(256, size).length);
+
+ Options opt = new OptionsBuilder() //
+ .include(AppendEntriesBenchmark.class.getSimpleName()) //
+ .warmupIterations(1) //
+ .warmupTime(TimeValue.seconds(5)) //
+ .measurementIterations(3) //
+ .measurementTime(TimeValue.seconds(10)) //
+ .threads(8) //
+ .forks(1) //
+ .build();
+
+ new Runner(opt).run();
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.Throughput)
+ @OutputTimeUnit(TimeUnit.MILLISECONDS)
+ public void copy() {
+ sendEntries1(this.entryCount, this.sizeOfEntry);
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.Throughput)
+ @OutputTimeUnit(TimeUnit.MILLISECONDS)
+ public void pooled() {
+ sendEntries2(this.entryCount, this.sizeOfEntry);
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.Throughput)
+ @OutputTimeUnit(TimeUnit.MILLISECONDS)
+ public void adaptiveAndPooled() {
+ sendEntries3(this.entryCount, this.sizeOfEntry, handleThreadLocal.get());
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.Throughput)
+ @OutputTimeUnit(TimeUnit.MILLISECONDS)
+ public void zeroCopy() {
+ sendEntries4(this.entryCount, this.sizeOfEntry);
+ }
+
+ private static byte[] sendEntries1(final int entryCount, final int sizeOfEntry) {
+ final AppendEntriesRequest.Builder rb = AppendEntriesRequest.newBuilder();
+ fillCommonFields(rb);
+ final ByteBufferCollector dataBuffer = ByteBufferCollector.allocate();
+ for (int i = 0; i < entryCount; i++) {
+ final byte[] bytes = new byte[sizeOfEntry];
+ ThreadLocalRandom.current().nextBytes(bytes);
+ final ByteBuffer buf = ByteBuffer.wrap(bytes);
+ dataBuffer.put(buf.slice());
+ }
+ final ByteBuffer buf = dataBuffer.getBuffer();
+ buf.flip();
+ rb.setData(new ByteString(buf));
+ return rb.build().toByteArray();
+ }
+
+ private static byte[] sendEntries2(final int entryCount, final int sizeOfEntry) {
+ final AppendEntriesRequest.Builder rb = AppendEntriesRequest.newBuilder();
+ fillCommonFields(rb);
+ final ByteBufferCollector dataBuffer = ByteBufferCollector.allocateByRecyclers();
+ try {
+ for (int i = 0; i < entryCount; i++) {
+ final byte[] bytes = new byte[sizeOfEntry];
+ ThreadLocalRandom.current().nextBytes(bytes);
+ final ByteBuffer buf = ByteBuffer.wrap(bytes);
+ dataBuffer.put(buf.slice());
+ }
+ final ByteBuffer buf = dataBuffer.getBuffer();
+ buf.flip();
+ rb.setData(new ByteString(buf));
+ return rb.build().toByteArray();
+ } finally {
+ RecycleUtil.recycle(dataBuffer);
+ }
+ }
+
+ private static byte[] sendEntries3(final int entryCount, final int sizeOfEntry,
+ AdaptiveBufAllocator.Handle allocator) {
+ final AppendEntriesRequest.Builder rb = AppendEntriesRequest.newBuilder();
+ fillCommonFields(rb);
+ final ByteBufferCollector dataBuffer = allocator.allocateByRecyclers();
+ try {
+ for (int i = 0; i < entryCount; i++) {
+ final byte[] bytes = new byte[sizeOfEntry];
+ ThreadLocalRandom.current().nextBytes(bytes);
+ final ByteBuffer buf = ByteBuffer.wrap(bytes);
+ dataBuffer.put(buf.slice());
+ }
+ final ByteBuffer buf = dataBuffer.getBuffer();
+ buf.flip();
+ final int remaining = buf.remaining();
+ allocator.record(remaining);
+ rb.setData(new ByteString(buf));
+ return rb.build().toByteArray();
+ } finally {
+ RecycleUtil.recycle(dataBuffer);
+ }
+ }
+
+ private static byte[] sendEntries4(final int entryCount, final int sizeOfEntry) {
+ final AppendEntriesRequest.Builder rb = AppendEntriesRequest.newBuilder();
+ fillCommonFields(rb);
+ final RecyclableByteBufferList dataBuffer = RecyclableByteBufferList.newInstance();
+
+ try {
+ for (int i = 0; i < entryCount; i++) {
+ final byte[] bytes = new byte[sizeOfEntry];
+ ThreadLocalRandom.current().nextBytes(bytes);
+ final ByteBuffer buf = ByteBuffer.wrap(bytes);
+ dataBuffer.add(buf.slice());
+ }
+ rb.setData(RecyclableByteBufferList.concatenate(dataBuffer));
+ return rb.build().toByteArray();
+ } finally {
+ RecycleUtil.recycle(dataBuffer);
+ }
+ }
+
+ private static void fillCommonFields(final AppendEntriesRequest.Builder rb) {
+ rb.setTerm(1) //
+ .setGroupId("1") //
+ .setServerId("test") //
+ .setPeerId("127.0.0.1:8080") //
+ .setPrevLogIndex(2) //
+ .setPrevLogTerm(3) //
+ .setCommittedIndex(4);
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/RpcResponseFactoryTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/RpcResponseFactoryTest.java
new file mode 100644
index 0000000..7485244
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/RpcResponseFactoryTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc;
+
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse;
+import com.alipay.sofa.jraft.util.RpcFactoryHelper;
+
+import static org.junit.Assert.assertEquals;
+
+public class RpcResponseFactoryTest {
+ @Test
+ public void testNewResponseFromStatus() {
+ ErrorResponse response = (ErrorResponse) RpcFactoryHelper.responseFactory().newResponse(null, Status.OK());
+ assertEquals(response.getErrorCode(), 0);
+ assertEquals(response.getErrorMsg(), "");
+ }
+
+ @Test
+ public void testNewResponseWithErrorStatus() {
+ ErrorResponse response = (ErrorResponse) RpcFactoryHelper.responseFactory().newResponse(null,
+ new Status(300, "test"));
+ assertEquals(response.getErrorCode(), 300);
+ assertEquals(response.getErrorMsg(), "test");
+ }
+
+ @Test
+ public void testNewResponseWithVaridicArgs() {
+ ErrorResponse response = (ErrorResponse) RpcFactoryHelper.responseFactory().newResponse(null, 300,
+ "hello %s %d", "world", 99);
+ assertEquals(response.getErrorCode(), 300);
+ assertEquals(response.getErrorMsg(), "hello world 99");
+ }
+
+ @Test
+ public void testNewResponseWithArgs() {
+ ErrorResponse response = (ErrorResponse) RpcFactoryHelper.responseFactory().newResponse(null, 300,
+ "hello world");
+ assertEquals(response.getErrorCode(), 300);
+ assertEquals(response.getErrorMsg(), "hello world");
+ }
+
+ @Test
+ public void testNewResponseWithRaftError() {
+ ErrorResponse response = (ErrorResponse) RpcFactoryHelper.responseFactory().newResponse(null, RaftError.EAGAIN,
+ "hello world");
+ assertEquals(response.getErrorCode(), RaftError.EAGAIN.getNumber());
+ assertEquals(response.getErrorMsg(), "hello world");
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/impl/FutureTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/impl/FutureTest.java
new file mode 100644
index 0000000..ebd2b2a
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/impl/FutureTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl;
+
+import java.io.IOException;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class FutureTest {
+
+ private static final Logger log = LoggerFactory.getLogger(FutureImpl.class);
+
+ private static final class NotifyFutureRunner implements Runnable {
+ FutureImpl<Boolean> future;
+ long sleepTime;
+ Throwable throwable;
+
+ public NotifyFutureRunner(FutureImpl<Boolean> future, long sleepTime, Throwable throwable) {
+ super();
+ this.future = future;
+ this.sleepTime = sleepTime;
+ this.throwable = throwable;
+ }
+
+ @Override
+ public void run() {
+ try {
+ Thread.sleep(this.sleepTime);
+ if (this.throwable != null) {
+ this.future.failure(this.throwable);
+ } else {
+ this.future.setResult(true);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Test
+ public void testGet() throws Exception {
+ FutureImpl<Boolean> future = new FutureImpl<Boolean>();
+ new Thread(new NotifyFutureRunner(future, 2000, null)).start();
+ boolean result = future.get();
+ assertTrue(result);
+ assertTrue(future.isDone());
+ assertFalse(future.isCancelled());
+ }
+
+ @Test
+ public void testGetImmediately() throws Exception {
+ FutureImpl<Boolean> future = new FutureImpl<Boolean>();
+ future.setResult(true);
+ boolean result = future.get();
+ assertTrue(result);
+ assertTrue(future.isDone());
+ assertFalse(future.isCancelled());
+ }
+
+ @Test
+ public void testGetException() throws Exception {
+ FutureImpl<Boolean> future = new FutureImpl<Boolean>();
+ new Thread(new NotifyFutureRunner(future, 2000, new IOException("hello"))).start();
+ try {
+ future.get();
+ fail();
+ } catch (ExecutionException e) {
+ assertEquals("hello", e.getCause().getMessage());
+
+ }
+ assertTrue(future.isDone());
+ assertFalse(future.isCancelled());
+
+ }
+
+ @Test
+ public void testCancel() throws Exception {
+ final FutureImpl<Boolean> future = new FutureImpl<Boolean>();
+ new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ Thread.sleep(3000);
+ future.cancel(true);
+ } catch (Exception e) {
+ log.error(e.getMessage(), e);
+ }
+ }
+ }).start();
+ try {
+ future.get();
+ fail();
+ } catch (CancellationException e) {
+ assertTrue(true);
+
+ }
+ assertTrue(future.isDone());
+ assertTrue(future.isCancelled());
+ }
+
+ @Test
+ public void testGetTimeout() throws Exception {
+ FutureImpl<Boolean> future = new FutureImpl<Boolean>();
+ try {
+ future.get(1000, TimeUnit.MILLISECONDS);
+ fail();
+ } catch (TimeoutException e) {
+ assertTrue(true);
+ }
+ }
+}
\ No newline at end of file
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/impl/PingRequestProcessorTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/impl/PingRequestProcessorTest.java
new file mode 100644
index 0000000..e7d137e
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/impl/PingRequestProcessorTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl;
+
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse;
+import com.alipay.sofa.jraft.test.MockAsyncContext;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+import static org.junit.Assert.assertEquals;
+
+public class PingRequestProcessorTest {
+
+ @Test
+ public void testHandlePing() throws Exception {
+ PingRequestProcessor processor = new PingRequestProcessor();
+ MockAsyncContext ctx = new MockAsyncContext();
+ processor.handleRequest(ctx, TestUtils.createPingRequest());
+ ErrorResponse response = (ErrorResponse) ctx.getResponseObject();
+ assertEquals(0, response.getErrorCode());
+ }
+}
diff --git a/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/AbstractCliRequestProcessorTest.java b/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/AbstractCliRequestProcessorTest.java
new file mode 100644
index 0000000..a56933e
--- /dev/null
+++ b/modules/raft/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/AbstractCliRequestProcessorTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.cli;
+
+import com.alipay.sofa.jraft.rpc.Message;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.NodeManager;
+import com.alipay.sofa.jraft.entity.NodeId;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.test.MockAsyncContext;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public abstract class AbstractCliRequestProcessorTest<T extends Message> {
+ @Mock
+ private Node node;
+ private final String groupId = "test";
+ private final String peerIdStr = "localhost:8081";
+ protected MockAsyncContext asyncContext;
+
+ public abstract T createRequest(String groupId, PeerId peerId);
+
+ public abstract BaseCliRequestProcessor<T> newProcessor();
+
+ public abstract void verify(String interest, Node node, ArgumentCaptor<Closure> doneArg);
+
+ public void mockNodes(final int n) {
+ ArrayList<PeerId> peers = new ArrayList<>();
+ for (int i = 0; i < n; i++) {
+ peers.add(JRaftUtils.getPeerId("localhost:" + (8081 + i)));
+ }
+ List<PeerId> learners = new ArrayList<>();
+ for (int i = 0; i < n; i++) {
+ learners.add(JRaftUtils.getPeerId("learner:" + (8081 + i)));
+ }
+ Mockito.when(this.node.listPeers()).thenReturn(peers);
+ Mockito.when(this.node.listLearners()).thenReturn(learners);
+ }
+
+ @Before
+ public void setup() {
+ this.asyncContext = new MockAsyncContext();
+ }
+
+ @After
+ public void teardown() {
+ NodeManager.getInstance().clear();
+ }
+
+ @Test
+ public void testHandleRequest() {
+ this.mockNodes(3);
+ Mockito.when(this.node.getGroupId()).thenReturn(this.groupId);
... 7068 lines suppressed ...