You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vp...@apache.org on 2023/08/22 13:45:48 UTC
[ignite-3] branch main updated: IGNITE-20257 fix flaky ItCliServiceTest#testSnapshot test @alievmirza (#2469)
This is an automated email from the ASF dual-hosted git repository.
vpyatkov pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new 3afaddcda9 IGNITE-20257 fix flaky ItCliServiceTest#testSnapshot test @alievmirza (#2469)
3afaddcda9 is described below
commit 3afaddcda91d52cd31417afac43ec82078b791fa
Author: Mirza Aliev <al...@gmail.com>
AuthorDate: Tue Aug 22 17:45:42 2023 +0400
IGNITE-20257 fix flaky ItCliServiceTest#testSnapshot test @alievmirza (#2469)
---
.../ignite/raft/jraft/core/ItCliServiceTest.java | 20 +++++++++++++++++---
.../apache/ignite/raft/jraft/core/ItNodeTest.java | 8 ++++----
.../ignite/raft/jraft/core/MockStateMachine.java | 13 +++++++------
3 files changed, 28 insertions(+), 13 deletions(-)
diff --git a/modules/raft/src/integrationTest/java/org/apache/ignite/raft/jraft/core/ItCliServiceTest.java b/modules/raft/src/integrationTest/java/org/apache/ignite/raft/jraft/core/ItCliServiceTest.java
index 82b8c0cae6..9aa7c98948 100644
--- a/modules/raft/src/integrationTest/java/org/apache/ignite/raft/jraft/core/ItCliServiceTest.java
+++ b/modules/raft/src/integrationTest/java/org/apache/ignite/raft/jraft/core/ItCliServiceTest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.raft.jraft.core;
import static java.lang.Thread.sleep;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.stream.Collectors.toList;
+import static org.apache.ignite.raft.jraft.core.ItNodeTest.waitForTopologyOnEveryNode;
import static org.apache.ignite.raft.jraft.core.TestCluster.ELECTION_TIMEOUT_MILLIS;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -66,6 +67,7 @@ import org.apache.ignite.raft.jraft.util.ExecutorServiceHelper;
import org.apache.ignite.utils.ClusterServiceTestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.RepeatedTest;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -84,6 +86,8 @@ public class ItCliServiceTest {
private TestCluster cluster;
+ private ClusterService clientSvc;
+
private final String groupId = "CliServiceTest";
private CliService cliService;
@@ -131,7 +135,7 @@ public class ItCliServiceTest {
.map(p -> new NetworkAddress(TestUtils.getLocalAddress(), p.getPort()))
.collect(toList());
- ClusterService clientSvc = ClusterServiceTestUtils.clusterService(
+ clientSvc = ClusterServiceTestUtils.clusterService(
testInfo,
TestUtils.INIT_PORT - 1,
new StaticNodeFinder(addressList)
@@ -153,6 +157,7 @@ public class ItCliServiceTest {
@AfterEach
public void teardown(TestInfo testInfo) throws Exception {
+ LOG.info(">>>>>>>>>>>>>>> Teardown started: " + testInfo.getDisplayName());
cliService.shutdown();
cluster.stopAll();
ExecutorServiceHelper.shutdownAndAwaitTermination(clientExecutor);
@@ -353,16 +358,21 @@ public class ItCliServiceTest {
public void testSnapshot() throws Exception {
sendTestTaskAndWait(cluster.getLeader(), 0);
assertEquals(5, cluster.getFsms().size());
+
+ waitForTopologyOnEveryNode(6, cluster);
+
+ assertTrue(waitForTopology(clientSvc, 6,10_000));
+
for (MockStateMachine fsm : cluster.getFsms()) {
assertEquals(0, fsm.getSaveSnapshotTimes());
}
for (PeerId peer : conf) {
- assertTrue(cliService.snapshot(groupId, peer).isOk());
+ assertTrue(cliService.snapshot(groupId, peer).isOk(), "Failed to trigger snapshot from peer = " + peer);
}
for (PeerId peer : conf.getLearners()) {
- assertTrue(cliService.snapshot(groupId, peer).isOk());
+ assertTrue(cliService.snapshot(groupId, peer).isOk(), "Failed to trigger snapshot from learner = " + peer);
}
for (MockStateMachine fsm : cluster.getFsms()) {
@@ -370,6 +380,10 @@ public class ItCliServiceTest {
}
}
+ private static boolean waitForTopology(ClusterService cluster, int expected, int timeout) {
+ return waitForCondition(() -> cluster.topologyService().allMembers().size() >= expected, timeout);
+ }
+
@Test
public void testGetPeers() throws Exception {
PeerId leader = cluster.getLeader().getNodeId().getPeerId();
diff --git a/modules/raft/src/integrationTest/java/org/apache/ignite/raft/jraft/core/ItNodeTest.java b/modules/raft/src/integrationTest/java/org/apache/ignite/raft/jraft/core/ItNodeTest.java
index 523fe6d8f2..6f253162d2 100644
--- a/modules/raft/src/integrationTest/java/org/apache/ignite/raft/jraft/core/ItNodeTest.java
+++ b/modules/raft/src/integrationTest/java/org/apache/ignite/raft/jraft/core/ItNodeTest.java
@@ -2932,7 +2932,7 @@ public class ItNodeTest {
assertTrue(cluster.start(peer, false, 300));
}
- waitForTopologyOnEveryNode(numPeers);
+ waitForTopologyOnEveryNode(numPeers, cluster);
for (int i = 0; i < 9; i++) {
leader = cluster.waitAndGetLeader();
@@ -3027,7 +3027,7 @@ public class ItNodeTest {
// Wait until every node sees every other node, otherwise
// changePeersAsync can fail.
- waitForTopologyOnEveryNode(numPeers);
+ waitForTopologyOnEveryNode(numPeers, cluster);
for (int i = 0; i < 4; i++) {
leader = cluster.getLeader();
@@ -3767,7 +3767,7 @@ public class ItNodeTest {
* @param timeout
* @return
*/
- private boolean waitForTopology(TestCluster cluster, PeerId peerId, int expected, long timeout) {
+ private static boolean waitForTopology(TestCluster cluster, PeerId peerId, int expected, long timeout) {
RaftGroupService grp = cluster.getServer(peerId);
if (grp == null) {
@@ -3972,7 +3972,7 @@ public class ItNodeTest {
}
}
- private void waitForTopologyOnEveryNode(int count) {
+ static void waitForTopologyOnEveryNode(int count, TestCluster cluster) {
cluster.getAllNodes().forEach(peerId -> {
assertTrue(waitForTopology(cluster, peerId, count, TimeUnit.SECONDS.toMillis(10)));
});
diff --git a/modules/raft/src/testFixtures/java/org/apache/ignite/raft/jraft/core/MockStateMachine.java b/modules/raft/src/testFixtures/java/org/apache/ignite/raft/jraft/core/MockStateMachine.java
index c63b0ad96c..f256fa8c69 100644
--- a/modules/raft/src/testFixtures/java/org/apache/ignite/raft/jraft/core/MockStateMachine.java
+++ b/modules/raft/src/testFixtures/java/org/apache/ignite/raft/jraft/core/MockStateMachine.java
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@@ -51,8 +52,8 @@ public class MockStateMachine extends StateMachineAdapter {
private volatile long snapshotIndex = -1L;
private final List<ByteBuffer> logs = new ArrayList<>();
private final PeerId peerId;
- private volatile int saveSnapshotTimes;
- private volatile int loadSnapshotTimes;
+ private final AtomicInteger saveSnapshotTimes = new AtomicInteger(0);
+ private final AtomicInteger loadSnapshotTimes= new AtomicInteger(0);
public PeerId getPeerId() {
return this.peerId;
@@ -64,11 +65,11 @@ public class MockStateMachine extends StateMachineAdapter {
}
public int getSaveSnapshotTimes() {
- return this.saveSnapshotTimes;
+ return this.saveSnapshotTimes.get();
}
public int getLoadSnapshotTimes() {
- return this.loadSnapshotTimes;
+ return this.loadSnapshotTimes.get();
}
public int getOnStartFollowingTimes() {
@@ -136,7 +137,7 @@ public class MockStateMachine extends StateMachineAdapter {
@Override
public void onSnapshotSave(final SnapshotWriter writer, final Closure done) {
- this.saveSnapshotTimes++;
+ this.saveSnapshotTimes.incrementAndGet();
final String path = writer.getPath() + File.separator + "data";
final File file = new File(path);
try (FileOutputStream fout = new FileOutputStream(file);
@@ -168,7 +169,7 @@ public class MockStateMachine extends StateMachineAdapter {
public boolean onSnapshotLoad(final SnapshotReader reader) {
SnapshotMeta meta = reader.load();
this.lastAppliedIndex.set(meta.lastIncludedIndex());
- this.loadSnapshotTimes++;
+ this.loadSnapshotTimes.incrementAndGet();
final String path = reader.getPath() + File.separator + "data";
final File file = new File(path);
if (!file.exists()) {