You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ib...@apache.org on 2022/10/05 06:43:26 UTC
[ignite-3] branch main updated: IGNITE-17799 Intergrate onLeaderElected callback into CMG Manager (#1155)
This is an automated email from the ASF dual-hosted git repository.
ibessonov pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new ab417701e9 IGNITE-17799 Intergrate onLeaderElected callback into CMG Manager (#1155)
ab417701e9 is described below
commit ab417701e9776e22648d4dd5d8090b216abaf217
Author: Alexander Polovtcev <al...@gmail.com>
AuthorDate: Wed Oct 5 09:43:21 2022 +0300
IGNITE-17799 Intergrate onLeaderElected callback into CMG Manager (#1155)
---
.../cluster/management/ItClusterManagerTest.java | 78 ++++++++---
.../management/ClusterManagementGroupManager.java | 153 ++++++++++-----------
.../java/org/apache/ignite/internal/raft/Loza.java | 36 ++++-
.../cluster/management/ItClusterInitTest.java | 2 +-
.../runner/app/ItIgniteNodeRestartTest.java | 2 +-
5 files changed, 164 insertions(+), 107 deletions(-)
diff --git a/modules/cluster-management/src/integrationTest/java/org/apache/ignite/internal/cluster/management/ItClusterManagerTest.java b/modules/cluster-management/src/integrationTest/java/org/apache/ignite/internal/cluster/management/ItClusterManagerTest.java
index a278c2d0a7..509d1036bc 100644
--- a/modules/cluster-management/src/integrationTest/java/org/apache/ignite/internal/cluster/management/ItClusterManagerTest.java
+++ b/modules/cluster-management/src/integrationTest/java/org/apache/ignite/internal/cluster/management/ItClusterManagerTest.java
@@ -45,6 +45,7 @@ import org.apache.ignite.network.ClusterNode;
import org.apache.ignite.network.NetworkAddress;
import org.apache.ignite.network.StaticNodeFinder;
import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -71,7 +72,7 @@ public class ItClusterManagerTest {
.mapToObj(i -> new NetworkAddress("localhost", PORT_BASE + i))
.collect(toList());
- StaticNodeFinder nodeFinder = new StaticNodeFinder(addrs);
+ var nodeFinder = new StaticNodeFinder(addrs);
for (int i = 0; i < numNodes; ++i) {
var node = new MockNode(testInfo, addrs.get(i), nodeFinder, workDir.resolve("node" + i));
@@ -94,6 +95,18 @@ public class ItClusterManagerTest {
}
}
+ private MockNode addNodeToCluster(TestInfo testInfo) throws IOException {
+ var addr = new NetworkAddress("localhost", PORT_BASE + cluster.size());
+
+ var nodeFinder = new StaticNodeFinder(clusterNodeAddresses());
+
+ var node = new MockNode(testInfo, addr, nodeFinder, workDir.resolve("node" + cluster.size()));
+
+ cluster.add(node);
+
+ return node;
+ }
+
/**
* Tests initial cluster setup.
*/
@@ -209,6 +222,10 @@ public class ItClusterManagerTest {
// successful init
clusterManager.initCluster(List.of(cluster.get(0).name()), List.of(), "cluster");
+ for (MockNode node : cluster) {
+ assertThat(node.clusterManager().joinFuture(), willCompleteSuccessfully());
+ }
+
// different node
assertThrowsWithCause(
() -> clusterManager.initCluster(List.of(cluster.get(1).name()), List.of(), "cluster"),
@@ -260,13 +277,7 @@ public class ItClusterManagerTest {
initCluster(cmgNodes, cmgNodes);
// create and start a new node
- var addr = new NetworkAddress("localhost", PORT_BASE + cluster.size());
-
- var nodeFinder = new StaticNodeFinder(Arrays.asList(clusterNodeAddresses()));
-
- var node = new MockNode(testInfo, addr, nodeFinder, workDir.resolve("node" + cluster.size()));
-
- cluster.add(node);
+ MockNode node = addNodeToCluster(testInfo);
node.start();
@@ -340,7 +351,7 @@ public class ItClusterManagerTest {
* Tests a scenario when a node starts joining a cluster having a CMG leader, but finishes the join after the CMG leader changed.
*/
@Test
- void testJoinLeaderChange(TestInfo testInfo) throws Exception {
+ void testLeaderChangeDuringJoin(TestInfo testInfo) throws Exception {
// Start a cluster of 3 nodes so that the CMG leader node could be stopped later.
startCluster(3, testInfo);
@@ -350,18 +361,12 @@ public class ItClusterManagerTest {
initCluster(cmgNodes, cmgNodes);
// Start a new node, but do not send the JoinReadyCommand.
- var addr = new NetworkAddress("localhost", PORT_BASE + cluster.size());
-
- var nodeFinder = new StaticNodeFinder(Arrays.asList(clusterNodeAddresses()));
-
- var node = new MockNode(testInfo, addr, nodeFinder, workDir.resolve("node" + cluster.size()));
+ MockNode node = addNodeToCluster(testInfo);
node.startComponents();
assertThat(node.clusterManager().joinFuture(), willCompleteSuccessfully());
- cluster.add(node);
-
// Find the CMG leader and stop it
MockNode leaderNode = cluster.stream()
.filter(n -> {
@@ -374,12 +379,49 @@ public class ItClusterManagerTest {
.findAny()
.orElseThrow();
+ leaderNode.beforeNodeStop();
leaderNode.stop();
// Issue the JoinReadCommand on the joining node. It is expected that the joining node is still treated as validated.
assertThat(node.clusterManager().onJoinReady(), willCompleteSuccessfully());
}
+ @Disabled("https://issues.apache.org/jira/browse/IGNITE-17814")
+ @Test
+ void testLeaderChangeBeforeJoin(TestInfo testInfo) throws Exception {
+ // Start a cluster of 3 nodes so that the CMG leader node could be stopped later.
+ startCluster(3, testInfo);
+
+ String[] cmgNodes = clusterNodeNames();
+
+ // Start the CMG on all 3 nodes.
+ initCluster(cmgNodes, cmgNodes);
+
+ // Find the CMG leader and stop it
+ MockNode leaderNode = cluster.stream()
+ .filter(n -> {
+ CompletableFuture<Boolean> isLeader = n.clusterManager().isCmgLeader();
+
+ assertThat(isLeader, willCompleteSuccessfully());
+
+ return isLeader.join();
+ })
+ .findAny()
+ .orElseThrow();
+
+ leaderNode.beforeNodeStop();
+ leaderNode.stop();
+
+ // Start a new node.
+ MockNode node = addNodeToCluster(testInfo);
+
+ node.start();
+
+ Thread.sleep(10000);
+
+ assertThat(node.clusterManager().joinFuture(), willCompleteSuccessfully());
+ }
+
private ClusterNode[] currentPhysicalTopology() {
return cluster.stream()
.map(MockNode::localMember)
@@ -392,11 +434,11 @@ public class ItClusterManagerTest {
.toArray(String[]::new);
}
- private NetworkAddress[] clusterNodeAddresses() {
+ private List<NetworkAddress> clusterNodeAddresses() {
return cluster.stream()
.map(MockNode::localMember)
.map(ClusterNode::address)
- .toArray(NetworkAddress[]::new);
+ .collect(toList());
}
private void waitForLogicalTopology() throws InterruptedException {
diff --git a/modules/cluster-management/src/main/java/org/apache/ignite/internal/cluster/management/ClusterManagementGroupManager.java b/modules/cluster-management/src/main/java/org/apache/ignite/internal/cluster/management/ClusterManagementGroupManager.java
index 3db35e4bfe..48bbe2c507 100644
--- a/modules/cluster-management/src/main/java/org/apache/ignite/internal/cluster/management/ClusterManagementGroupManager.java
+++ b/modules/cluster-management/src/main/java/org/apache/ignite/internal/cluster/management/ClusterManagementGroupManager.java
@@ -52,6 +52,7 @@ import org.apache.ignite.internal.logger.Loggers;
import org.apache.ignite.internal.manager.IgniteComponent;
import org.apache.ignite.internal.properties.IgniteProductVersion;
import org.apache.ignite.internal.raft.Loza;
+import org.apache.ignite.internal.raft.server.RaftGroupEventsListener;
import org.apache.ignite.internal.raft.server.RaftGroupOptions;
import org.apache.ignite.internal.thread.NamedThreadFactory;
import org.apache.ignite.internal.util.IgniteSpinBusyLock;
@@ -65,6 +66,8 @@ import org.apache.ignite.network.NetworkAddress;
import org.apache.ignite.network.NetworkMessage;
import org.apache.ignite.network.TopologyEventHandler;
import org.apache.ignite.network.TopologyService;
+import org.apache.ignite.raft.jraft.Status;
+import org.apache.ignite.raft.jraft.entity.PeerId;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.TestOnly;
@@ -225,26 +228,7 @@ public class ClusterManagementGroupManager implements IgniteComponent {
LOG.info("Local CMG state recovered, starting the CMG");
return startCmgRaftService(localState.cmgNodeNames())
- .thenCompose(service -> joinCluster(service, localState.clusterTag())
- .thenCompose(v -> service.isCurrentNodeLeader())
- .thenCompose(isLeader -> {
- if (!isLeader) {
- return completedFuture(service);
- }
-
- return service.readClusterState()
- .thenCompose(state -> {
- if (state == null) {
- // Raft state might not have been initialized in case of leader failure during cluster init
- // TODO: properly handle this case, see https://issues.apache.org/jira/browse/IGNITE-16819
-
- return failedFuture(new IllegalStateException("Cluster state is empty"));
- } else {
- return onLeaderElected(service, state).thenApply(v -> service);
- }
- });
- })
- );
+ .thenCompose(service -> joinCluster(service, localState.clusterTag()));
}
/**
@@ -307,21 +291,13 @@ public class ClusterManagementGroupManager implements IgniteComponent {
}
}
- private CompletableFuture<Void> doInit(CmgRaftService service, CmgInitMessage msg) {
+ private CompletableFuture<CmgRaftService> doInit(CmgRaftService service, CmgInitMessage msg) {
return service.initClusterState(createClusterState(msg))
.thenCompose(state -> {
var localState = new LocalState(state.cmgNodes(), state.clusterTag());
return localStateStorage.saveLocalState(localState)
- .thenCompose(v -> joinCluster(service, state.clusterTag()))
- .thenCompose(v -> service.isCurrentNodeLeader()
- .thenCompose(isLeader -> {
- if (isLeader) {
- return onLeaderElected(service, state);
- } else {
- return completedFuture(null);
- }
- }));
+ .thenCompose(v -> joinCluster(service, state.clusterTag()));
});
}
@@ -341,25 +317,27 @@ public class ClusterManagementGroupManager implements IgniteComponent {
* <li>Broadcasts the current CMG state to all nodes in the physical topology.</li>
* </ol>
*/
- private CompletableFuture<Void> onLeaderElected(CmgRaftService service, ClusterState state) {
+ private void onLeaderElected() {
LOG.info("CMG leader has been elected, executing onLeaderElected callback");
- return updateLogicalTopology(service)
- .whenComplete((v, e) -> {
- if (e == null) {
- LOG.info("onLeaderElected callback executed successfully");
+ raftServiceAfterJoin()
+ .thenCompose(this::updateLogicalTopology)
+ .thenAccept(service -> {
+ // Register a listener to send ClusterState messages to new nodes.
+ TopologyService topologyService = clusterService.topologyService();
- // Register a listener to send ClusterState messages to new nodes.
- TopologyService topologyService = clusterService.topologyService();
+ // TODO: remove listeners if leadership is lost, see https://issues.apache.org/jira/browse/IGNITE-16842
+ topologyService.addEventHandler(cmgLeaderTopologyEventHandler(service));
- // TODO: remove listeners if leadership is lost, see https://issues.apache.org/jira/browse/IGNITE-16842
- topologyService.addEventHandler(cmgLeaderTopologyEventHandler(service));
+ // Send the ClusterStateMessage to all members of the physical topology. We do not wait for the send operation
+ // because being unable to send ClusterState messages should not fail the CMG service startup.
+ sendClusterState(service, topologyService.allMembers());
- // Send the ClusterStateMessage to all members of the physical topology. We do not wait for the send operation
- // because being unable to send ClusterState messages should not fail the CMG service startup.
- sendClusterState(state, clusterService.topologyService().allMembers());
- } else {
- LOG.info("Error when executing onLeaderElected callback", e);
+ LOG.info("onLeaderElected callback executed successfully");
+ })
+ .whenComplete((v, e) -> {
+ if (e != null) {
+ LOG.warn("Error when executing onLeaderElected callback", e);
}
});
}
@@ -369,7 +347,7 @@ public class ClusterManagementGroupManager implements IgniteComponent {
* physical topology during the election. Newly appeared nodes will be added automatically after the new leader broadcasts the current
* cluster state.
*/
- private CompletableFuture<Void> updateLogicalTopology(CmgRaftService service) {
+ private CompletableFuture<CmgRaftService> updateLogicalTopology(CmgRaftService service) {
return service.logicalTopology()
.thenCompose(logicalTopology -> {
Set<String> physicalTopologyIds = clusterService.topologyService().allMembers()
@@ -382,7 +360,8 @@ public class ClusterManagementGroupManager implements IgniteComponent {
.collect(toUnmodifiableSet());
return nodesToRemove.isEmpty() ? completedFuture(null) : service.removeFromCluster(nodesToRemove);
- });
+ })
+ .thenApply(v -> service);
}
private void handleCancelInit(CancelInitMessage msg) {
@@ -435,7 +414,7 @@ public class ClusterManagementGroupManager implements IgniteComponent {
raftService = raftService
.handle((service, e) -> {
if (service != null && service.nodeNames().equals(state.cmgNodes())) {
- LOG.debug("ClusterStateMessage received, but the CMG service is already started");
+ LOG.info("ClusterStateMessage received, but the CMG service is already started");
return completedFuture(service);
}
@@ -453,10 +432,10 @@ public class ClusterManagementGroupManager implements IgniteComponent {
return CompletableFuture.<CmgRaftService>failedFuture(e);
}
- LOG.debug("CMG service could not be started on previous attempts. "
+ LOG.warn("CMG service could not be started on previous attempts. "
+ "Re-creating the CMG Raft service [reason={}]", e, e.getMessage());
} else {
- LOG.debug("CMG service started, but the cluster state is different. "
+ LOG.warn("CMG service started, but the cluster state is different. "
+ "Re-creating the CMG Raft service [localState={}, clusterState={}]",
service.nodeNames(), state.cmgNodes());
@@ -470,8 +449,9 @@ public class ClusterManagementGroupManager implements IgniteComponent {
}
}
- private CompletableFuture<Void> joinCluster(CmgRaftService service, ClusterTag clusterTag) {
+ private CompletableFuture<CmgRaftService> joinCluster(CmgRaftService service, ClusterTag clusterTag) {
return service.startJoinCluster(clusterTag)
+ .thenApply(v -> service)
.whenComplete((v, e) -> {
if (e == null) {
LOG.info("Successfully joined the cluster [name={}]", clusterTag.clusterName());
@@ -487,27 +467,44 @@ public class ClusterManagementGroupManager implements IgniteComponent {
* Starts the CMG Raft service using the provided node names as its peers.
*/
private CompletableFuture<CmgRaftService> startCmgRaftService(Collection<String> nodeNames) {
- // TODO: wait for nodes to appear, see https://issues.apache.org/jira/browse/IGNITE-16811
- List<ClusterNode> nodes = resolveNodes(clusterService, nodeNames);
-
try {
return raftManager
.prepareRaftGroup(
CMG_RAFT_GROUP_NAME,
- nodes,
+ resolveNodes(clusterService, nodeNames),
() -> {
clusterStateStorage.start();
return new CmgRaftGroupListener(clusterStateStorage);
},
+ this::createCmgRaftGroupEventsListener,
RaftGroupOptions.defaults()
)
.thenApply(service -> new CmgRaftService(service, clusterService));
- } catch (NodeStoppingException e) {
+ } catch (Exception e) {
return failedFuture(e);
}
}
+ private RaftGroupEventsListener createCmgRaftGroupEventsListener() {
+ return new RaftGroupEventsListener() {
+ @Override
+ public void onLeaderElected(long term) {
+ ClusterManagementGroupManager.this.onLeaderElected();
+ }
+
+ @Override
+ public void onNewPeersConfigurationApplied(List<PeerId> peers) {
+ // No-op.
+ }
+
+ @Override
+ public void onReconfigurationError(Status status, List<PeerId> peers, long term) {
+ // No-op.
+ }
+ };
+ }
+
/**
* Starts the CMG Raft service using the given {@code state} and persists it to the local storage.
*/
@@ -517,8 +514,7 @@ public class ClusterManagementGroupManager implements IgniteComponent {
var localState = new LocalState(state.cmgNodes(), state.clusterTag());
return localStateStorage.saveLocalState(localState)
- .thenCompose(v -> joinCluster(service, state.clusterTag()))
- .thenApply(v -> service);
+ .thenCompose(v -> joinCluster(service, state.clusterTag()));
});
}
@@ -553,34 +549,31 @@ public class ClusterManagementGroupManager implements IgniteComponent {
}
private void sendClusterState(CmgRaftService raftService, ClusterNode node) {
+ sendClusterState(raftService, List.of(node));
+ }
+
+ private void sendClusterState(CmgRaftService raftService, Collection<ClusterNode> nodes) {
raftService.readClusterState()
.thenAccept(state -> {
- if (state != null) {
- sendClusterState(state, node);
- } else {
- LOG.warn("Unable to send cluster state to a newly added node. Cluster state is empty [node={}]", node);
+ // Raft state might not have been initialized in case of leader failure during cluster init
+ // TODO: properly handle this case, see https://issues.apache.org/jira/browse/IGNITE-16819
+ if (state == null) {
+ throw new IllegalStateException("Cluster state is empty");
}
- });
- }
- private CompletableFuture<Void> sendClusterState(ClusterState clusterState, ClusterNode node) {
- NetworkMessage msg = msgFactory.clusterStateMessage()
- .clusterState(clusterState)
- .build();
+ NetworkMessage msg = msgFactory.clusterStateMessage()
+ .clusterState(state)
+ .build();
- return sendWithRetry(node, msg);
- }
-
- private CompletableFuture<Void> sendClusterState(ClusterState clusterState, Collection<ClusterNode> nodes) {
- NetworkMessage msg = msgFactory.clusterStateMessage()
- .clusterState(clusterState)
- .build();
-
- CompletableFuture<?>[] futures = nodes.stream()
- .map(node -> sendWithRetry(node, msg))
- .toArray(CompletableFuture[]::new);
-
- return CompletableFuture.allOf(futures);
+ for (ClusterNode node : nodes) {
+ sendWithRetry(node, msg);
+ }
+ })
+ .whenComplete((v, e) -> {
+ if (e != null) {
+ LOG.error("Unable to send cluster state", e);
+ }
+ });
}
private CompletableFuture<Void> sendWithRetry(ClusterNode node, NetworkMessage msg) {
diff --git a/modules/raft/src/main/java/org/apache/ignite/internal/raft/Loza.java b/modules/raft/src/main/java/org/apache/ignite/internal/raft/Loza.java
index 1da1d1e317..aa01a8a697 100644
--- a/modules/raft/src/main/java/org/apache/ignite/internal/raft/Loza.java
+++ b/modules/raft/src/main/java/org/apache/ignite/internal/raft/Loza.java
@@ -102,7 +102,7 @@ public class Loza implements IgniteComponent {
* The constructor.
*
* @param clusterNetSvc Cluster network service.
- * @param dataPath Data path.
+ * @param dataPath Data path.
*/
public Loza(ClusterService clusterNetSvc, RaftConfiguration raftConfiguration, Path dataPath) {
this.clusterNetSvc = clusterNetSvc;
@@ -150,8 +150,8 @@ public class Loza implements IgniteComponent {
* Creates a raft group service providing operations on a raft group. If {@code nodes} contains the current node, then raft group starts
* on the current node.
*
- * @param groupId Raft group id.
- * @param nodes Raft group nodes.
+ * @param groupId Raft group id.
+ * @param nodes Raft group nodes.
* @param lsnrSupplier Raft group listener supplier.
* @param groupOptions Options to apply to the group.
* @return Future representing pending completion of the operation.
@@ -162,13 +162,35 @@ public class Loza implements IgniteComponent {
List<ClusterNode> nodes,
Supplier<RaftGroupListener> lsnrSupplier,
RaftGroupOptions groupOptions
+ ) throws NodeStoppingException {
+ return prepareRaftGroup(groupId, nodes, lsnrSupplier, () -> RaftGroupEventsListener.noopLsnr, groupOptions);
+ }
+
+ /**
+ * Creates a raft group service providing operations on a raft group. If {@code nodes} contains the current node, then raft group starts
+ * on the current node.
+ *
+ * @param groupId Raft group id.
+ * @param nodes Raft group nodes.
+ * @param lsnrSupplier Raft group listener supplier.
+ * @param raftGrpEvtsLsnrSupplier Raft group events listener supplier.
+ * @param groupOptions Options to apply to the group.
+ * @return Future representing pending completion of the operation.
+ * @throws NodeStoppingException If node stopping intention was detected.
+ */
+ public CompletableFuture<RaftGroupService> prepareRaftGroup(
+ String groupId,
+ List<ClusterNode> nodes,
+ Supplier<RaftGroupListener> lsnrSupplier,
+ Supplier<RaftGroupEventsListener> raftGrpEvtsLsnrSupplier,
+ RaftGroupOptions groupOptions
) throws NodeStoppingException {
if (!busyLock.enterBusy()) {
throw new NodeStoppingException();
}
try {
- return prepareRaftGroupInternal(groupId, nodes, lsnrSupplier, () -> RaftGroupEventsListener.noopLsnr, groupOptions);
+ return prepareRaftGroupInternal(groupId, nodes, lsnrSupplier, raftGrpEvtsLsnrSupplier, groupOptions);
} finally {
busyLock.leaveBusy();
}
@@ -177,9 +199,9 @@ public class Loza implements IgniteComponent {
/**
* Internal method to a raft group creation.
*
- * @param groupId Raft group id.
- * @param nodes Raft group nodes.
- * @param lsnrSupplier Raft group listener supplier.
+ * @param groupId Raft group id.
+ * @param nodes Raft group nodes.
+ * @param lsnrSupplier Raft group listener supplier.
* @param raftGrpEvtsLsnrSupplier Raft group events listener supplier.
* @param groupOptions Options to apply to the group.
* @return Future representing pending completion of the operation.
diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/cluster/management/ItClusterInitTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/cluster/management/ItClusterInitTest.java
index 8bae78d260..69adf7ea80 100644
--- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/cluster/management/ItClusterInitTest.java
+++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/cluster/management/ItClusterInitTest.java
@@ -65,7 +65,7 @@ public class ItClusterInitTest extends IgniteAbstractTest {
// init is idempotent
IgnitionManager.init(nodeName, List.of(nodeName), "cluster");
- // TODO: remove 'waitForCondition' after https://issues.apache.org/jira/browse/IGNITE-16811 is fixed
+ // TODO: remove 'waitForCondition' after https://issues.apache.org/jira/browse/IGNITE-17814 is fixed
assertTrue(
waitForCondition(() -> {
// init should fail if the list of nodes is different
diff --git a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java
index 69cd50d6a1..14eec473c4 100644
--- a/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java
+++ b/modules/runner/src/integrationTest/java/org/apache/ignite/internal/runner/app/ItIgniteNodeRestartTest.java
@@ -677,7 +677,7 @@ public class ItIgniteNodeRestartTest extends IgniteAbstractTest {
* @param testInfo Test information object.
*/
@Test
- @Disabled("https://issues.apache.org/jira/browse/IGNITE-16811")
+ @Disabled("https://issues.apache.org/jira/browse/IGNITE-17814")
public void testTwoNodesRestartReverse(TestInfo testInfo) {
twoNodesRestart(testInfo, false);
}