You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sk...@apache.org on 2026/03/03 11:47:04 UTC
(ignite-3) branch main updated: IGNITE-27714 Complete RAFT client (#7507)
This is an automated email from the ASF dual-hosted git repository.
sk0x50 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new 0cc773ce878 IGNITE-27714 Complete RAFT client (#7507)
0cc773ce878 is described below
commit 0cc773ce87851042f1caa262c349a92920ae1427
Author: Cyrill <cy...@gmail.com>
AuthorDate: Tue Mar 3 14:46:59 2026 +0300
IGNITE-27714 Complete RAFT client (#7507)
Co-authored-by: Kirill Sizov <si...@gmail.com>
---
.../raft/service/TimeAwareRaftGroupService.java | 108 +++--
.../raft/client/LeaderAvailabilityState.java | 113 ++++-
.../ignite/internal/raft/client/PeerTracking.java | 30 ++
.../PhysicalTopologyAwareRaftGroupService.java | 452 ++++++++++++------
.../internal/raft/client/RaftCommandExecutor.java | 526 +++++++++++++--------
.../ignite/internal/raft/client/RaftPeerUtils.java | 77 +++
.../ignite/internal/raft/client/RetryContext.java | 8 +-
.../raft/client/RetryExecutionStrategy.java | 66 +++
.../internal/raft/client/RetryPeerResult.java | 56 +++
.../raft/client/SubscriptionMessageSender.java | 140 ++++++
.../internal/raft/client/TargetPeerStrategy.java | 41 ++
.../raft/client/LeaderAvailabilityStateTest.java | 147 ++++--
...ysicalTopologyAwareRaftGroupServiceRunTest.java | 413 ++++++++++++----
.../PhysicalTopologyAwareRaftGroupServiceTest.java | 18 +-
...ilablePartitionsRecoveryByFilterUpdateTest.java | 6 +-
.../ItHighAvailablePartitionsRecoveryTest.java | 12 +-
16 files changed, 1647 insertions(+), 566 deletions(-)
diff --git a/modules/raft-api/src/main/java/org/apache/ignite/internal/raft/service/TimeAwareRaftGroupService.java b/modules/raft-api/src/main/java/org/apache/ignite/internal/raft/service/TimeAwareRaftGroupService.java
index e43bcfc0e5d..25c0e3f8063 100644
--- a/modules/raft-api/src/main/java/org/apache/ignite/internal/raft/service/TimeAwareRaftGroupService.java
+++ b/modules/raft-api/src/main/java/org/apache/ignite/internal/raft/service/TimeAwareRaftGroupService.java
@@ -25,23 +25,28 @@ import org.apache.ignite.internal.network.ClusterService;
import org.apache.ignite.internal.raft.Command;
import org.apache.ignite.internal.raft.Peer;
import org.apache.ignite.internal.raft.PeersAndLearners;
-import org.apache.ignite.internal.raft.configuration.RaftConfiguration;
import org.apache.ignite.internal.replicator.ReplicationGroupId;
import org.jetbrains.annotations.Nullable;
/**
- * A service providing operations on a replication group with operations timeout.
+ * A service providing operations on a replication group with explicit timeout control.
*
- * <p>Most of operations require a known group leader. The group leader can be refreshed at any time by calling {@link #refreshLeader()}
- * method, otherwise it will happen automatically on a first call.
+ * <p>Most of operations require a known group leader. The group leader can be refreshed at any time by calling
+ * {@link #refreshLeader(long)} method, otherwise it will happen automatically on a first call.
*
* <p>If a leader has been changed while the operation in progress, the operation will be transparently retried until timeout is reached.
* The current leader will be refreshed automatically (maybe several times) in the process.
*
- * <p>Each asynchronous method (returning a future) uses a default timeout to finish, see {@link RaftConfiguration#retryTimeoutMillis()}.
- * If a result is not available within the timeout, the future will be completed with a {@link TimeoutException}
+ * <p>Each asynchronous method takes a {@code timeoutMillis} parameter with the following semantics:
+ * <ul>
+ * <li>{@code 0} - single attempt without retries.</li>
+ * <li>{@code Long.MAX_VALUE} - infinite wait.</li>
+ * <li>negative values - treated as infinite for compatibility.</li>
+ * <li>positive values - bounded wait up to the specified timeout.</li>
+ * </ul>
+ * If a result is not available within the timeout, the future will be completed with a {@link TimeoutException}.
*
- * <p>If an error is occurred during operation execution, the future will be completed with the corresponding IgniteException having an
+ * <p>If an error occurs during operation execution, the future will be completed with the corresponding IgniteException having an
* error code and a related message.
*
* <p>All async operations provided by the service are not cancellable.
@@ -54,7 +59,8 @@ public interface TimeAwareRaftGroupService {
* <p>Read commands always see up to date data.
*
* @param cmd The command.
- * @param timeoutMillis Timeout to use, in milliseconds (if this is negative, the timeout is disabled).
+ * @param timeoutMillis Timeout in milliseconds. {@code 0} means single attempt without retries;
+ * {@code Long.MAX_VALUE} means infinite wait; negative values are treated as infinite for compatibility.
* @param <R> Execution result type.
* @return A future with the execution result.
*/
@@ -82,27 +88,31 @@ public interface TimeAwareRaftGroupService {
@Nullable List<Peer> learners();
/**
- * Refreshes a replication group leader.
+ * Refreshes a replication group leader with the given timeout.
*
* <p>After the future completion the method {@link #leader()} can be used to retrieve a current group leader.
*
* <p>This operation is executed on a group leader.
*
+ * @param timeoutMillis Timeout in milliseconds. {@code 0} means single attempt without retries;
+ * {@code Long.MAX_VALUE} means infinite wait; negative values are treated as infinite for compatibility.
* @return A future.
*/
- CompletableFuture<Void> refreshLeader();
+ CompletableFuture<Void> refreshLeader(long timeoutMillis);
/**
- * Refreshes a replication group leader and returns (leader, term) tuple.
+ * Refreshes a replication group leader and returns (leader, term) tuple with the given timeout.
*
* <p>This operation is executed on a group leader.
*
+ * @param timeoutMillis Timeout in milliseconds. {@code 0} means single attempt without retries;
+ * {@code Long.MAX_VALUE} means infinite wait; negative values are treated as infinite for compatibility.
* @return A future, with (leader, term) tuple.
*/
- CompletableFuture<LeaderWithTerm> refreshAndGetLeaderWithTerm();
+ CompletableFuture<LeaderWithTerm> refreshAndGetLeaderWithTerm(long timeoutMillis);
/**
- * Refreshes replication group members.
+ * Refreshes replication group members with the given timeout.
*
* <p>After the future completion methods like {@link #peers()} and {@link #learners()} can be used to retrieve current members of a
* group.
@@ -110,12 +120,14 @@ public interface TimeAwareRaftGroupService {
* <p>This operation is executed on a group leader.
*
* @param onlyAlive {@code True} to exclude dead nodes.
+ * @param timeoutMillis Timeout in milliseconds. {@code 0} means single attempt without retries;
+ * {@code Long.MAX_VALUE} means infinite wait; negative values are treated as infinite for compatibility.
* @return A future.
*/
- CompletableFuture<Void> refreshMembers(boolean onlyAlive);
+ CompletableFuture<Void> refreshMembers(boolean onlyAlive, long timeoutMillis);
/**
- * Adds a voting peer to the replication group.
+ * Adds a voting peer to the replication group with the given timeout.
*
* <p>After the future completion methods like {@link #peers()} and {@link #learners()} can be used to retrieve current members of a
* group.
@@ -124,13 +136,15 @@ public interface TimeAwareRaftGroupService {
*
* @param peer Peer
* @param sequenceToken Sequence token of the current change.
+ * @param timeoutMillis Timeout in milliseconds. {@code 0} means single attempt without retries;
+ * {@code Long.MAX_VALUE} means infinite wait; negative values are treated as infinite for compatibility.
*
* @return A future.
*/
- CompletableFuture<Void> addPeer(Peer peer, long sequenceToken);
+ CompletableFuture<Void> addPeer(Peer peer, long sequenceToken, long timeoutMillis);
/**
- * Removes peer from the replication group.
+ * Removes peer from the replication group with the given timeout.
*
* <p>After the future completion methods like {@link #peers()} and {@link #learners()} can be used to retrieve current members of a
* group.
@@ -139,13 +153,15 @@ public interface TimeAwareRaftGroupService {
*
* @param peer Peer.
* @param sequenceToken Sequence token of the current change.
+ * @param timeoutMillis Timeout in milliseconds. {@code 0} means single attempt without retries;
+ * {@code Long.MAX_VALUE} means infinite wait; negative values are treated as infinite for compatibility.
*
* @return A future.
*/
- CompletableFuture<Void> removePeer(Peer peer, long sequenceToken);
+ CompletableFuture<Void> removePeer(Peer peer, long sequenceToken, long timeoutMillis);
/**
- * Changes peers and learners of a replication group.
+ * Changes peers and learners of a replication group with the given timeout.
*
* <p>After the future completion methods like {@link #peers()} and {@link #learners()} can be used to retrieve current members of a
* group.
@@ -156,13 +172,19 @@ public interface TimeAwareRaftGroupService {
* @param term Current known leader term.
* If real raft group term will be different - configuration update will be skipped.
* @param sequenceToken Sequence token of the current change.
+ * @param timeoutMillis Timeout in milliseconds. {@code 0} means single attempt without retries;
+ * {@code Long.MAX_VALUE} means infinite wait; negative values are treated as infinite for compatibility.
*
* @return A future.
*/
- CompletableFuture<Void> changePeersAndLearners(PeersAndLearners peersAndLearners, @Deprecated long term, long sequenceToken);
+ CompletableFuture<Void> changePeersAndLearners(
+ PeersAndLearners peersAndLearners,
+ @Deprecated long term,
+ long sequenceToken,
+ long timeoutMillis);
/**
- * Changes peers and learners of a replication group.
+ * Changes peers and learners of a replication group with the given timeout.
*
* <p>Asynchronous variant of the previous method.
* When the future completed, it just means, that {@code changePeers} process has successfully started.
@@ -176,13 +198,19 @@ public interface TimeAwareRaftGroupService {
* @param term Current known leader term.
* If real raft group term will be different - configuration update will be skipped.
* @param sequenceToken Sequence token of the current change.
+ * @param timeoutMillis Timeout in milliseconds. {@code 0} means single attempt without retries;
+ * {@code Long.MAX_VALUE} means infinite wait; negative values are treated as infinite for compatibility.
*
* @return A future.
*/
- CompletableFuture<Void> changePeersAndLearnersAsync(PeersAndLearners peersAndLearners, @Deprecated long term, long sequenceToken);
+ CompletableFuture<Void> changePeersAndLearnersAsync(
+ PeersAndLearners peersAndLearners,
+ @Deprecated long term,
+ long sequenceToken,
+ long timeoutMillis);
/**
- * Adds learners (non-voting members).
+ * Adds learners (non-voting members) with the given timeout.
*
* <p>After the future completion methods like {@link #peers()} and {@link #learners()} can be used to retrieve current members of a
* group.
@@ -191,13 +219,15 @@ public interface TimeAwareRaftGroupService {
*
* @param learners Collection of learners.
* @param sequenceToken Sequence token of the current change.
+ * @param timeoutMillis Timeout in milliseconds. {@code 0} means single attempt without retries;
+ * {@code Long.MAX_VALUE} means infinite wait; negative values are treated as infinite for compatibility.
*
* @return A future.
*/
- CompletableFuture<Void> addLearners(Collection<Peer> learners, long sequenceToken);
+ CompletableFuture<Void> addLearners(Collection<Peer> learners, long sequenceToken, long timeoutMillis);
/**
- * Removes learners.
+ * Removes learners with the given timeout.
*
* <p>After the future completion methods like {@link #peers()} and {@link #learners()} can be used to retrieve current members of a
* group.
@@ -206,13 +236,15 @@ public interface TimeAwareRaftGroupService {
*
* @param learners Collection of learners.
* @param sequenceToken Sequence token of the current change.
+ * @param timeoutMillis Timeout in milliseconds. {@code 0} means single attempt without retries;
+ * {@code Long.MAX_VALUE} means infinite wait; negative values are treated as infinite for compatibility.
*
* @return A future.
*/
- CompletableFuture<Void> removeLearners(Collection<Peer> learners, long sequenceToken);
+ CompletableFuture<Void> removeLearners(Collection<Peer> learners, long sequenceToken, long timeoutMillis);
/**
- * Set learners of the raft group to needed list of learners.
+ * Set learners of the raft group to needed list of learners with the given timeout.
*
* <p>After the future completion methods like {@link #peers()} and {@link #learners()} can be used to retrieve current members of a
* group.
@@ -221,29 +253,35 @@ public interface TimeAwareRaftGroupService {
*
* @param learners Collection of learners.
* @param sequenceToken Sequence token of the current change.
+ * @param timeoutMillis Timeout in milliseconds. {@code 0} means single attempt without retries;
+ * {@code Long.MAX_VALUE} means infinite wait; negative values are treated as infinite for compatibility.
*
* @return A future.
*/
- CompletableFuture<Void> resetLearners(Collection<Peer> learners, long sequenceToken);
+ CompletableFuture<Void> resetLearners(Collection<Peer> learners, long sequenceToken, long timeoutMillis);
/**
- * Takes a state machine snapshot on a given group peer.
+ * Takes a state machine snapshot on a given group peer with the given timeout.
*
* @param peer Peer.
* @param forced {@code True} to force snapshot and log truncation.
+ * @param timeoutMillis Timeout in milliseconds. {@code 0} means single attempt without retries;
+ * {@code Long.MAX_VALUE} means infinite wait; negative values are treated as infinite for compatibility.
* @return A future.
*/
- CompletableFuture<Void> snapshot(Peer peer, boolean forced);
+ CompletableFuture<Void> snapshot(Peer peer, boolean forced, long timeoutMillis);
/**
- * Transfers leadership to other peer.
+ * Transfers leadership to other peer with the given timeout.
*
* <p>This operation is executed on a group leader.
*
* @param newLeader New leader.
+ * @param timeoutMillis Timeout in milliseconds. {@code 0} means single attempt without retries;
+ * {@code Long.MAX_VALUE} means infinite wait; negative values are treated as infinite for compatibility.
* @return A future.
*/
- CompletableFuture<Void> transferLeadership(Peer newLeader);
+ CompletableFuture<Void> transferLeadership(Peer newLeader, long timeoutMillis);
/**
* Shutdown and cleanup resources for this instance.
@@ -251,11 +289,13 @@ public interface TimeAwareRaftGroupService {
void shutdown();
/**
- * Reads index from the group leader.
+ * Reads index from the group leader with the given timeout.
*
+ * @param timeoutMillis Timeout in milliseconds. {@code 0} means single attempt without retries;
+ * {@code Long.MAX_VALUE} means infinite wait; negative values are treated as infinite for compatibility.
* @return Future containing the index.
*/
- CompletableFuture<Long> readIndex();
+ CompletableFuture<Long> readIndex(long timeoutMillis);
/**
* Returns a cluster service.
diff --git a/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityState.java b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityState.java
index ed427840e06..5dd46b1416e 100644
--- a/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityState.java
+++ b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityState.java
@@ -20,7 +20,8 @@ package org.apache.ignite.internal.raft.client;
import java.util.concurrent.CompletableFuture;
import org.apache.ignite.internal.logger.IgniteLogger;
import org.apache.ignite.internal.logger.Loggers;
-import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.Peer;
+import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.VisibleForTesting;
/**
@@ -34,11 +35,14 @@ import org.jetbrains.annotations.VisibleForTesting;
*
* <p>State transitions:
* <pre>
- * WAITING_FOR_LEADER --[onLeaderElected]--> LEADER_AVAILABLE
+ * WAITING_FOR_LEADER --[updateKnownLeaderAndTerm(non-null peer)]--> LEADER_AVAILABLE
* LEADER_AVAILABLE --[onGroupUnavailable]--> WAITING_FOR_LEADER
+ * Any state --[resetLeaderState]--> WAITING_FOR_LEADER (leader=null, term=-1)
* Any state --[stop]--> stopped (terminal state)
* </pre>
*
+ * <p>This class is the single source of truth for the current leader peer and term.
+ *
* <p>Thread-safe: all public methods are synchronized or use proper synchronization.
*/
@VisibleForTesting
@@ -63,6 +67,9 @@ class LeaderAvailabilityState {
/** Current leader term. Initialized to -1 to accept term 0 as the first valid term. Guarded by {@code mutex}. */
private long currentTerm = -1;
+ /** Current leader peer. Guarded by {@code mutex}. */
+ private @Nullable Peer leader;
+
/** Future that waiters block on. Guarded by {@code mutex}. */
private CompletableFuture<Long> waiters = new CompletableFuture<>();
@@ -76,7 +83,7 @@ class LeaderAvailabilityState {
* Returns a future that completes when a leader becomes available.
*
* <p>If a leader is already available, returns an already-completed future with the current term.
- * Otherwise, returns a future that will complete when {@link #onLeaderElected} is called.
+ * Otherwise, returns a future that will complete when {@link #updateKnownLeaderAndTerm} is called.
* If the state machine has been destroyed, returns a failed future.
*
* @return Future that completes with the leader term when a leader is available.
@@ -94,16 +101,23 @@ class LeaderAvailabilityState {
}
/**
- * Handles leader election notification.
+ * Updates the known leader and term.
*
- * <p>Transitions from {@link State#WAITING_FOR_LEADER} to {@link State#LEADER_AVAILABLE}
- * if the new term is greater than the current term. Stale notifications (with term <= current)
- * are ignored. Has no effect if the state machine has been stopped.
+ * <p>If the new term is greater than the current term:
+ * <ul>
+ * <li>Updates the term and stores the peer.</li>
+ * <li>If peer is non-null: transitions from {@link State#WAITING_FOR_LEADER} to {@link State#LEADER_AVAILABLE}
+ * and completes waiters.</li>
+ * <li>If peer is null: updates the term only, does NOT transition state or wake waiters.</li>
+ * </ul>
+ * Stale notifications (with term <= current) are ignored.
+ * Has no effect if the state machine has been stopped.
*
- * @param leader The newly elected leader node.
- * @param term The term of the new leader.
+ * @param leader The leader peer (may be {@code null} if only the term is known).
+ * @param term The term of the leader.
+ * @return {@code true} if accepted (term is newer), {@code false} if ignored (stale or stopped).
*/
- void onLeaderElected(InternalClusterNode leader, long term) {
+ boolean updateKnownLeaderAndTerm(@Nullable Peer leader, long term) {
if (term < 0) {
throw new IllegalArgumentException("Term must be non-negative: " + term);
}
@@ -112,27 +126,28 @@ class LeaderAvailabilityState {
synchronized (mutex) {
if (stopped) {
- LOG.debug("Ignoring leader election after stop [leader={}, term={}]", leader, term);
- return;
+ LOG.debug("Ignoring leader update after stop [leader={}, term={}]", leader, term);
+ return false;
}
// Ignore stale term notifications.
if (term <= currentTerm) {
LOG.debug("Ignoring stale leader [newTerm={}, currentTerm={}]", term, currentTerm);
- return;
+ return false;
}
long previousTerm = currentTerm;
State previousState = currentState;
currentTerm = term;
+ this.leader = leader;
- if (currentState == State.WAITING_FOR_LEADER) {
+ if (leader != null && currentState == State.WAITING_FOR_LEADER) {
currentState = State.LEADER_AVAILABLE;
futureToComplete = waiters;
}
- LOG.debug("Leader elected [leader={}, term={}, previousTerm={}, stateChange={}->{}]",
+ LOG.debug("Leader updated [leader={}, term={}, previousTerm={}, stateChange={}->{}]",
leader, term, previousTerm, previousState, currentState);
}
@@ -140,6 +155,66 @@ class LeaderAvailabilityState {
if (futureToComplete != null) {
futureToComplete.complete(term);
}
+
+ return true;
+ }
+
+ /**
+ * Updates the cached leader without changing the term or availability state.
+ *
+ * <p>This is an optimistic cache update used when a response or redirect provides leader information
+ * without a term change. Has no effect after stop.
+ *
+ * @param leader The cached leader peer (may be {@code null} to clear).
+ */
+ void setCachedLeader(@Nullable Peer leader) {
+ synchronized (mutex) {
+ if (stopped) {
+ return;
+ }
+
+ this.leader = leader;
+ }
+ }
+
+ /**
+ * Returns the current leader peer.
+ *
+ * @return Current leader or {@code null} if unknown.
+ */
+ @Nullable Peer leader() {
+ synchronized (mutex) {
+ return leader;
+ }
+ }
+
+ /**
+ * Resets the leader state completely: clears the cached leader, resets term to -1,
+ * and transitions to {@link State#WAITING_FOR_LEADER}.
+ *
+ * <p>Used when the peer configuration changes (e.g., {@code resetPeers}) and the cached
+ * leader/term may no longer be valid. Resetting term to -1 ensures the next leader update
+ * (even with the same term) will be accepted by the stale term guard.
+ *
+ * <p>Has no effect if the state machine has been stopped.
+ */
+ void resetLeaderState() {
+ synchronized (mutex) {
+ if (stopped) {
+ return;
+ }
+
+ State previousState = currentState;
+ long previousTerm = currentTerm;
+
+ leader = null;
+ currentTerm = -1;
+ currentState = State.WAITING_FOR_LEADER;
+ waiters = new CompletableFuture<>();
+
+ LOG.debug("Leader state reset [previousTerm={}, stateChange={}->{}]",
+ previousTerm, previousState, currentState);
+ }
}
/**
@@ -148,7 +223,10 @@ class LeaderAvailabilityState {
* <p>Transitions from {@link State#LEADER_AVAILABLE} to {@link State#WAITING_FOR_LEADER}
* only if the term hasn't changed since the unavailability was detected. This prevents
* resetting the state if a new leader was already elected.
- * Has no effect if the state machine has been stopped.
+ *
+ * <p>Note: does NOT clear the leader field — intentionally allows retry to the old leader as a first guess.
+ *
+ * <p>Has no effect if the state machine has been stopped.
*
* @param termWhenDetected The term at which unavailability was detected.
*/
@@ -210,7 +288,8 @@ class LeaderAvailabilityState {
* <p>After stop, the state machine is no longer active:
* <ul>
* <li>{@link #awaitLeader()} returns a failed future</li>
- * <li>{@link #onLeaderElected} is ignored</li>
+ * <li>{@link #updateKnownLeaderAndTerm} is ignored</li>
+ * <li>{@link #setCachedLeader} is ignored</li>
* <li>{@link #onGroupUnavailable} is ignored</li>
* </ul>
*
diff --git a/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PeerTracking.java b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PeerTracking.java
new file mode 100644
index 00000000000..6ac005e714a
--- /dev/null
+++ b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PeerTracking.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+/**
+ * How to track the current peer when moving to a new one.
+ */
+enum PeerTracking {
+ /** Don't mark the current peer (transient errors, leader redirects). */
+ COMMON,
+ /** Mark as unavailable (down, shutting down). */
+ UNAVAILABLE,
+ /** Mark as "no leader" (working but doesn't know leader). */
+ NO_LEADER
+}
diff --git a/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java
index 6e6d00b311a..7c7ec1a6434 100644
--- a/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java
+++ b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java
@@ -18,22 +18,27 @@
package org.apache.ignite.internal.raft.client;
import static java.util.concurrent.CompletableFuture.runAsync;
-import static org.apache.ignite.internal.util.ExceptionUtils.unwrapCause;
+import static org.apache.ignite.internal.raft.client.RaftPeerUtils.parsePeer;
+import static org.apache.ignite.internal.raft.client.RaftPeerUtils.parsePeerList;
+import static org.apache.ignite.internal.raft.client.RaftPeerUtils.peerId;
+import static org.apache.ignite.internal.raft.client.RaftPeerUtils.peerIds;
+import static org.apache.ignite.internal.util.CompletableFutures.nullCompletedFuture;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import org.apache.ignite.internal.failure.FailureContext;
import org.apache.ignite.internal.failure.FailureManager;
-import org.apache.ignite.internal.lang.NodeStoppingException;
import org.apache.ignite.internal.logger.IgniteLogger;
import org.apache.ignite.internal.logger.Loggers;
import org.apache.ignite.internal.network.ClusterService;
import org.apache.ignite.internal.network.InternalClusterNode;
-import org.apache.ignite.internal.network.RecipientLeftException;
import org.apache.ignite.internal.network.TopologyEventHandler;
import org.apache.ignite.internal.network.TopologyService;
import org.apache.ignite.internal.raft.Command;
@@ -43,15 +48,21 @@ import org.apache.ignite.internal.raft.Loza;
import org.apache.ignite.internal.raft.Marshaller;
import org.apache.ignite.internal.raft.Peer;
import org.apache.ignite.internal.raft.PeersAndLearners;
-import org.apache.ignite.internal.raft.ThrottlingContextHolder;
import org.apache.ignite.internal.raft.configuration.RaftConfiguration;
import org.apache.ignite.internal.raft.service.LeaderWithTerm;
-import org.apache.ignite.internal.raft.service.RaftGroupService;
import org.apache.ignite.internal.raft.service.TimeAwareRaftGroupService;
import org.apache.ignite.internal.replicator.ReplicationGroupId;
import org.apache.ignite.internal.util.CompletableFutures;
import org.apache.ignite.raft.jraft.RaftMessagesFactory;
+import org.apache.ignite.raft.jraft.rpc.CliRequests.AddPeerResponse;
+import org.apache.ignite.raft.jraft.rpc.CliRequests.ChangePeersAndLearnersAsyncResponse;
+import org.apache.ignite.raft.jraft.rpc.CliRequests.ChangePeersAndLearnersResponse;
+import org.apache.ignite.raft.jraft.rpc.CliRequests.GetLeaderResponse;
+import org.apache.ignite.raft.jraft.rpc.CliRequests.GetPeersResponse;
+import org.apache.ignite.raft.jraft.rpc.CliRequests.LearnersOpResponse;
+import org.apache.ignite.raft.jraft.rpc.CliRequests.RemovePeerResponse;
import org.apache.ignite.raft.jraft.rpc.CliRequests.SubscriptionLeaderChangeRequest;
+import org.apache.ignite.raft.jraft.rpc.RpcRequests.ReadIndexResponse;
import org.apache.ignite.raft.jraft.rpc.impl.RaftGroupEventsClientListener;
import org.jetbrains.annotations.Nullable;
@@ -67,6 +78,9 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
/** Raft message factory. */
private static final RaftMessagesFactory MESSAGES_FACTORY = Loza.FACTORY;
+ /** Replication group ID. */
+ private final ReplicationGroupId groupId;
+
/** General leader election listener. */
private final ServerEventHandler generalLeaderElectionListener;
@@ -76,21 +90,30 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
/** Cluster service. */
private final ClusterService clusterService;
- /** RPC RAFT client. */
- private final RaftGroupService raftClient;
-
- /** Executor to invoke RPC requests. */
- private final ScheduledExecutorService executor;
-
- /** RAFT configuration. */
- private final RaftConfiguration raftConfiguration;
-
/** Factory for creating stopping exceptions. */
private final ExceptionFactory stoppingExceptionFactory;
/** Command executor with retry semantics. */
private final RaftCommandExecutor commandExecutor;
+ /** Sender for subscription messages with retry logic. */
+ private final SubscriptionMessageSender subscriptionMessageSender;
+
+ /** Current peers. */
+ private volatile List<Peer> peers;
+
+ /** Current learners. */
+ private volatile List<Peer> learners;
+
+ /** Topology event handler for cleanup during shutdown. */
+ private final TopologyEventHandler topologyEventHandler;
+
+ /** Events client listener for cleanup during shutdown. */
+ private final RaftGroupEventsClientListener eventsClientListener;
+
+ /** Timeout in milliseconds to wait for subscription cleanup during shutdown. */
+ private static final long SUBSCRIPTION_CLEANUP_TIMEOUT_MS = 30_000;
+
/**
* Constructor.
*
@@ -109,29 +132,18 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
PeersAndLearners configuration,
Marshaller cmdMarshaller,
ExceptionFactory stoppingExceptionFactory,
- ThrottlingContextHolder throttlingContextHolder,
RaftGroupEventsClientListener eventsClientListener
) {
+ this.groupId = groupId;
this.failureManager = failureManager;
this.clusterService = clusterService;
- this.executor = executor;
- this.raftConfiguration = raftConfiguration;
this.stoppingExceptionFactory = stoppingExceptionFactory;
- this.raftClient = RaftGroupServiceImpl.start(
- groupId,
- clusterService,
- MESSAGES_FACTORY,
- raftConfiguration,
- configuration,
- executor,
- cmdMarshaller,
- stoppingExceptionFactory,
- throttlingContextHolder
- );
+ this.peers = List.copyOf(configuration.peers());
+ this.learners = List.copyOf(configuration.learners());
this.commandExecutor = new RaftCommandExecutor(
groupId,
- raftClient::peers,
+ this::peers,
clusterService,
executor,
raftConfiguration,
@@ -139,23 +151,39 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
stoppingExceptionFactory
);
+ this.subscriptionMessageSender = new SubscriptionMessageSender(clusterService.messagingService(), executor, raftConfiguration);
+
this.generalLeaderElectionListener = new ServerEventHandler(executor);
- eventsClientListener.addLeaderElectionListener(raftClient.groupId(), generalLeaderElectionListener);
+ this.eventsClientListener = eventsClientListener;
+ eventsClientListener.addLeaderElectionListener(groupId, generalLeaderElectionListener);
// Subscribe the command executor's leader availability state to leader election notifications.
subscribeLeader(commandExecutor.leaderElectionListener());
TopologyService topologyService = clusterService.topologyService();
- topologyService.addEventHandler(new TopologyEventHandler() {
+ this.topologyEventHandler = new TopologyEventHandler() {
@Override
public void onAppeared(InternalClusterNode member) {
CompletableFuture<Boolean> fut = changeNodeSubscriptionIfNeed(member, true);
- requestLeaderManually(topologyService, executor, raftClient, fut);
+ requestLeaderManually(topologyService, executor, fut);
}
- });
+
+ @Override
+ public void onDisappeared(InternalClusterNode member) {
+ Peer cachedLeader = commandExecutor.leader();
+
+ if (cachedLeader != null && member.name().equals(cachedLeader.consistentId())) {
+ commandExecutor.setLeader(null);
+
+ LOG.info("Cleared cached leader due to node disappearance [grp={}, leader={}].",
+ groupId, member.name());
+ }
+ }
+ };
+ topologyService.addEventHandler(topologyEventHandler);
ArrayList<CompletableFuture<?>> futures = new ArrayList<>();
@@ -163,7 +191,7 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
futures.add(changeNodeSubscriptionIfNeed(member, true));
}
- requestLeaderManually(topologyService, executor, raftClient, CompletableFutures.allOf(futures));
+ requestLeaderManually(topologyService, executor, CompletableFutures.allOf(futures));
}
/**
@@ -171,18 +199,16 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
*
* @param topologyService Topology service.
* @param executor Executor to run asynchronous tasks.
- * @param raftClient RAFT client to interact with the RAFT group.
* @param subscriptionsFut Future representing the completion of subscription updates.
*/
// TODO: IGNITE-27256 Remove the method after implementing a notification after subscription.
private void requestLeaderManually(
TopologyService topologyService,
Executor executor,
- RaftGroupService raftClient,
CompletableFuture<?> subscriptionsFut
) {
subscriptionsFut.thenRunAsync(
- () -> raftClient.refreshAndGetLeaderWithTerm().whenCompleteAsync(
+ () -> refreshAndGetLeaderWithTerm(Long.MAX_VALUE).whenCompleteAsync(
(leaderWithTerm, throwable) -> {
if (throwable != null) {
LOG.warn("Could not refresh and get leader with term [grp={}].", groupId(), throwable);
@@ -214,9 +240,12 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
Peer peer = new Peer(member.name());
if (peers().contains(peer)) {
- SubscriptionLeaderChangeRequest msg = subscriptionLeaderChangeRequest(subscribe);
+ SubscriptionLeaderChangeRequest msg = MESSAGES_FACTORY.subscriptionLeaderChangeRequest()
+ .groupId(groupId)
+ .subscribe(subscribe)
+ .build();
- return sendMessage(member, msg).whenComplete((isSent, err) -> {
+ return subscriptionMessageSender.send(member, msg).whenComplete((isSent, err) -> {
if (err != null) {
failureManager.process(new FailureContext(err, "Could not change subscription to leader updates [grp="
+ groupId() + "]."));
@@ -241,7 +270,6 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
* @param eventsClientListener Listener for RAFT group events.
* @param cmdMarshaller Marshaller for RAFT commands.
* @param stoppingExceptionFactory Factory for creating stopping exceptions.
- * @param throttlingContextHolder Context holder for throttling.
* @param failureManager Manager for handling failures.
* @return A new instance of PhysicalTopologyAwareRaftGroupService.
*/
@@ -254,7 +282,6 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
RaftGroupEventsClientListener eventsClientListener,
Marshaller cmdMarshaller,
ExceptionFactory stoppingExceptionFactory,
- ThrottlingContextHolder throttlingContextHolder,
FailureManager failureManager
) {
return new PhysicalTopologyAwareRaftGroupService(
@@ -266,14 +293,27 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
configuration,
cmdMarshaller,
stoppingExceptionFactory,
- throttlingContextHolder,
eventsClientListener
);
}
private void finishSubscriptions() {
+ List<CompletableFuture<Boolean>> futures = new ArrayList<>();
+
for (InternalClusterNode member : clusterService.topologyService().allMembers()) {
- changeNodeSubscriptionIfNeed(member, false);
+ futures.add(changeNodeSubscriptionIfNeed(member, false));
+ }
+
+ // Wait for unsubscription to complete with a timeout, but don't fail if it times out.
+ try {
+ CompletableFutures.allOf(futures).get(SUBSCRIPTION_CLEANUP_TIMEOUT_MS, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ LOG.debug("Interrupted while waiting for subscription cleanup [grp={}].", groupId);
+ } catch (ExecutionException e) {
+ LOG.debug("Error during subscription cleanup [grp={}].", groupId, e.getCause());
+ } catch (TimeoutException e) {
+ LOG.debug("Timeout waiting for subscription cleanup [grp={}].", groupId);
}
}
@@ -285,60 +325,6 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
generalLeaderElectionListener.removeCallbackAndNotify(callback);
}
- private SubscriptionLeaderChangeRequest subscriptionLeaderChangeRequest(boolean subscribe) {
- return MESSAGES_FACTORY.subscriptionLeaderChangeRequest()
- .groupId(groupId())
- .subscribe(subscribe)
- .build();
- }
-
- private CompletableFuture<Boolean> sendMessage(InternalClusterNode node, SubscriptionLeaderChangeRequest msg) {
- var msgSendFut = new CompletableFuture<Boolean>();
-
- sendWithRetry(node, msg, msgSendFut);
-
- return msgSendFut;
- }
-
- private void sendWithRetry(InternalClusterNode node, SubscriptionLeaderChangeRequest msg, CompletableFuture<Boolean> msgSendFut) {
- Long responseTimeout = raftConfiguration.responseTimeoutMillis().value();
-
- clusterService.messagingService().invoke(node, msg, responseTimeout).whenCompleteAsync((unused, invokeThrowable) -> {
- if (invokeThrowable == null) {
- msgSendFut.complete(true);
-
- return;
- }
-
- Throwable invokeCause = unwrapCause(invokeThrowable);
- if (!msg.subscribe()) {
- // We don't want to propagate exceptions when unsubscribing (if it's not an Error!).
- if (invokeCause instanceof Error) {
- msgSendFut.completeExceptionally(invokeThrowable);
- } else {
- LOG.debug("An exception while trying to unsubscribe.", invokeThrowable);
-
- msgSendFut.complete(false);
- }
- } else if (RaftErrorUtils.recoverable(invokeCause)) {
- sendWithRetry(node, msg, msgSendFut);
- } else if (invokeCause instanceof RecipientLeftException) {
- LOG.info(
- "Could not subscribe to leader update from a specific node, because the node had left the cluster: [node={}].",
- node
- );
-
- msgSendFut.complete(false);
- } else if (invokeCause instanceof NodeStoppingException) {
- msgSendFut.complete(false);
- } else {
- LOG.error("Could not send the subscribe message to the node: [node={}, msg={}].", invokeThrowable, node, msg);
-
- msgSendFut.completeExceptionally(invokeThrowable);
- }
- }, executor);
- }
-
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
return commandExecutor.run(cmd, timeoutMillis);
@@ -346,107 +332,264 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
@Override
public ReplicationGroupId groupId() {
- return raftClient.groupId();
+ return groupId;
}
@Override
public @Nullable Peer leader() {
- return raftClient.leader();
+ return commandExecutor.leader();
}
@Override
public List<Peer> peers() {
- return raftClient.peers();
+ return peers;
}
@Override
public @Nullable List<Peer> learners() {
- return raftClient.learners();
+ return learners;
}
@Override
- public CompletableFuture<Void> refreshLeader() {
- return raftClient.refreshLeader();
+ public CompletableFuture<Void> refreshLeader(long timeoutMillis) {
+ return commandExecutor.<GetLeaderResponse>send(
+ peer -> MESSAGES_FACTORY.getLeaderRequest()
+ .peerId(peerId(peer))
+ .groupId(groupId.toString())
+ .build(),
+ TargetPeerStrategy.RANDOM,
+ timeoutMillis
+ ).thenAccept(resp -> {
+ // Only update cached leader if the term is newer to avoid overwriting
+ // fresher info from leader election notifications with stale responses.
+ commandExecutor.setLeaderIfTermNewer(parsePeer(resp.leaderId()), resp.currentTerm());
+ });
}
@Override
- public CompletableFuture<LeaderWithTerm> refreshAndGetLeaderWithTerm() {
- return raftClient.refreshAndGetLeaderWithTerm();
+ public CompletableFuture<LeaderWithTerm> refreshAndGetLeaderWithTerm(long timeoutMillis) {
+ return commandExecutor.<GetLeaderResponse>send(
+ peer -> MESSAGES_FACTORY.getLeaderRequest()
+ .peerId(peerId(peer))
+ .groupId(groupId.toString())
+ .build(),
+ TargetPeerStrategy.RANDOM,
+ timeoutMillis
+ ).thenApply(resp -> {
+ if (resp.leaderId() == null) {
+ return LeaderWithTerm.NO_LEADER;
+ }
+
+ Peer respLeader = parsePeer(resp.leaderId());
+ // Only update cached leader if the term is newer to avoid overwriting
+ // fresher info from leader election notifications with stale responses.
+ commandExecutor.setLeaderIfTermNewer(respLeader, resp.currentTerm());
+ return new LeaderWithTerm(respLeader, resp.currentTerm());
+ });
}
@Override
- public CompletableFuture<Void> refreshMembers(boolean onlyAlive) {
- return raftClient.refreshMembers(onlyAlive);
+ public CompletableFuture<Void> refreshMembers(boolean onlyAlive, long timeoutMillis) {
+ return commandExecutor.<GetPeersResponse>send(
+ peer -> MESSAGES_FACTORY.getPeersRequest()
+ .leaderId(peerId(peer))
+ .onlyAlive(onlyAlive)
+ .groupId(groupId.toString())
+ .build(),
+ TargetPeerStrategy.LEADER,
+ timeoutMillis
+ ).thenAccept(resp -> {
+ this.peers = parsePeerList(resp.peersList());
+ this.learners = parsePeerList(resp.learnersList());
+ });
}
@Override
- public CompletableFuture<Void> addPeer(Peer peer, long sequenceToken) {
- return raftClient.addPeer(peer, sequenceToken);
+ public CompletableFuture<Void> addPeer(Peer peer, long sequenceToken, long timeoutMillis) {
+ return commandExecutor.<AddPeerResponse>send(
+ targetPeer -> MESSAGES_FACTORY.addPeerRequest()
+ .leaderId(peerId(targetPeer))
+ .groupId(groupId.toString())
+ .peerId(peerId(peer))
+ .sequenceToken(sequenceToken)
+ .build(),
+ TargetPeerStrategy.LEADER,
+ timeoutMillis
+ ).thenAccept(resp -> this.peers = parsePeerList(resp.newPeersList()));
}
@Override
- public CompletableFuture<Void> removePeer(Peer peer, long sequenceToken) {
- return raftClient.removePeer(peer, sequenceToken);
+ public CompletableFuture<Void> removePeer(Peer peer, long sequenceToken, long timeoutMillis) {
+ return commandExecutor.<RemovePeerResponse>send(
+ targetPeer -> MESSAGES_FACTORY.removePeerRequest()
+ .leaderId(peerId(targetPeer))
+ .groupId(groupId.toString())
+ .peerId(peerId(peer))
+ .sequenceToken(sequenceToken)
+ .build(),
+ TargetPeerStrategy.LEADER,
+ timeoutMillis
+ ).thenAccept(resp -> this.peers = parsePeerList(resp.newPeersList()));
}
@Override
- public CompletableFuture<Void> changePeersAndLearners(PeersAndLearners peersAndLearners, long term, long sequenceToken) {
- return raftClient.changePeersAndLearners(peersAndLearners, term, sequenceToken);
+ public CompletableFuture<Void> changePeersAndLearners(
+ PeersAndLearners peersAndLearners, long term, long sequenceToken, long timeoutMillis) {
+ LOG.info("Sending changePeersAndLearners request for group={} to peers={} and learners={} with leader term={}",
+ groupId, peersAndLearners.peers(), peersAndLearners.learners(), term);
+
+ return commandExecutor.<ChangePeersAndLearnersResponse>send(
+ targetPeer -> MESSAGES_FACTORY.changePeersAndLearnersRequest()
+ .leaderId(peerId(targetPeer))
+ .groupId(groupId.toString())
+ .term(term)
+ .newPeersList(peerIds(peersAndLearners.peers()))
+ .newLearnersList(peerIds(peersAndLearners.learners()))
+ .sequenceToken(sequenceToken)
+ .build(),
+ TargetPeerStrategy.LEADER,
+ timeoutMillis
+ ).thenAccept(resp -> {
+ this.peers = parsePeerList(resp.newPeersList());
+ this.learners = parsePeerList(resp.newLearnersList());
+ });
}
@Override
- public CompletableFuture<Void> changePeersAndLearnersAsync(PeersAndLearners peersAndLearners, long term, long sequenceToken) {
- return raftClient.changePeersAndLearnersAsync(peersAndLearners, term, sequenceToken);
+ public CompletableFuture<Void> changePeersAndLearnersAsync(
+ PeersAndLearners peersAndLearners, long term, long sequenceToken, long timeoutMillis) {
+ LOG.info("Sending changePeersAndLearnersAsync request for group={} to peers={} and learners={} with leader term={}",
+ groupId, peersAndLearners.peers(), peersAndLearners.learners(), term);
+
+ return commandExecutor.<ChangePeersAndLearnersAsyncResponse>send(
+ targetPeer -> MESSAGES_FACTORY.changePeersAndLearnersAsyncRequest()
+ .leaderId(peerId(targetPeer))
+ .groupId(groupId.toString())
+ .term(term)
+ .newPeersList(peerIds(peersAndLearners.peers()))
+ .newLearnersList(peerIds(peersAndLearners.learners()))
+ .sequenceToken(sequenceToken)
+ .build(),
+ TargetPeerStrategy.LEADER,
+ timeoutMillis
+ ).thenAccept(resp -> {
+ this.peers = parsePeerList(resp.newPeersList());
+ this.learners = parsePeerList(resp.newLearnersList());
+ });
}
@Override
- public CompletableFuture<Void> addLearners(Collection<Peer> learners, long sequenceToken) {
- return raftClient.addLearners(learners, sequenceToken);
+ public CompletableFuture<Void> addLearners(Collection<Peer> learners, long sequenceToken, long timeoutMillis) {
+ return commandExecutor.<LearnersOpResponse>send(
+ targetPeer -> MESSAGES_FACTORY.addLearnersRequest()
+ .leaderId(peerId(targetPeer))
+ .groupId(groupId.toString())
+ .learnersList(peerIds(learners))
+ .sequenceToken(sequenceToken)
+ .build(),
+ TargetPeerStrategy.LEADER,
+ timeoutMillis
+ ).thenAccept(resp -> this.learners = parsePeerList(resp.newLearnersList()));
}
@Override
- public CompletableFuture<Void> removeLearners(Collection<Peer> learners, long sequenceToken) {
- return raftClient.removeLearners(learners, sequenceToken);
+ public CompletableFuture<Void> removeLearners(Collection<Peer> learners, long sequenceToken, long timeoutMillis) {
+ return commandExecutor.<LearnersOpResponse>send(
+ targetPeer -> MESSAGES_FACTORY.removeLearnersRequest()
+ .leaderId(peerId(targetPeer))
+ .groupId(groupId.toString())
+ .learnersList(peerIds(learners))
+ .sequenceToken(sequenceToken)
+ .build(),
+ TargetPeerStrategy.LEADER,
+ timeoutMillis
+ ).thenAccept(resp -> this.learners = parsePeerList(resp.newLearnersList()));
}
@Override
- public CompletableFuture<Void> resetLearners(Collection<Peer> learners, long sequenceToken) {
- return raftClient.resetLearners(learners, sequenceToken);
+ public CompletableFuture<Void> resetLearners(Collection<Peer> learners, long sequenceToken, long timeoutMillis) {
+ return commandExecutor.<LearnersOpResponse>send(
+ targetPeer -> MESSAGES_FACTORY.resetLearnersRequest()
+ .leaderId(peerId(targetPeer))
+ .groupId(groupId.toString())
+ .learnersList(peerIds(learners))
+ .sequenceToken(sequenceToken)
+ .build(),
+ TargetPeerStrategy.LEADER,
+ timeoutMillis
+ ).thenAccept(resp -> this.learners = parsePeerList(resp.newLearnersList()));
}
@Override
- public CompletableFuture<Void> snapshot(Peer peer, boolean forced) {
- return raftClient.snapshot(peer, forced);
+ public CompletableFuture<Void> snapshot(Peer peer, boolean forced, long timeoutMillis) {
+ return commandExecutor.send(
+ targetPeer -> MESSAGES_FACTORY.snapshotRequest()
+ .peerId(peerId(peer))
+ .groupId(groupId.toString())
+ .forced(forced)
+ .build(),
+ TargetPeerStrategy.SPECIFIC,
+ peer,
+ timeoutMillis
+ ).thenApply(unused -> null);
}
@Override
- public CompletableFuture<Void> transferLeadership(Peer newLeader) {
- return raftClient.transferLeadership(newLeader);
+ public CompletableFuture<Void> transferLeadership(Peer newLeader, long timeoutMillis) {
+ return commandExecutor.send(
+ targetPeer -> MESSAGES_FACTORY.transferLeaderRequest()
+ .groupId(groupId.toString())
+ .leaderId(peerId(targetPeer))
+ .peerId(peerId(newLeader))
+ .build(),
+ TargetPeerStrategy.LEADER,
+ timeoutMillis
+ ).thenAccept(resp -> commandExecutor.setLeader(newLeader));
}
@Override
public void shutdown() {
- // Stop the command executor first - blocks new run() calls, cancels leader waiters.
+ // Remove topology event handler to prevent new topology events from triggering activity.
+ clusterService.topologyService().removeEventHandler(topologyEventHandler);
+
+ // Remove leader election listener to prevent memory leaks and stop receiving events.
+ eventsClientListener.removeLeaderElectionListener(groupId, generalLeaderElectionListener);
+
+ // Stop the command executor - blocks new run() calls, cancels leader waiters.
commandExecutor.shutdown(stoppingExceptionFactory.create("Raft client is stopping [groupId=" + groupId() + "]."));
+ // Unsubscribe from leader updates with timeout to ensure clean shutdown.
finishSubscriptions();
-
- raftClient.shutdown();
}
@Override
- public CompletableFuture<Long> readIndex() {
- return raftClient.readIndex();
+ public CompletableFuture<Long> readIndex(long timeoutMillis) {
+ return commandExecutor.<ReadIndexResponse>send(
+ peer -> MESSAGES_FACTORY.readIndexRequest()
+ .groupId(groupId.toString())
+ .peerId(peer.consistentId())
+ .serverId(peer.consistentId())
+ .build(),
+ TargetPeerStrategy.LEADER,
+ timeoutMillis
+ ).thenApply(ReadIndexResponse::index);
}
@Override
public ClusterService clusterService() {
- return raftClient.clusterService();
+ return clusterService;
}
@Override
public void updateConfiguration(PeersAndLearners configuration) {
- raftClient.updateConfiguration(configuration);
+ this.peers = List.copyOf(configuration.peers());
+ this.learners = List.copyOf(configuration.learners());
+ commandExecutor.resetLeaderState();
+ }
+
+ @Override
+ public void markAsStopping() {
+ commandExecutor.markAsStopping();
}
/**
@@ -455,7 +598,8 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
private static class ServerEventHandler implements LeaderElectionListener {
private final Executor executor;
- CompletableFuture<Void> fut = CompletableFutures.nullCompletedFuture();
+ /** Future chain for async notifications. */
+ private CompletableFuture<Void> fut = nullCompletedFuture();
/** A leader elected callback. */
private final ArrayList<LeaderElectionListener> callbacks = new ArrayList<>();
@@ -475,6 +619,17 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
this.executor = executor;
}
+ /**
+ * Chains an async action to the future, starting a new chain if the current one is done.
+ */
+ private void chainAsync(Runnable action) {
+ if (fut.isDone()) {
+ fut = runAsync(action, executor);
+ } else {
+ fut = fut.thenRunAsync(action, executor);
+ }
+ }
+
/**
* Notifies about a new leader elected, if it did not make before.
*
@@ -494,19 +649,11 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
ArrayList<LeaderElectionListener> listeners = new ArrayList<>(callbacks);
// Avoid notifying in the synchronized block.
- if (fut.isDone()) {
- fut = runAsync(() -> {
- for (LeaderElectionListener listener : listeners) {
- listener.onLeaderElected(node, term);
- }
- }, executor);
- } else {
- fut = fut.thenRunAsync(() -> {
- for (LeaderElectionListener listener : listeners) {
- listener.onLeaderElected(node, term);
- }
- }, executor);
- }
+ chainAsync(() -> {
+ for (LeaderElectionListener listener : listeners) {
+ listener.onLeaderElected(node, term);
+ }
+ });
}
}
@@ -517,12 +664,8 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
long finalLeaderTerm = this.leaderTerm;
InternalClusterNode finalLeaderNode = this.leaderNode;
- // Notify about the current leader outside of the synchronized block.
- if (fut.isDone()) {
- fut = runAsync(() -> callback.onLeaderElected(finalLeaderNode, finalLeaderTerm), executor);
- } else {
- fut = fut.thenRunAsync(() -> callback.onLeaderElected(finalLeaderNode, finalLeaderTerm), executor);
- }
+ // Notify about the current leader outside the synchronized block.
+ chainAsync(() -> callback.onLeaderElected(finalLeaderNode, finalLeaderTerm));
}
}
@@ -530,9 +673,4 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
callbacks.remove(callback);
}
}
-
- @Override
- public void markAsStopping() {
- raftClient.markAsStopping();
- }
}
diff --git a/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/RaftCommandExecutor.java b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/RaftCommandExecutor.java
index 07bf06417f4..cba5b9e629d 100644
--- a/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/RaftCommandExecutor.java
+++ b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/RaftCommandExecutor.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.raft.client;
import static org.apache.ignite.internal.lang.IgniteStringFormatter.format;
+import static org.apache.ignite.internal.raft.client.RaftPeerUtils.parsePeer;
import static org.apache.ignite.internal.tostring.IgniteToStringBuilder.includeSensitive;
import static org.apache.ignite.internal.util.ExceptionUtils.unwrapCause;
import static org.apache.ignite.lang.ErrorGroups.Common.INTERNAL_ERR;
@@ -55,10 +56,11 @@ import org.apache.ignite.internal.raft.rebalance.RaftStaleUpdateException;
import org.apache.ignite.internal.replicator.ReplicationGroupId;
import org.apache.ignite.internal.util.IgniteSpinBusyLock;
import org.apache.ignite.raft.jraft.RaftMessagesFactory;
-import org.apache.ignite.raft.jraft.entity.PeerId;
import org.apache.ignite.raft.jraft.error.RaftError;
import org.apache.ignite.raft.jraft.rpc.ActionRequest;
import org.apache.ignite.raft.jraft.rpc.ActionResponse;
+import org.apache.ignite.raft.jraft.rpc.CliRequests.ChangePeersAndLearnersAsyncRequest;
+import org.apache.ignite.raft.jraft.rpc.CliRequests.GetLeaderRequest;
import org.apache.ignite.raft.jraft.rpc.RpcRequests.ErrorResponse;
import org.apache.ignite.raft.jraft.rpc.RpcRequests.SMErrorResponse;
import org.apache.ignite.raft.jraft.rpc.impl.RaftException;
@@ -112,15 +114,15 @@ class RaftCommandExecutor {
/** State machine for tracking leader availability. */
private final LeaderAvailabilityState leaderAvailabilityState;
- /** Current leader. */
- private volatile Peer leader;
-
/** Busy lock for shutdown. */
private final IgniteSpinBusyLock busyLock = new IgniteSpinBusyLock();
/** This flag is used only for logging. */
private final AtomicBoolean peersAreUnavailable = new AtomicBoolean();
+ /** Flag indicating the executor is stopping. When true, operations fail with NodeStoppingException. */
+ private volatile boolean stopping;
+
/**
* Constructor.
*
@@ -157,7 +159,7 @@ class RaftCommandExecutor {
* @return Leader election listener callback.
*/
LeaderElectionListener leaderElectionListener() {
- return leaderAvailabilityState::onLeaderElected;
+ return (node, term) -> leaderAvailabilityState.updateKnownLeaderAndTerm(new Peer(node.name()), term);
}
/**
@@ -168,16 +170,70 @@ class RaftCommandExecutor {
* @return Future that completes with the command result.
*/
<R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
+ return this.<ActionResponse>send(
+ createRequestFactory(cmd),
+ TargetPeerStrategy.LEADER,
+ null,
+ timeoutMillis
+ ).thenApply(resp -> (R) resp.result());
+ }
+
+ /**
+ * Sends a request with leader-aware retry semantics.
+ *
+ * @param requestFactory Factory creating requests for target peer.
+ * @param targetStrategy How to select the initial target peer.
+ * @param timeoutMillis Timeout (0=single attempt, MAX_VALUE=infinite, positive=bounded).
+ * @param <R> Response type.
+ * @return Future with response.
+ */
+ <R extends NetworkMessage> CompletableFuture<R> send(
+ Function<Peer, ? extends NetworkMessage> requestFactory,
+ TargetPeerStrategy targetStrategy,
+ long timeoutMillis
+ ) {
+ return send(requestFactory, targetStrategy, null, timeoutMillis);
+ }
+
+ /**
+ * Sends a request with leader-aware retry semantics.
+ *
+ * @param requestFactory Factory creating requests for target peer.
+ * @param targetStrategy How to select the initial target peer.
+ * @param specificPeer Target peer for SPECIFIC strategy (ignored for other strategies).
+ * @param timeoutMillis Timeout (0=single attempt, MAX_VALUE=infinite, positive=bounded).
+ * @param <R> Response type.
+ * @return Future with response.
+ */
+ <R extends NetworkMessage> CompletableFuture<R> send(
+ Function<Peer, ? extends NetworkMessage> requestFactory,
+ TargetPeerStrategy targetStrategy,
+ @Nullable Peer specificPeer,
+ long timeoutMillis
+ ) {
// Normalize timeout: negative values mean infinite wait.
long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
- // Wait for leader mode (bounded or infinite).
long deadline = Utils.monotonicMsAfter(effectiveTimeout);
return executeWithBusyLock(responseFuture -> {
+ Peer initialPeer = resolveInitialPeer(targetStrategy, specificPeer);
+
+ if (initialPeer == null) {
+ responseFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId));
+ return;
+ }
+
if (effectiveTimeout == 0) {
- tryAllPeersOnce(responseFuture, cmd);
+ tryAllPeersOnce(responseFuture, requestFactory, initialPeer, targetStrategy);
} else {
- startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ startRetryPhase(
+ responseFuture,
+ requestFactory,
+ initialPeer,
+ targetStrategy,
+ deadline,
+ leaderAvailabilityState.currentTerm()
+ );
}
});
}
@@ -202,18 +258,85 @@ class RaftCommandExecutor {
}
/**
- * Resolves initial target peer for a command execution.
+ * Marks the executor as stopping. After this call, operations will fail with an exception from the
+ * stopping exception factory instead of TimeoutException.
+ */
+ void markAsStopping() {
+ stopping = true;
+ }
+
+ /**
+ * Returns the current leader.
+ *
+ * @return Current leader or {@code null} if unknown.
+ */
+ @Nullable Peer leader() {
+ return leaderAvailabilityState.leader();
+ }
+
+ /**
+ * Sets the current cached leader. Used by PhysicalTopologyAwareRaftGroupService to update state from response.
*
- * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ * @param leader New leader.
+ */
+ void setLeader(@Nullable Peer leader) {
+ leaderAvailabilityState.setCachedLeader(leader);
+ }
+
+ /**
+ * Resets the leader state completely (leader, term, availability).
+ * Used on peer reconfiguration when the cached state may no longer be valid.
+ */
+ void resetLeaderState() {
+ leaderAvailabilityState.resetLeaderState();
+ }
+
+ /**
+ * Updates the leader only if the given term is newer than the currently known term.
*
+ * <p>This prevents stale leader information from overwriting fresher information
+ * that may have been received via leader election notifications or previous refresh calls.
+ *
+ * @param leader New leader (can be {@code null}).
+ * @param term Term associated with this leader.
+ * @return {@code true} if the leader was updated (term is newer), {@code false} otherwise.
+ */
+ boolean setLeaderIfTermNewer(@Nullable Peer leader, long term) {
+ return leaderAvailabilityState.updateKnownLeaderAndTerm(leader, term);
+ }
+
+ /**
+ * Returns the cluster service.
+ *
+ * @return Cluster service.
+ */
+ ClusterService clusterService() {
+ return clusterService;
+ }
+
+ /**
+ * Resolves initial target peer based on strategy.
+ *
+ * @param strategy Target peer strategy.
+ * @param specificPeer Specific peer for SPECIFIC strategy.
* @return Initial target peer, or {@code null}.
*/
- private @Nullable Peer resolveInitialPeer() {
- Peer targetPeer = leader;
- if (targetPeer == null) {
- targetPeer = randomNode(null, false);
+ private @Nullable Peer resolveInitialPeer(TargetPeerStrategy strategy, @Nullable Peer specificPeer) {
+ switch (strategy) {
+ case LEADER:
+ // Current behavior: try known leader, fallback to random.
+ Peer targetPeer = leaderAvailabilityState.leader();
+ return targetPeer != null ? targetPeer : randomNode(null, false);
+ case RANDOM:
+ return randomNode(null, false);
+ case SPECIFIC:
+ if (specificPeer == null) {
+ throw new IllegalArgumentException("specificPeer required for SPECIFIC strategy");
+ }
+ return specificPeer;
+ default:
+ throw new AssertionError("Unknown strategy: " + strategy);
}
- return targetPeer;
}
/**
@@ -221,55 +344,44 @@ class RaftCommandExecutor {
*
* @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
* peer responds successfully.
- * @param cmd The command to execute.
+ * @param requestFactory Factory creating requests for target peer.
+ * @param initialPeer Initial peer to send the request to.
+ * @param targetStrategy Target peer strategy.
*/
- private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
- Peer targetPeer = resolveInitialPeer();
- if (targetPeer == null) {
- resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId));
-
- return;
- }
-
+ private <R extends NetworkMessage> void tryAllPeersOnce(
+ CompletableFuture<R> resultFuture,
+ Function<Peer, ? extends NetworkMessage> requestFactory,
+ Peer initialPeer,
+ TargetPeerStrategy targetStrategy
+ ) {
var context = new RetryContext(
groupId.toString(),
- targetPeer,
- cmd::toStringForLightLogging,
- createRequestFactory(cmd),
+ initialPeer,
+ () -> null,
+ requestFactory,
0, // Single attempt - no retry timeout
raftConfiguration.responseTimeoutMillis().value()
);
- sendWithRetrySingleAttempt(resultFuture, context);
+ sendWithRetrySingleAttempt(resultFuture, context, targetStrategy);
}
/**
- * Executes an action within busy lock and transforms the response.
+ * Executes an action within busy lock.
*/
- @SuppressWarnings("unchecked")
- private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
- var responseFuture = new CompletableFuture<ActionResponse>();
- var resultFuture = new CompletableFuture<R>();
+ private <R extends NetworkMessage> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<R>> action) {
+ var responseFuture = new CompletableFuture<R>();
if (!busyLock.enterBusy()) {
- resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [groupId=" + groupId + "]."));
-
- return resultFuture;
+ responseFuture.completeExceptionally(
+ stoppingExceptionFactory.create("Raft client is stopping [groupId=" + groupId + "].")
+ );
+ return responseFuture;
}
try {
action.accept(responseFuture);
-
- // Transform ActionResponse to result type.
- responseFuture.whenComplete((resp, err) -> {
- if (err != null) {
- resultFuture.completeExceptionally(err);
- } else {
- resultFuture.complete((R) resp.result());
- }
- });
-
- return resultFuture;
+ return responseFuture;
} finally {
busyLock.leaveBusy();
}
@@ -304,9 +416,18 @@ class RaftCommandExecutor {
}
/**
- * Waits for leader to become available and then retries the command.
+ * Waits for leader to become available and then retries the request.
+ *
+ * @param originalPeer Original peer from strategy. For SPECIFIC, used both for resolving and retry validation.
+ * For LEADER/RANDOM, passed to resolveInitialPeer (which ignores it and resolves fresh).
*/
- private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ private <R extends NetworkMessage> void waitForLeaderAndRetry(
+ CompletableFuture<R> resultFuture,
+ Function<Peer, ? extends NetworkMessage> requestFactory,
+ TargetPeerStrategy targetStrategy,
+ Peer originalPeer,
+ long deadline
+ ) {
CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
// Apply timeout if bounded.
@@ -324,13 +445,22 @@ class RaftCommandExecutor {
}
if (!busyLock.enterBusy()) {
- resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [groupId=" + groupId + "]."));
+ resultFuture.completeExceptionally(
+ stoppingExceptionFactory.create("Raft client is stopping [groupId=" + groupId + "].")
+ );
return;
}
try {
- // Leader is available, now run the command with retry logic.
- startRetryPhase(resultFuture, cmd, deadline, term);
+ // Leader is available, now run the request with retry logic.
+ // For SPECIFIC, resolveInitialPeer returns originalPeer unchanged.
+ // For LEADER/RANDOM, it resolves a fresh peer (ignoring originalPeer).
+ Peer initialPeer = resolveInitialPeer(targetStrategy, originalPeer);
+ if (initialPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId));
+ return;
+ }
+ startRetryPhase(resultFuture, requestFactory, initialPeer, targetStrategy, deadline, term);
} finally {
busyLock.leaveBusy();
}
@@ -339,12 +469,22 @@ class RaftCommandExecutor {
/**
* Starts the retry phase after leader is available.
+ *
+ * @param initialPeer Initial peer to send to. For SPECIFIC strategy, this is also used for retry validation.
*/
- private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
- Peer targetPeer = resolveInitialPeer();
- if (targetPeer == null) {
- resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId));
-
+ private <R extends NetworkMessage> void startRetryPhase(
+ CompletableFuture<R> resultFuture,
+ Function<Peer, ? extends NetworkMessage> requestFactory,
+ Peer initialPeer,
+ TargetPeerStrategy targetStrategy,
+ long deadline,
+ long term
+ ) {
+ // Check stopping flag before timeout check.
+ if (stopping) {
+ resultFuture.completeExceptionally(
+ stoppingExceptionFactory.create("Raft client is stopping [groupId=" + groupId + "].")
+ );
return;
}
@@ -357,19 +497,18 @@ class RaftCommandExecutor {
// Use retry timeout bounded by remaining time until deadline.
long configTimeout = raftConfiguration.retryTimeoutMillis().value();
-
long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
var context = new RetryContext(
groupId.toString(),
- targetPeer,
- cmd::toStringForLightLogging,
- createRequestFactory(cmd),
+ initialPeer,
+ () -> null,
+ requestFactory,
sendWithRetryTimeoutMillis,
raftConfiguration.responseTimeoutMillis().value()
);
- sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ sendWithRetryWaitingForLeader(resultFuture, context, requestFactory, targetStrategy, initialPeer, deadline, term);
}
/**
@@ -398,7 +537,8 @@ class RaftCommandExecutor {
*/
private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
CompletableFuture<R> fut,
- RetryContext retryContext
+ RetryContext retryContext,
+ TargetPeerStrategy targetStrategy
) {
if (!busyLock.enterBusy()) {
fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [groupId=" + groupId + "]."));
@@ -418,7 +558,7 @@ class RaftCommandExecutor {
resp,
err,
retryContext,
- new SingleAttemptRetryStrategy(fut)
+ new SingleAttemptRetryStrategy<>(fut, targetStrategy)
),
executor
);
@@ -452,41 +592,6 @@ class RaftCommandExecutor {
return RetryPeerResult.success(nextPeer);
}
- /**
- * Result of attempting to select a peer for retry.
- */
- private static final class RetryPeerResult {
- private final @Nullable Peer peer;
- private final @Nullable Throwable error;
-
- private RetryPeerResult(@Nullable Peer peer, @Nullable Throwable error) {
- this.peer = peer;
- this.error = error;
- }
-
- static RetryPeerResult success(Peer peer) {
- return new RetryPeerResult(peer, null);
- }
-
- static RetryPeerResult fail(Throwable error) {
- return new RetryPeerResult(null, error);
- }
-
- boolean isSuccess() {
- return peer != null;
- }
-
- Peer peer() {
- assert peer != null : "Check isSuccess() before calling peer()";
- return peer;
- }
-
- Throwable error() {
- assert error != null : "Check isSuccess() before calling error()";
- return error;
- }
- }
-
private static void logRecoverableError(RetryContext retryContext, Peer nextPeer) {
if (LOG.isDebugEnabled()) {
LOG.debug(
@@ -532,7 +637,12 @@ class RaftCommandExecutor {
} else if (resp instanceof SMErrorResponse) {
fut.completeExceptionally(extractSmError((SMErrorResponse) resp, retryContext));
} else {
- leader = retryContext.targetPeer();
+ // Only cache the leader when using LEADER strategy - in that case, a successful
+ // response means the target peer is the leader. For RANDOM/SPECIFIC strategies,
+ // the target peer is not necessarily the leader.
+ if (strategy.targetSelectionStrategy() == TargetPeerStrategy.LEADER) {
+ leaderAvailabilityState.setCachedLeader(retryContext.targetPeer());
+ }
fut.complete((R) resp);
}
} catch (Throwable e) {
@@ -565,11 +675,15 @@ class RaftCommandExecutor {
/**
* Sends a request with retry and waits for leader on leader absence.
+ *
+ * @param originalPeer Original peer from strategy. For SPECIFIC, used for retry validation.
*/
- private void sendWithRetryWaitingForLeader(
- CompletableFuture<ActionResponse> fut,
+ private <R extends NetworkMessage> void sendWithRetryWaitingForLeader(
+ CompletableFuture<R> fut,
RetryContext retryContext,
- Command cmd,
+ Function<Peer, ? extends NetworkMessage> requestFactory,
+ TargetPeerStrategy targetStrategy,
+ Peer originalPeer,
long deadline,
long termWhenStarted
) {
@@ -579,13 +693,32 @@ class RaftCommandExecutor {
}
try {
+ // Check stopping flag before timeout check.
+ if (stopping) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [groupId=" + groupId + "]."));
+ return;
+ }
+
long requestStartTime = Utils.monotonicMs();
long stopTime = retryContext.stopTime();
if (requestStartTime >= stopTime) {
- // Retry timeout expired - fail with timeout exception.
- // For non-leader-related retriable errors, we don't wait for leader, just fail.
- fut.completeExceptionally(createTimeoutException());
+ // Retry phase timeout expired.
+ if (deadline == Long.MAX_VALUE) {
+ // Infinite wait mode: start a new retry phase.
+ // We may not have probed all peers yet (e.g., with many peers and short retry timeout),
+ // so we can't assume there's no leader. Start fresh and keep trying.
+ LOG.debug("Retry phase timeout expired with infinite deadline, starting new retry phase [groupId={}]", groupId);
+ Peer initialPeer = resolveInitialPeer(targetStrategy, originalPeer);
+ if (initialPeer == null) {
+ fut.completeExceptionally(new ReplicationGroupUnavailableException(groupId));
+ return;
+ }
+ startRetryPhase(fut, requestFactory, initialPeer, targetStrategy, deadline, termWhenStarted);
+ } else {
+ // Bounded wait mode: fail with timeout exception.
+ fut.completeExceptionally(createTimeoutException());
+ }
return;
}
@@ -604,7 +737,9 @@ class RaftCommandExecutor {
resp,
err,
retryContext,
- new LeaderWaitRetryStrategy(fut, cmd, deadline, termWhenStarted)
+ new LeaderWaitRetryStrategy<>(
+ fut, requestFactory, targetStrategy, originalPeer, deadline, termWhenStarted
+ )
),
executor
);
@@ -663,18 +798,6 @@ class RaftCommandExecutor {
return CompletableFuture.completedFuture(node);
}
- /**
- * Parses a peer ID string to a Peer object.
- *
- * @param peerId Peer ID string in format "consistentId:idx" or just "consistentId".
- * @return Parsed Peer object, or {@code null} if parsing fails.
- */
- private static @Nullable Peer parsePeer(@Nullable String peerId) {
- PeerId id = PeerId.parsePeer(peerId);
-
- return id == null ? null : new Peer(id.getConsistentId(), id.getIdx());
- }
-
/**
* Returns a random peer excluding unavailable peers and optionally "no leader" peers.
*
@@ -719,13 +842,12 @@ class RaftCommandExecutor {
if (!peersAreUnavailable.getAndSet(true)) {
LOG.warn(
"All peers are unavailable, going to keep retrying until timeout [peers = {}, group = {}, trace ID: {}, "
- + "request {}, origin command {}, instance={}].",
+ + "request {}, origin command {}].",
localPeers,
groupId,
retryContext.errorTraceId(),
retryContext.request().toStringForLightLogging(),
- retryContext.originCommandDescription(),
- this
+ retryContext.originCommandDescription()
);
}
@@ -759,59 +881,6 @@ class RaftCommandExecutor {
return format("Peer {} returned code {}: {}", retryContext.targetPeer().consistentId(), error, resp.errorMsg());
}
- /**
- * How to track the current peer when moving to a new one.
- */
- private enum PeerTracking {
- /** Don't mark the current peer (transient errors, leader redirects). */
- COMMON,
- /** Mark as unavailable (down, shutting down). */
- UNAVAILABLE,
- /** Mark as "no leader" (working but doesn't know leader). */
- NO_LEADER
- }
-
- /**
- * Strategy for executing retries. Each method receives everything needed to perform the retry.
- *
- * <p>This interface abstracts the differences between single-attempt mode and leader-wait mode:
- * <ul>
- * <li><b>Single-attempt mode</b>: Each peer is tried at most once. All errors mark the peer
- * as unavailable. When all peers exhausted, fail immediately.</li>
- * <li><b>Leader-wait mode</b>: Transient errors retry on the same peer with delay.
- * "No leader" errors track peers separately. When exhausted, wait for leader notification.</li>
- * </ul>
- */
- private interface RetryExecutionStrategy {
- /**
- * Executes retry on the specified peer with the given tracking for the current peer.
- *
- * @param context Current retry context.
- * @param nextPeer Peer to retry on.
- * @param trackCurrentAs How to track the current peer ({@link PeerTracking#COMMON} for "don't track").
- * @param reason Human-readable reason for the retry.
- */
- void executeRetry(RetryContext context, Peer nextPeer, PeerTracking trackCurrentAs, String reason);
-
- /**
- * Called when all peers have been exhausted.
- *
- * <p>In leader-wait mode, this triggers waiting for leader notification.
- * In single-attempt mode, this completes with {@link ReplicationGroupUnavailableException}.
- */
- void onAllPeersExhausted();
-
- /**
- * Whether to track "no leader" peers separately from unavailable peers.
- *
- * <p>In leader-wait mode, peers that return "no leader" are tracked separately so that
- * when all peers are exhausted, the strategy can wait for a leader notification rather
- * than failing immediately. In single-attempt mode, all errors are treated uniformly
- * as unavailable.
- */
- boolean trackNoLeaderSeparately();
- }
-
/**
* Retry strategy for single-attempt mode.
*
@@ -819,15 +888,26 @@ class RaftCommandExecutor {
* as unavailable. When all peers have been tried, the request fails with
* {@link ReplicationGroupUnavailableException}.
*/
- private class SingleAttemptRetryStrategy implements RetryExecutionStrategy {
- private final CompletableFuture<? extends NetworkMessage> fut;
+ private class SingleAttemptRetryStrategy<R extends NetworkMessage> implements RetryExecutionStrategy {
+ private final CompletableFuture<R> futureInvokeResult;
+ private final TargetPeerStrategy targetSelectionStrategy;
- SingleAttemptRetryStrategy(CompletableFuture<? extends NetworkMessage> fut) {
- this.fut = fut;
+ SingleAttemptRetryStrategy(
+ CompletableFuture<R> futureInvokeResult,
+ TargetPeerStrategy targetSelectionStrategy
+ ) {
+ this.futureInvokeResult = futureInvokeResult;
+ this.targetSelectionStrategy = targetSelectionStrategy;
}
@Override
public void executeRetry(RetryContext context, Peer nextPeer, PeerTracking trackCurrentAs, String reason) {
+ // For SPECIFIC strategy, don't try other peers.
+ if (targetSelectionStrategy == TargetPeerStrategy.SPECIFIC) {
+ onAllPeersExhausted();
+ return;
+ }
+
// In single-attempt mode, each peer is tried at most once.
// For transient errors, the caller passes the same peer (meaning "retry on same peer"),
// but we must select a different one instead.
@@ -840,18 +920,24 @@ class RaftCommandExecutor {
return;
}
- sendWithRetrySingleAttempt(fut, context.nextAttemptForUnavailablePeer(effectiveNextPeer, reason));
+ sendWithRetrySingleAttempt(
+ futureInvokeResult, context.nextAttemptForUnavailablePeer(effectiveNextPeer, reason), targetSelectionStrategy);
}
@Override
public void onAllPeersExhausted() {
- fut.completeExceptionally(new ReplicationGroupUnavailableException(groupId));
+ futureInvokeResult.completeExceptionally(new ReplicationGroupUnavailableException(groupId));
}
@Override
public boolean trackNoLeaderSeparately() {
return false;
}
+
+ @Override
+ public TargetPeerStrategy targetSelectionStrategy() {
+ return targetSelectionStrategy;
+ }
}
/**
@@ -864,20 +950,27 @@ class RaftCommandExecutor {
* <li>When all peers are exhausted, waits for leader notification</li>
* </ul>
*/
- private class LeaderWaitRetryStrategy implements RetryExecutionStrategy {
- private final CompletableFuture<ActionResponse> fut;
- private final Command cmd;
+ private class LeaderWaitRetryStrategy<R extends NetworkMessage> implements RetryExecutionStrategy {
+ private final CompletableFuture<R> futureInvokeResult;
+ private final Function<Peer, ? extends NetworkMessage> requestFactory;
+ private final TargetPeerStrategy targetSelectionStrategy;
+ /** Original peer from strategy. For SPECIFIC, used for retry validation and re-resolution. */
+ private final Peer originalPeer;
private final long deadline;
private final long termWhenStarted;
LeaderWaitRetryStrategy(
- CompletableFuture<ActionResponse> fut,
- Command cmd,
+ CompletableFuture<R> futureInvokeResult,
+ Function<Peer, ? extends NetworkMessage> requestFactory,
+ TargetPeerStrategy targetSelectionStrategy,
+ Peer originalPeer,
long deadline,
long termWhenStarted
) {
- this.fut = fut;
- this.cmd = cmd;
+ this.futureInvokeResult = futureInvokeResult;
+ this.requestFactory = requestFactory;
+ this.targetSelectionStrategy = targetSelectionStrategy;
+ this.originalPeer = originalPeer;
this.deadline = deadline;
this.termWhenStarted = termWhenStarted;
}
@@ -899,8 +992,17 @@ class RaftCommandExecutor {
throw new AssertionError("Unexpected tracking: " + trackCurrentAs);
}
+ // For SPECIFIC strategy, only allow retry to the same peer.
+ if (targetSelectionStrategy == TargetPeerStrategy.SPECIFIC && !nextPeer.equals(originalPeer)) {
+ onAllPeersExhausted();
+ return;
+ }
+
executor.schedule(
- () -> sendWithRetryWaitingForLeader(fut, nextContext, cmd, deadline, termWhenStarted),
+ () -> sendWithRetryWaitingForLeader(
+ futureInvokeResult, nextContext, requestFactory,
+ targetSelectionStrategy, originalPeer, deadline, termWhenStarted
+ ),
raftConfiguration.retryDelayMillis().value(),
TimeUnit.MILLISECONDS
);
@@ -908,14 +1010,27 @@ class RaftCommandExecutor {
@Override
public void onAllPeersExhausted() {
+ // For RANDOM and SPECIFIC strategies, fail immediately - don't wait for leader.
+ if (targetSelectionStrategy != TargetPeerStrategy.LEADER) {
+ futureInvokeResult.completeExceptionally(new ReplicationGroupUnavailableException(groupId));
+ return;
+ }
+
LOG.debug("All peers exhausted, waiting for leader [groupId={}, term={}]", groupId, termWhenStarted);
leaderAvailabilityState.onGroupUnavailable(termWhenStarted);
- waitForLeaderAndRetry(fut, cmd, deadline);
+ waitForLeaderAndRetry(futureInvokeResult, requestFactory, targetSelectionStrategy, originalPeer, deadline);
}
@Override
public boolean trackNoLeaderSeparately() {
- return true;
+ // For LEADER strategy, track no-leader peers separately so we can wait for leader.
+ // For RANDOM/SPECIFIC, treat all errors uniformly.
+ return targetSelectionStrategy == TargetPeerStrategy.LEADER;
+ }
+
+ @Override
+ public TargetPeerStrategy targetSelectionStrategy() {
+ return targetSelectionStrategy;
}
}
@@ -978,19 +1093,38 @@ class RaftCommandExecutor {
switch (error) {
case SUCCESS:
- leader = retryContext.targetPeer();
+ leaderAvailabilityState.setCachedLeader(retryContext.targetPeer());
fut.complete(null);
break;
case EBUSY:
case EAGAIN:
- case UNKNOWN:
- case EINTERNAL:
- case ENOENT:
// Transient errors - retry on same peer (COMMON = don't mark current peer).
strategy.executeRetry(retryContext, retryContext.targetPeer(), PeerTracking.COMMON, reason);
break;
+ case UNKNOWN:
+ case EINTERNAL:
+ case ENOENT: {
+ // These errors may indicate the target peer is doing rebalancing.
+ // For leader discovery and configuration change requests, try another peer.
+ // For other requests, retry on the same peer.
+ NetworkMessage request = retryContext.request();
+ Peer newTargetPeer;
+
+ if (request instanceof GetLeaderRequest || request instanceof ChangePeersAndLearnersAsyncRequest) {
+ newTargetPeer = randomNode(retryContext, false);
+ if (newTargetPeer == null) {
+ newTargetPeer = retryContext.targetPeer();
+ }
+ } else {
+ newTargetPeer = retryContext.targetPeer();
+ }
+
+ strategy.executeRetry(retryContext, newTargetPeer, PeerTracking.COMMON, reason);
+ break;
+ }
+
case EHOSTDOWN:
case ESHUTDOWN:
case ENODESHUTDOWN:
@@ -1012,7 +1146,7 @@ class RaftCommandExecutor {
throw new IllegalStateException("parsePeer returned null for non-null leaderId: " + resp.leaderId());
}
- leader = leaderPeer;
+ leaderAvailabilityState.setCachedLeader(leaderPeer);
strategy.executeRetry(retryContext, leaderPeer, PeerTracking.COMMON, reason);
}
break;
diff --git a/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/RaftPeerUtils.java b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/RaftPeerUtils.java
new file mode 100644
index 00000000000..0940cf7734b
--- /dev/null
+++ b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/RaftPeerUtils.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static java.util.stream.Collectors.toList;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.ignite.internal.raft.Peer;
+import org.apache.ignite.raft.jraft.entity.PeerId;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Utility methods for RAFT peer operations.
+ */
+final class RaftPeerUtils {
+ private RaftPeerUtils() {
+ // No-op.
+ }
+
+ /**
+ * Parses a peer ID string to a Peer object.
+ *
+ * @param peerId Peer ID string in format "consistentId:idx" or just "consistentId".
+ * @return Parsed Peer object, or {@code null} if parsing fails or input is null.
+ */
+ static @Nullable Peer parsePeer(@Nullable String peerId) {
+ PeerId id = PeerId.parsePeer(peerId);
+
+ return id == null ? null : new Peer(id.getConsistentId(), id.getIdx());
+ }
+
+ /**
+ * Parse list of peers from list with string representations.
+ */
+ static List<Peer> parsePeerList(@Nullable Collection<String> peers) {
+ if (peers == null) {
+ return List.of();
+ }
+
+ List<Peer> res = new ArrayList<>(peers.size());
+ for (String peer : peers) {
+ res.add(parsePeer(peer));
+ }
+ return res;
+ }
+
+ /**
+ * Converts a Peer to string ID.
+ */
+ static String peerId(Peer peer) {
+ return PeerId.fromPeer(peer).toString();
+ }
+
+ /**
+ * Converts a collection of Peers to string IDs.
+ */
+ static List<String> peerIds(Collection<Peer> peers) {
+ return peers.stream().map(RaftPeerUtils::peerId).collect(toList());
+ }
+}
diff --git a/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/RetryContext.java b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/RetryContext.java
index ccf60bede8c..8e567979839 100644
--- a/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/RetryContext.java
+++ b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/RetryContext.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.raft.client;
+import static java.util.Collections.unmodifiableSet;
import static org.apache.ignite.internal.lang.IgniteStringFormatter.format;
import java.time.Instant;
@@ -44,9 +45,6 @@ import org.jetbrains.annotations.Nullable;
* calls.
*/
class RetryContext {
- /** Indicates that default response timeout should be used. */
- static final long USE_DEFAULT_RESPONSE_TIMEOUT = -1;
-
private static final int MAX_RETRY_REASONS = 25;
private static final DateTimeFormatter TIMESTAMP_FORMATTER =
@@ -165,7 +163,7 @@ class RetryContext {
}
Set<Peer> unavailablePeers() {
- return unavailablePeers;
+ return unmodifiableSet(unavailablePeers);
}
UUID errorTraceId() {
@@ -189,7 +187,7 @@ class RetryContext {
}
Set<Peer> noLeaderPeers() {
- return noLeaderPeers;
+ return unmodifiableSet(noLeaderPeers);
}
/**
diff --git a/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/RetryExecutionStrategy.java b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/RetryExecutionStrategy.java
new file mode 100644
index 00000000000..99eeb577401
--- /dev/null
+++ b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/RetryExecutionStrategy.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import org.apache.ignite.internal.raft.Peer;
+
+/**
+ * Strategy for executing retries. Each method receives everything needed to perform the retry.
+ *
+ * <p>This interface abstracts the differences between single-attempt mode and leader-wait mode:
+ * <ul>
+ * <li><b>Single-attempt mode</b>: Each peer is tried at most once. All errors mark the peer
+ * as unavailable. When all peers exhausted, fail immediately.</li>
+ * <li><b>Leader-wait mode</b>: Transient errors retry on the same peer with delay.
+ * "No leader" errors track peers separately. When exhausted, wait for leader notification.</li>
+ * </ul>
+ */
+interface RetryExecutionStrategy {
+ /**
+ * Executes retry on the specified peer with the given tracking for the current peer.
+ *
+ * @param context Current retry context.
+ * @param nextPeer Peer to retry on.
+ * @param trackCurrentAs How to track the current peer ({@link PeerTracking#COMMON} for "don't track").
+ * @param reason Human-readable reason for the retry.
+ */
+ void executeRetry(RetryContext context, Peer nextPeer, PeerTracking trackCurrentAs, String reason);
+
+ /**
+ * Called when all peers have been exhausted.
+ *
+ * <p>In leader-wait mode, this triggers waiting for leader notification.
+ * In single-attempt mode, this completes with {@link org.apache.ignite.internal.raft.ReplicationGroupUnavailableException}.
+ */
+ void onAllPeersExhausted();
+
+ /**
+ * Whether to track "no leader" peers separately from unavailable peers.
+ *
+ * <p>In leader-wait mode, peers that return "no leader" are tracked separately so that
+ * when all peers are exhausted, the strategy can wait for a leader notification rather
+ * than failing immediately. In single-attempt mode, all errors are treated uniformly
+ * as unavailable.
+ */
+ boolean trackNoLeaderSeparately();
+
+ /**
+ * Returns the target peer strategy used for this request.
+ */
+ TargetPeerStrategy targetSelectionStrategy();
+}
diff --git a/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/RetryPeerResult.java b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/RetryPeerResult.java
new file mode 100644
index 00000000000..ef2f3c24658
--- /dev/null
+++ b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/RetryPeerResult.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import org.apache.ignite.internal.raft.Peer;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Result of attempting to select a peer for retry.
+ */
+final class RetryPeerResult {
+ private final @Nullable Peer peer;
+ private final @Nullable Throwable error;
+
+ private RetryPeerResult(@Nullable Peer peer, @Nullable Throwable error) {
+ this.peer = peer;
+ this.error = error;
+ }
+
+ static RetryPeerResult success(Peer peer) {
+ return new RetryPeerResult(peer, null);
+ }
+
+ static RetryPeerResult fail(Throwable error) {
+ return new RetryPeerResult(null, error);
+ }
+
+ boolean isSuccess() {
+ return peer != null;
+ }
+
+ Peer peer() {
+ assert peer != null : "Check isSuccess() before calling peer()";
+ return peer;
+ }
+
+ Throwable error() {
+ assert error != null : "Check isSuccess() before calling error()";
+ return error;
+ }
+}
diff --git a/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/SubscriptionMessageSender.java b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/SubscriptionMessageSender.java
new file mode 100644
index 00000000000..9ebea5704a8
--- /dev/null
+++ b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/SubscriptionMessageSender.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.util.ExceptionUtils.unwrapCause;
+
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ScheduledExecutorService;
+import org.apache.ignite.internal.lang.NodeStoppingException;
+import org.apache.ignite.internal.logger.IgniteLogger;
+import org.apache.ignite.internal.logger.Loggers;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.network.MessagingService;
+import org.apache.ignite.internal.network.RecipientLeftException;
+import org.apache.ignite.internal.network.handshake.HandshakeException;
+import org.apache.ignite.internal.raft.configuration.RaftConfiguration;
+import org.apache.ignite.raft.jraft.rpc.CliRequests.SubscriptionLeaderChangeRequest;
+
+/**
+ * Handles sending subscription messages to RAFT group members with retry logic.
+ */
+class SubscriptionMessageSender {
+ private static final IgniteLogger LOG = Loggers.forClass(SubscriptionMessageSender.class);
+
+ private final ScheduledExecutorService executor;
+
+ private final RaftConfiguration raftConfiguration;
+
+ private final MessagingService messagingService;
+
+ /**
+ * Constructor.
+ *
+ * @param messagingService Messaging service.
+ * @param executor Executor for async operations.
+ * @param raftConfiguration RAFT configuration.
+ */
+ SubscriptionMessageSender(
+ MessagingService messagingService,
+ ScheduledExecutorService executor,
+ RaftConfiguration raftConfiguration
+ ) {
+ this.messagingService = messagingService;
+ this.executor = executor;
+ this.raftConfiguration = raftConfiguration;
+ }
+
+ /**
+ * Sends a subscription message to the specified node with retry logic.
+ *
+ * @param node Target node.
+ * @param msg Subscription message to send.
+ * @return Future that completes with {@code true} if the message was sent successfully,
+ * {@code false} if sending failed but should not be treated as an error,
+ * or completes exceptionally on unrecoverable errors.
+ */
+ CompletableFuture<Boolean> send(InternalClusterNode node, SubscriptionLeaderChangeRequest msg) {
+ var msgSendFut = new CompletableFuture<Boolean>();
+
+ sendWithRetry(node, msg, msgSendFut);
+
+ return msgSendFut;
+ }
+
+ private void sendWithRetry(InternalClusterNode node, SubscriptionLeaderChangeRequest msg, CompletableFuture<Boolean> msgSendFut) {
+ Long responseTimeout = raftConfiguration.responseTimeoutMillis().value();
+
+ messagingService.invoke(node, msg, responseTimeout).whenCompleteAsync((unused, invokeThrowable) -> {
+ if (invokeThrowable == null) {
+ msgSendFut.complete(true);
+
+ return;
+ }
+
+ Throwable retryCause = unwrapCause(invokeThrowable);
+ if (!msg.subscribe()) {
+ // We don't want to propagate exceptions when unsubscribing (if it's not an Error!).
+ if (retryCause instanceof Error) {
+ msgSendFut.completeExceptionally(invokeThrowable);
+ } else {
+ LOG.debug("An exception while trying to unsubscribe.", invokeThrowable);
+
+ msgSendFut.complete(false);
+ }
+ } else if (RaftErrorUtils.recoverable(retryCause)) {
+ sendWithRetry(node, msg, msgSendFut);
+ } else if (retryCause instanceof RecipientLeftException) {
+ LOG.info(
+ "Could not subscribe to leader update from a specific node, because the node had left the cluster: [node={}].",
+ node
+ );
+
+ msgSendFut.complete(false);
+ } else if (retryCause instanceof HandshakeException) {
+ LOG.info(
+ "Could not subscribe to leader update from a specific node, because the handshake failed "
+ + "(node may be unavailable): [node={}].",
+ node
+ );
+
+ msgSendFut.complete(false);
+ } else if (retryCause instanceof CancellationException) {
+ LOG.info(
+ "Could not subscribe to leader update from a specific node, because the operation was cancelled "
+ + "(node may be stopping): [node={}].",
+ node
+ );
+
+ msgSendFut.complete(false);
+ } else if (retryCause instanceof NodeStoppingException) {
+ LOG.warn(
+ "Could not subscribe to leader update from a specific node, because the node is stopping: [node={}].",
+ node
+ );
+
+ msgSendFut.complete(false);
+ } else {
+ LOG.error("Could not send the subscribe message to the node: [node={}, msg={}].", invokeThrowable, node, msg);
+
+ msgSendFut.completeExceptionally(invokeThrowable);
+ }
+ }, executor);
+ }
+}
diff --git a/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/TargetPeerStrategy.java b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/TargetPeerStrategy.java
new file mode 100644
index 00000000000..832cd1d4476
--- /dev/null
+++ b/modules/raft/src/main/java/org/apache/ignite/internal/raft/client/TargetPeerStrategy.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+/**
+ * Strategy for selecting the target peer for a request.
+ */
+enum TargetPeerStrategy {
+ /**
+ * Most operations - targets the leader, waits for leader if not available.
+ * On all peers exhausted, waits for leader notification.
+ */
+ LEADER,
+
+ /**
+ * Operations that don't need a leader (e.g., refreshLeader).
+ * Any peer works, fails immediately when all peers exhausted.
+ */
+ RANDOM,
+
+ /**
+ * Operations that must go to a specific peer (e.g., snapshot).
+ * Never retries to other peers, only retries to the same peer on transient errors.
+ */
+ SPECIFIC
+}
diff --git a/modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java b/modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java
index acc1b3a7af8..b99e091d49f 100644
--- a/modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java
+++ b/modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java
@@ -17,9 +17,9 @@
package org.apache.ignite.internal.raft.client;
-import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -28,11 +28,9 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
-import org.apache.ignite.internal.network.ClusterNodeImpl;
-import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.Peer;
import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
-import org.apache.ignite.network.NetworkAddress;
import org.junit.jupiter.api.Test;
/**
@@ -50,6 +48,7 @@ public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
assertEquals(State.WAITING_FOR_LEADER, state.currentState());
assertEquals(-1, state.currentTerm());
+ assertNull(state.leader());
// Already in WAITING_FOR_LEADER, should have no effect
state.onGroupUnavailable(0);
@@ -69,14 +68,14 @@ public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
}
/**
- * {@link LeaderAvailabilityState#onLeaderElected(InternalClusterNode, long)} transitions to {@link State#LEADER_AVAILABLE} state and
+ * {@link LeaderAvailabilityState#updateKnownLeaderAndTerm(Peer, long)} transitions to {@link State#LEADER_AVAILABLE} state and
* completes waiting futures.
* {@link LeaderAvailabilityState#awaitLeader()} returns completed future when leader is available.
*/
@Test
void testAwaitLeader() {
LeaderAvailabilityState state = new LeaderAvailabilityState();
- InternalClusterNode leaderNode = createNode("leader-node");
+ Peer leaderPeer = new Peer("leader-node");
long expectedTerm = 1;
@@ -84,11 +83,12 @@ public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
CompletableFuture<Long> waiter = state.awaitLeader();
assertFalse(waiter.isDone());
- state.onLeaderElected(leaderNode, expectedTerm);
+ state.updateKnownLeaderAndTerm(leaderPeer, expectedTerm);
- // Verify leader is available after onLeaderElected.
+ // Verify leader is available after updateKnownLeaderAndTerm.
assertEquals(State.LEADER_AVAILABLE, state.currentState());
assertEquals(expectedTerm, state.currentTerm());
+ assertEquals(leaderPeer, state.leader());
assertTrue(waiter.isDone());
assertEquals(expectedTerm, waiter.join());
@@ -104,50 +104,50 @@ public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
assertEquals(expectedTerm, state.currentTerm());
}
- /** Stale term notifications are ignored. */
+ /** Stale term notifications are ignored and return false. */
@Test
void testStaleTermIgnored() {
LeaderAvailabilityState state = new LeaderAvailabilityState();
- InternalClusterNode leaderNode1 = createNode("leader-1");
- InternalClusterNode leaderNode2 = createNode("leader-2");
+ Peer peer1 = new Peer("leader-1");
+ Peer peer2 = new Peer("leader-2");
long expectedTerm = 5;
- state.onLeaderElected(leaderNode1, expectedTerm);
+ assertTrue(state.updateKnownLeaderAndTerm(peer1, expectedTerm));
assertEquals(expectedTerm, state.currentTerm());
- // Stale notification with lower term should be ignored.
- state.onLeaderElected(leaderNode2, 3);
+ // Stale notification with lower term should be ignored and return false.
+ assertFalse(state.updateKnownLeaderAndTerm(peer2, 3));
assertEquals(expectedTerm, state.currentTerm());
}
- /** Equal term notifications are ignored. */
+ /** Equal term notifications are ignored and return false. */
@Test
void testEqualTermIgnored() {
LeaderAvailabilityState state = new LeaderAvailabilityState();
- InternalClusterNode leaderNode = createNode("leader");
+ Peer peer = new Peer("leader");
long expectedTerm = 5;
- state.onLeaderElected(leaderNode, expectedTerm);
+ assertTrue(state.updateKnownLeaderAndTerm(peer, expectedTerm));
assertEquals(expectedTerm, state.currentTerm());
- // Same term should be ignored.
- state.onLeaderElected(leaderNode, expectedTerm);
+ // Same term should be ignored and return false.
+ assertFalse(state.updateKnownLeaderAndTerm(peer, expectedTerm));
assertEquals(expectedTerm, state.currentTerm());
}
- /** Higher term updates current term. */
+ /** Higher term updates current term and returns true. */
@Test
void testHigherTermUpdates() {
LeaderAvailabilityState state = new LeaderAvailabilityState();
- InternalClusterNode leaderNode1 = createNode("leader-1");
- InternalClusterNode leaderNode2 = createNode("leader-2");
+ Peer peer1 = new Peer("leader-1");
+ Peer peer2 = new Peer("leader-2");
- state.onLeaderElected(leaderNode1, 1);
+ assertTrue(state.updateKnownLeaderAndTerm(peer1, 1));
assertEquals(1, state.currentTerm());
- state.onLeaderElected(leaderNode2, 5);
+ assertTrue(state.updateKnownLeaderAndTerm(peer2, 5));
assertEquals(5, state.currentTerm());
assertEquals(State.LEADER_AVAILABLE, state.currentState());
}
@@ -158,17 +158,19 @@ public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
@Test
void testOnGroupUnavailableTransition() {
LeaderAvailabilityState state = new LeaderAvailabilityState();
- InternalClusterNode leaderNode = createNode("leader-node");
+ Peer peer = new Peer("leader-node");
long expectedTerm = 1;
- state.onLeaderElected(leaderNode, expectedTerm);
+ state.updateKnownLeaderAndTerm(peer, expectedTerm);
assertEquals(State.LEADER_AVAILABLE, state.currentState());
assertEquals(expectedTerm, state.currentTerm());
state.onGroupUnavailable(expectedTerm);
assertEquals(State.WAITING_FOR_LEADER, state.currentState());
assertEquals(expectedTerm, state.currentTerm());
+ // Cached leader is intentionally preserved for retry to old leader as a first guess.
+ assertEquals(peer, state.leader());
}
/**
@@ -177,10 +179,10 @@ public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
@Test
void testOnGroupUnavailableIgnoredIfTermChanged() {
LeaderAvailabilityState state = new LeaderAvailabilityState();
- InternalClusterNode leaderNode = createNode("leader-node");
+ Peer peer = new Peer("leader-node");
- state.onLeaderElected(leaderNode, 1);
- state.onLeaderElected(leaderNode, 2);
+ state.updateKnownLeaderAndTerm(peer, 1);
+ state.updateKnownLeaderAndTerm(peer, 2);
assertEquals(State.LEADER_AVAILABLE, state.currentState());
assertEquals(2, state.currentTerm());
@@ -196,9 +198,9 @@ public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
@Test
void testNewWaitersAfterUnavailable() {
LeaderAvailabilityState state = new LeaderAvailabilityState();
- InternalClusterNode leaderNode = createNode("leader-node");
+ Peer peer = new Peer("leader-node");
- state.onLeaderElected(leaderNode, 1);
+ state.updateKnownLeaderAndTerm(peer, 1);
CompletableFuture<Long> future1 = state.awaitLeader();
assertTrue(future1.isDone());
@@ -212,7 +214,7 @@ public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
@Test
void testMultipleWaitersCompleted() {
LeaderAvailabilityState state = new LeaderAvailabilityState();
- InternalClusterNode leaderNode = createNode("leader-node");
+ Peer peer = new Peer("leader-node");
CompletableFuture<Long> waiter1 = state.awaitLeader();
CompletableFuture<Long> waiter2 = state.awaitLeader();
@@ -222,7 +224,7 @@ public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
assertFalse(waiter2.isDone());
assertFalse(waiter3.isDone());
- state.onLeaderElected(leaderNode, 10);
+ state.updateKnownLeaderAndTerm(peer, 10);
assertTrue(waiter1.isDone());
assertTrue(waiter2.isDone());
@@ -237,11 +239,11 @@ public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
@Test
void testTermZeroAccepted() {
LeaderAvailabilityState state = new LeaderAvailabilityState();
- InternalClusterNode leaderNode = createNode("leader");
+ Peer peer = new Peer("leader");
assertEquals(-1, state.currentTerm());
- state.onLeaderElected(leaderNode, 0);
+ state.updateKnownLeaderAndTerm(peer, 0);
assertEquals(0, state.currentTerm());
assertEquals(State.LEADER_AVAILABLE, state.currentState());
@@ -253,13 +255,13 @@ public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
@Test
void testNegativeTermRejected() {
LeaderAvailabilityState state = new LeaderAvailabilityState();
- InternalClusterNode leaderNode = createNode("leader");
+ Peer peer = new Peer("leader");
assertEquals(-1, state.currentTerm());
IllegalArgumentException thrown = assertThrows(
IllegalArgumentException.class,
- () -> state.onLeaderElected(leaderNode, -1)
+ () -> state.updateKnownLeaderAndTerm(peer, -1)
);
assertEquals("Term must be non-negative: -1", thrown.getMessage());
@@ -281,8 +283,8 @@ public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
threads[i] = new Thread(() -> {
try {
startLatch.await();
- InternalClusterNode leader = createNode("leader-" + term);
- state.onLeaderElected(leader, term);
+ Peer leader = new Peer("leader-" + term);
+ state.updateKnownLeaderAndTerm(leader, term);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} finally {
@@ -374,10 +376,10 @@ public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
@Test
void testStopWhenLeaderAvailable() {
LeaderAvailabilityState state = new LeaderAvailabilityState();
- InternalClusterNode leader = createNode("leader");
+ Peer peer = new Peer("leader");
RuntimeException testException = new RuntimeException("Test shutdown");
- state.onLeaderElected(leader, 1);
+ state.updateKnownLeaderAndTerm(peer, 1);
assertEquals(State.LEADER_AVAILABLE, state.currentState());
state.stop(testException);
@@ -416,20 +418,20 @@ public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
}
/**
- * {@link LeaderAvailabilityState#onLeaderElected(InternalClusterNode, long)} is ignored after stop.
+ * {@link LeaderAvailabilityState#updateKnownLeaderAndTerm(Peer, long)} is ignored after stop and returns false.
*/
@Test
- void testOnLeaderElectedIgnoredAfterStop() {
+ void testUpdateKnownLeaderAndTermIgnoredAfterStop() {
LeaderAvailabilityState state = new LeaderAvailabilityState();
- InternalClusterNode leader = createNode("leader");
+ Peer peer = new Peer("leader");
RuntimeException testException = new RuntimeException("Test shutdown");
CompletableFuture<Long> waiter1 = state.awaitLeader();
state.stop(testException);
assertTrue(waiter1.isCompletedExceptionally());
- // Leader election after stop should be ignored
- state.onLeaderElected(leader, 5);
+ // Leader update after stop should be ignored and return false.
+ assertFalse(state.updateKnownLeaderAndTerm(peer, 5));
// State should still be stopped.
assertTrue(state.stopped());
@@ -446,10 +448,10 @@ public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
@Test
void testOnGroupUnavailableIgnoredAfterStop() {
LeaderAvailabilityState state = new LeaderAvailabilityState();
- InternalClusterNode leader = createNode("leader");
+ Peer peer = new Peer("leader");
RuntimeException testException = new RuntimeException("Test shutdown");
- state.onLeaderElected(leader, 1);
+ state.updateKnownLeaderAndTerm(peer, 1);
assertEquals(State.LEADER_AVAILABLE, state.currentState());
state.stop(testException);
@@ -460,7 +462,52 @@ public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
assertTrue(state.stopped());
}
- private static InternalClusterNode createNode(String name) {
- return new ClusterNodeImpl(deriveUuidFrom(name), name, new NetworkAddress("localhost", 123));
+ /** {@link LeaderAvailabilityState#setCachedLeader(Peer)} updates the cached leader without changing term or state. */
+ @Test
+ void testSetCachedLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ Peer peerA = new Peer("peer-a");
+ Peer peerB = new Peer("peer-b");
+
+ // setLeaderHint works even before any updateKnownLeaderAndTerm call.
+ state.setCachedLeader(peerB);
+ assertEquals(peerB, state.leader());
+
+ // updateKnownLeaderAndTerm overwrites hint.
+ state.updateKnownLeaderAndTerm(peerA, 5);
+ assertEquals(peerA, state.leader());
+
+ // setLeaderHint overwrites updateKnownLeaderAndTerm's peer.
+ state.setCachedLeader(peerB);
+ assertEquals(peerB, state.leader());
+
+ // null clears the leader.
+ state.setCachedLeader(null);
+ assertNull(state.leader());
+
+ // No-op after stop.
+ state.setCachedLeader(peerA);
+ assertEquals(peerA, state.leader());
+ state.stop(new RuntimeException("Test shutdown"));
+
+ state.setCachedLeader(peerB);
+ assertEquals(peerA, state.leader());
+ }
+
+ /** {@code updateKnownLeaderAndTerm(null, term)} updates term but does NOT transition to LEADER_AVAILABLE. */
+ @Test
+ void testUpdateKnownLeaderWithNullPeerDoesNotTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ // Null peer: term should be updated, but state should remain WAITING_FOR_LEADER.
+ assertTrue(state.updateKnownLeaderAndTerm(null, 5));
+
+ assertEquals(5, state.currentTerm());
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertNull(state.leader());
+ assertFalse(waiter.isDone(), "Waiter should NOT be completed when peer is null");
}
}
diff --git a/modules/raft/src/test/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupServiceRunTest.java b/modules/raft/src/test/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupServiceRunTest.java
index 54dee9568e0..d66a4fb44fa 100644
--- a/modules/raft/src/test/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupServiceRunTest.java
+++ b/modules/raft/src/test/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupServiceRunTest.java
@@ -19,7 +19,6 @@ package org.apache.ignite.internal.raft.client;
import static java.util.concurrent.CompletableFuture.completedFuture;
import static java.util.stream.Collectors.toUnmodifiableList;
-import static org.apache.ignite.internal.raft.TestThrottlingContextHolder.throttlingContextHolder;
import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
import static org.apache.ignite.internal.testframework.matchers.CompletableFutureExceptionMatcher.willThrow;
import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willBe;
@@ -27,8 +26,10 @@ import static org.apache.ignite.internal.testframework.matchers.CompletableFutur
import static org.apache.ignite.raft.TestWriteCommand.testWriteCommand;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
@@ -47,6 +48,7 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.ignite.configuration.ConfigurationValue;
@@ -66,6 +68,7 @@ import org.apache.ignite.internal.raft.PeersAndLearners;
import org.apache.ignite.internal.raft.ReplicationGroupUnavailableException;
import org.apache.ignite.internal.raft.StoppingExceptionFactories;
import org.apache.ignite.internal.raft.configuration.RaftConfiguration;
+import org.apache.ignite.internal.raft.service.LeaderWithTerm;
import org.apache.ignite.internal.raft.util.OptimizedMarshaller;
import org.apache.ignite.internal.raft.util.ThreadLocalOptimizedMarshaller;
import org.apache.ignite.internal.replicator.TestReplicationGroupId;
@@ -84,6 +87,9 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.Mockito;
@@ -207,7 +213,6 @@ public class PhysicalTopologyAwareRaftGroupServiceRunTest extends BaseIgniteAbst
eventsClientListener,
commandsMarshaller,
StoppingExceptionFactories.indicateNodeStop(),
- throttlingContextHolder(),
NOOP_FAILURE_PROCESSOR
);
@@ -294,20 +299,18 @@ public class PhysicalTopologyAwareRaftGroupServiceRunTest extends BaseIgniteAbst
}
/**
- * Tests that with timeout=0, the command succeeds immediately when leader is available.
+ * Tests that command succeeds when leader is available, regardless of timeout value.
*/
- @Test
- void testZeroTimeoutSuccessWhenLeaderAvailable() {
+ @ParameterizedTest
+ @ValueSource(longs = {0, 5000, Long.MAX_VALUE, -1})
+ void testSuccessWhenLeaderAvailable(long timeout) {
mockLeaderRequest();
mockUserInputSuccess();
PhysicalTopologyAwareRaftGroupService svc = startService();
-
- // Simulate leader election and wait so the service knows about the leader.
simulateLeaderElectionAndWait(NODES.get(0), CURRENT_TERM);
- // With timeout=0, should succeed immediately since leader is available.
- CompletableFuture<Object> result = svc.run(testWriteCommand(), 0);
+ CompletableFuture<Object> result = svc.run(testWriteCommand(), timeout);
assertThat(result, willBe(instanceOf(TestResponse.class)));
}
@@ -330,24 +333,6 @@ public class PhysicalTopologyAwareRaftGroupServiceRunTest extends BaseIgniteAbst
verifyExact3PeersCalled();
}
- /**
- * Tests that with Long.MAX_VALUE timeout, the command succeeds when leader is available immediately.
- */
- @Test
- void testInfiniteTimeoutSuccessWhenLeaderAvailable() {
- mockLeaderRequest();
- mockUserInputSuccess();
-
- PhysicalTopologyAwareRaftGroupService svc = startService();
-
- // Simulate leader election and wait for it to be processed.
- simulateLeaderElectionAndWait(NODES.get(0), CURRENT_TERM);
-
- CompletableFuture<Object> result = svc.run(testWriteCommand(), Long.MAX_VALUE);
-
- assertThat(result, willBe(instanceOf(TestResponse.class)));
- }
-
/**
* Tests that with Long.MAX_VALUE timeout, all peers are tried first, then waits for leader, and succeeds when leader appears.
*/
@@ -391,43 +376,6 @@ public class PhysicalTopologyAwareRaftGroupServiceRunTest extends BaseIgniteAbst
assertThat(result, willCompleteSuccessfully());
}
- /**
- * Tests that negative timeout is treated as infinite wait.
- */
- @Test
- void testNegativeTimeoutTreatedAsInfinite() {
- mockLeaderRequest();
- mockUserInputSuccess();
-
- PhysicalTopologyAwareRaftGroupService svc = startService();
-
- // Simulate leader election and wait for it to be processed.
- simulateLeaderElectionAndWait(NODES.get(0), CURRENT_TERM);
-
- // Negative timeout should be treated as infinite.
- CompletableFuture<Object> result = svc.run(testWriteCommand(), -1);
-
- assertThat(result, willBe(instanceOf(TestResponse.class)));
- }
-
- /**
- * Tests that with bounded timeout, the command succeeds within timeout when leader is available.
- */
- @Test
- void testBoundedTimeoutSuccessWithinTimeout() {
- mockLeaderRequest();
- mockUserInputSuccess();
-
- PhysicalTopologyAwareRaftGroupService svc = startService();
-
- // Simulate leader election and wait for it to be processed.
- simulateLeaderElectionAndWait(NODES.get(0), CURRENT_TERM);
-
- CompletableFuture<Object> result = svc.run(testWriteCommand(), 5000);
-
- assertThat(result, willBe(instanceOf(TestResponse.class)));
- }
-
/**
* Tests that with bounded timeout, ReplicationGroupUnavailableException is thrown when timeout expires waiting for leader.
*/
@@ -807,8 +755,10 @@ public class PhysicalTopologyAwareRaftGroupServiceRunTest extends BaseIgniteAbst
*/
@Test
void testInfiniteTimeoutWithPreviousLeaderTriesPeersOnceBeforeWaiting() throws Exception {
+ int peerCount = NODES.size();
+
AtomicInteger callCount = new AtomicInteger(0);
- CountDownLatch allPeersTried = new CountDownLatch(3);
+ CountDownLatch allPeersTried = new CountDownLatch(peerCount);
// This latch will be counted down if more than 3 calls are made (indicating the bug).
CountDownLatch extraCallsMade = new CountDownLatch(1);
@@ -819,7 +769,7 @@ public class PhysicalTopologyAwareRaftGroupServiceRunTest extends BaseIgniteAbst
anyLong()
)).thenAnswer(invocation -> {
int count = callCount.incrementAndGet();
- if (count <= 3) {
+ if (count <= peerCount) {
allPeersTried.countDown();
} else {
// More than 3 calls means the bug exists - extra retry cycle happened.
@@ -857,7 +807,7 @@ public class PhysicalTopologyAwareRaftGroupServiceRunTest extends BaseIgniteAbst
+ ". If 6+ calls were made, the term was incorrectly passed as -1 "
+ "causing an extra retry cycle before proper waiting.",
extraCallsHappened, is(false));
- assertThat(callCount.get(), is(3));
+ assertThat(callCount.get(), is(peerCount));
}
private void verifyExact3PeersCalled() {
@@ -880,55 +830,231 @@ public class PhysicalTopologyAwareRaftGroupServiceRunTest extends BaseIgniteAbst
assertThat(triedPeers, equalTo(expectedPeers));
}
- private static class TestResponse {
+ /**
+ * Tests that for GetLeaderRequest with UNKNOWN/EINTERNAL/ENOENT errors, the executor tries another peer
+ * instead of retrying the same peer. This matches RaftGroupServiceImpl behavior.
+ *
+ * <p>The test verifies that after receiving one of these errors from a peer, the next request goes to
+ * a DIFFERENT peer, not the same one.
+ */
+ @ParameterizedTest
+ @EnumSource(names = {"UNKNOWN", "EINTERNAL", "ENOENT"})
+ void testGetLeaderRequestTriesDifferentPeerOnTransientError(RaftError error) throws Exception {
+ Set<String> calledPeers = ConcurrentHashMap.newKeySet();
+ AtomicInteger callCount = new AtomicInteger(0);
+
+ when(messagingService.invoke(
+ any(InternalClusterNode.class),
+ any(GetLeaderRequest.class),
+ anyLong())
+ ).thenAnswer(invocation -> {
+ InternalClusterNode target = invocation.getArgument(0);
+ calledPeers.add(target.name());
+ int count = callCount.incrementAndGet();
+
+ if (count == 1) {
+ // First call returns transient error.
+ return completedFuture(FACTORY.errorResponse()
+ .errorCode(error.getNumber())
+ .build());
+ }
+
+ // Second call succeeds.
+ return completedFuture(FACTORY.getLeaderResponse()
+ .leaderId(PeerId.fromPeer(NODES.get(0)).toString())
+ .currentTerm(CURRENT_TERM)
+ .build());
+ });
+
+ PhysicalTopologyAwareRaftGroupService svc = startService();
+
+ CompletableFuture<LeaderWithTerm> result = svc.refreshAndGetLeaderWithTerm(TIMEOUT);
+
+ assertThat(result, willCompleteSuccessfully());
+
+ // Verify that at least 2 different peers were called.
+ // If the same peer was retried, calledPeers would have size 1.
+ assertThat("Should try different peers on " + error + " error, but called peers were: " + calledPeers,
+ calledPeers.size(), greaterThan(1));
+ }
+
+ /**
+ * Tests that RANDOM strategy with bounded timeout keeps cycling through peers until timeout.
+ *
+ * <p>When all peers return EPERM (no leader), RaftGroupServiceImpl resets unavailable peers
+ * and keeps cycling until timeout. RaftCommandExecutor should do the same for RANDOM strategy.
+ */
+ @Test
+ void testRandomStrategyWithBoundedTimeoutKeepsCyclingUntilTimeout() throws Exception {
+ int peerCount = NODES.size();
+
+ AtomicInteger callCount = new AtomicInteger(0);
+ CountDownLatch sixCallsReached = new CountDownLatch(2 * peerCount);
+
+ // All peers return EPERM (no leader) every time.
+ when(messagingService.invoke(
+ any(InternalClusterNode.class),
+ any(GetLeaderRequest.class),
+ anyLong())
+ ).thenAnswer(invocation -> {
+ callCount.incrementAndGet();
+ sixCallsReached.countDown();
+ return completedFuture(FACTORY.errorResponse()
+ .errorCode(RaftError.EPERM.getNumber())
+ .leaderId(null)
+ .build());
+ });
+
+ PhysicalTopologyAwareRaftGroupService svc = startService();
+
+ // With bounded timeout (500ms), should keep cycling until timeout.
+ // With 3 peers and 50ms retry delay, we should get more than 3 calls before timeout.
+ CompletableFuture<Void> result = svc.refreshLeader(500);
+
+ // Wait for at least 6 calls (2 complete cycles through all 3 peers).
+ boolean sixCallsHappened = sixCallsReached.await(2, TimeUnit.SECONDS);
+
+ assertThat(result, willThrow(ReplicationGroupUnavailableException.class, 2, TimeUnit.SECONDS));
+
+ // Verify more than 3 calls were made (cycling through peers multiple times).
+ assertThat("Should cycle through peers multiple times before timeout, but only " + callCount.get() + " calls were made",
+ sixCallsHappened, is(true));
+
+ assertThat(callCount.get(), greaterThan(peerCount));
+ }
+
+ /**
+ * Tests that the timeout parameter is respected for refreshLeader/refreshAndGetLeaderWithTerm.
+ * With a short timeout, the request should fail fast, not wait for the default response timeout.
+ */
+ @Test
+ void testTimeoutParameterIsRespectedForRefreshLeader() {
+ // All peers never respond (simulating network issue).
+ when(messagingService.invoke(
+ any(InternalClusterNode.class),
+ any(GetLeaderRequest.class),
+ anyLong())
+ ).thenReturn(new CompletableFuture<>()); // Never completes
+
+ PhysicalTopologyAwareRaftGroupService svc = startService();
+
+ long startTime = System.currentTimeMillis();
+
+ // With 200ms timeout, should fail within ~300ms, not the default 3000ms.
+ CompletableFuture<Void> result = svc.refreshLeader(200);
+
+ assertThat(result, willThrow(ReplicationGroupUnavailableException.class, 1, TimeUnit.SECONDS));
+
+ long elapsed = System.currentTimeMillis() - startTime;
+ assertThat("Expected to fail within 1000ms but took " + elapsed + "ms", elapsed < 1000, is(true));
}
/**
- * Tests single-attempt mode (timeout=0) with 5 nodes: all return "no leader".
+ * Tests that RANDOM strategy with timeout=0 (single attempt) tries all peers once and fails.
*
- * <p>In single-attempt mode, "no leader" is treated same as unavailable.
- * Each peer is tried exactly once, then fails with ReplicationGroupUnavailableException.
+ * <p>With timeout=0, each peer should be tried at most once, then fail with ReplicationGroupUnavailableException.
*/
@Test
- void testSingleAttemptModeWithAllNoLeader() {
+ void testRandomStrategySingleAttemptTriesAllPeersOnce() throws Exception {
+ int peerCount = NODES.size();
+
AtomicInteger callCount = new AtomicInteger(0);
Set<String> calledPeers = ConcurrentHashMap.newKeySet();
- // All peers return EPERM with no leader.
+ CountDownLatch constructorCallsDone = new CountDownLatch(peerCount); // Constructor tries all 3 peers
+
+ // All peers return EPERM (no leader).
when(messagingService.invoke(
any(InternalClusterNode.class),
- argThat(this::isTestWriteCommand),
- anyLong()
- )).thenAnswer(invocation -> {
+ any(GetLeaderRequest.class),
+ anyLong())
+ ).thenAnswer(invocation -> {
InternalClusterNode target = invocation.getArgument(0);
calledPeers.add(target.name());
- callCount.incrementAndGet();
+ int count = callCount.incrementAndGet();
+ // Signal when constructor's calls complete (first 3 calls).
+ if (count <= peerCount) {
+ constructorCallsDone.countDown();
+ }
return completedFuture(FACTORY.errorResponse()
.errorCode(RaftError.EPERM.getNumber())
.leaderId(null)
.build());
});
- PhysicalTopologyAwareRaftGroupService svc = startService(FIVE_NODES);
+ PhysicalTopologyAwareRaftGroupService svc = startService();
+
+ // Wait for constructor's refreshAndGetLeaderWithTerm() to try all peers.
+ assertTrue(constructorCallsDone.await(5, TimeUnit.SECONDS),
+ "Constructor's refreshAndGetLeaderWithTerm should try all peers");
+
+ // Reset counters to measure only the test call.
+ int callsBeforeTest = callCount.get();
+ calledPeers.clear();
+ callCount.set(0);
// With timeout=0, should try each peer once and fail.
- CompletableFuture<Object> result = svc.run(testWriteCommand(), 0);
+ CompletableFuture<Void> result = svc.refreshLeader(0);
- assertThat(result, willThrow(ReplicationGroupUnavailableException.class));
+ assertThat(result, willThrow(ReplicationGroupUnavailableException.class, 1, TimeUnit.SECONDS));
- // Verify each peer was tried exactly once.
- assertThat("Should call exactly 5 peers", callCount.get(), is(5));
- assertThat("Should call all 5 unique peers", calledPeers.size(), is(5));
+ // Verify each peer was tried exactly once for this call.
+ assertThat("Should call exactly 3 peers, but got " + callCount.get() + " (calls before test: " + callsBeforeTest + ")",
+ callCount.get(), is(peerCount));
+ assertThat("Should call all 3 unique peers", calledPeers.size(), is(peerCount));
}
/**
- * Tests single-attempt mode (timeout=0) with 5 nodes: 3 return "no leader", 2 return EHOSTDOWN.
+ * Tests that RANDOM strategy succeeds when one peer returns the leader after other peers fail.
+ */
+ @Test
+ void testRandomStrategySucceedsWhenOnePeerReturnsLeader() {
+ AtomicInteger callCount = new AtomicInteger(0);
+
+ when(messagingService.invoke(
+ any(InternalClusterNode.class),
+ any(GetLeaderRequest.class),
+ anyLong())
+ ).thenAnswer(invocation -> {
+ int count = callCount.incrementAndGet();
+
+ // First 2 calls return EPERM (no leader), third succeeds.
+ if (count <= 2) {
+ return completedFuture(FACTORY.errorResponse()
+ .errorCode(RaftError.EPERM.getNumber())
+ .leaderId(null)
+ .build());
+ }
+
+ return completedFuture(FACTORY.getLeaderResponse()
+ .leaderId(PeerId.fromPeer(NODES.get(0)).toString())
+ .currentTerm(CURRENT_TERM)
+ .build());
+ });
+
+ PhysicalTopologyAwareRaftGroupService svc = startService();
+
+ CompletableFuture<LeaderWithTerm> result = svc.refreshAndGetLeaderWithTerm(TIMEOUT);
+
+ assertThat(result, willCompleteSuccessfully());
+
+ LeaderWithTerm leaderWithTerm = result.join();
+ assertThat(leaderWithTerm.leader().consistentId(), is(NODES.get(0).consistentId()));
+ assertThat(leaderWithTerm.term(), is(CURRENT_TERM));
+ }
+
+ private static class TestResponse {
+ }
+
+ /**
+ * Tests single-attempt mode (timeout=0) with 5 nodes and mixed errors.
*
* <p>In single-attempt mode, all errors are treated the same - each peer is tried once.
* The request should fail after all 5 peers are tried.
*/
@Test
void testSingleAttemptModeWithMixedErrors() {
+ int peerCount = FIVE_NODES.size();
AtomicInteger callCount = new AtomicInteger(0);
Set<String> calledPeers = ConcurrentHashMap.newKeySet();
@@ -961,8 +1087,8 @@ public class PhysicalTopologyAwareRaftGroupServiceRunTest extends BaseIgniteAbst
assertThat(result, willThrow(ReplicationGroupUnavailableException.class));
// Verify each peer was tried exactly once.
- assertThat("Should call exactly 5 peers", callCount.get(), is(5));
- assertThat("Should call all 5 unique peers", calledPeers.size(), is(5));
+ assertThat("Should call exactly 5 peers", callCount.get(), is(peerCount));
+ assertThat("Should call all 5 unique peers", calledPeers.size(), is(peerCount));
}
/**
@@ -973,9 +1099,11 @@ public class PhysicalTopologyAwareRaftGroupServiceRunTest extends BaseIgniteAbst
*/
@Test
void testLeaderWaitModeRetriesNoLeaderPeersAfterLeaderElection() throws Exception {
+ int peerCount = NODES.size();
+
List<String> calledPeers = new CopyOnWriteArrayList<>();
AtomicInteger noLeaderResponseCount = new AtomicInteger(0);
- CountDownLatch allPeersTriedOnce = new CountDownLatch(3);
+ CountDownLatch allPeersTriedOnce = new CountDownLatch(peerCount);
when(messagingService.invoke(
any(InternalClusterNode.class),
@@ -986,7 +1114,7 @@ public class PhysicalTopologyAwareRaftGroupServiceRunTest extends BaseIgniteAbst
calledPeers.add(target.name());
// First round: all 3 peers return "no leader".
- if (noLeaderResponseCount.get() < 3) {
+ if (noLeaderResponseCount.get() < peerCount) {
noLeaderResponseCount.incrementAndGet();
allPeersTriedOnce.countDown();
return completedFuture(FACTORY.errorResponse()
@@ -1018,7 +1146,42 @@ public class PhysicalTopologyAwareRaftGroupServiceRunTest extends BaseIgniteAbst
// Verify that at least one peer was called twice (first with "no leader", then success).
long totalCalls = calledPeers.size();
- assertTrue(totalCalls > 3, "Should have more than 3 calls (some peers retried after leader election), got " + totalCalls);
+ assertTrue(totalCalls > peerCount, "Should have more than 3 calls (some peers retried after leader election), got " + totalCalls);
+ }
+
+ /**
+ * Tests that after leader election notification, the next request goes directly to the notified leader.
+ *
+ * <p>This verifies that the leader field is updated when leader election notification is received,
+ * so subsequent requests don't waste time trying random peers.
+ */
+ @Test
+ void testLeaderElectionNotificationUpdatesLeaderField() {
+ Peer expectedLeader = NODES.get(1); // Use node 1 as the leader (not node 0)
+ AtomicReference<String> firstCalledPeer = new AtomicReference<>();
+
+ when(messagingService.invoke(
+ any(InternalClusterNode.class),
+ argThat(this::isTestWriteCommand),
+ anyLong()
+ )).thenAnswer(invocation -> {
+ InternalClusterNode target = invocation.getArgument(0);
+ firstCalledPeer.compareAndSet(null, target.name());
+ return completedFuture(FACTORY.actionResponse().result(new TestResponse()).build());
+ });
+
+ PhysicalTopologyAwareRaftGroupService svc = startService();
+
+ // Simulate leader election notification for node 1.
+ simulateLeaderElectionAndWait(expectedLeader, CURRENT_TERM);
+
+ // Now run a command - it should go directly to the notified leader.
+ CompletableFuture<Object> result = svc.run(testWriteCommand(), Long.MAX_VALUE);
+ assertThat(result, willCompleteSuccessfully());
+
+ // Verify the first (and only) call went to the notified leader.
+ assertEquals(expectedLeader.consistentId(), firstCalledPeer.get(),
+ "After leader election notification, request should go directly to the notified leader");
}
/**
@@ -1057,4 +1220,76 @@ public class PhysicalTopologyAwareRaftGroupServiceRunTest extends BaseIgniteAbst
assertThat("Should call at most 3 peers, but got " + callCount.get(), callCount.get(), is(3));
assertThat("Should call all 3 unique peers", calledPeers.size(), is(3));
}
+
+ /**
+ * Tests that refreshAndGetLeaderWithTerm correctly tracks term progression.
+ *
+ * <p>Scenario:
+ * <ol>
+ * <li>Leader election sets term=2, leader=node0</li>
+ * <li>refreshAndGetLeaderWithTerm returns term=5, leader=node1 → update cached leader</li>
+ * <li>refreshAndGetLeaderWithTerm returns stale term=4, leader=node2 → should NOT update</li>
+ * </ol>
+ *
+ * <p>This tests that the fix tracks the highest term seen across all refreshAndGetLeaderWithTerm calls,
+ * not just the term from the last leader election notification.
+ */
+ @Test
+ void testRefreshAndGetLeaderWithTermTracksTermProgressionAcrossMultipleCalls() throws Exception {
+ Peer initialLeader = NODES.get(0);
+ Peer newerLeader = NODES.get(1);
+ Peer staleLeader = NODES.get(2);
+
+ // Track call count and signal when constructor's call completes.
+ AtomicInteger callCount = new AtomicInteger(0);
+ CountDownLatch constructorCallDone = new CountDownLatch(1);
+
+ when(messagingService.invoke(
+ any(InternalClusterNode.class),
+ any(GetLeaderRequest.class),
+ anyLong())
+ ).thenAnswer(invocation -> {
+ int count = callCount.incrementAndGet();
+ if (count == 1) {
+ // Constructor's call - return low term and signal completion.
+ return completedFuture(FACTORY.getLeaderResponse()
+ .leaderId(PeerId.fromPeer(initialLeader).toString())
+ .currentTerm(1)
+ .build())
+ .whenComplete((r, e) -> constructorCallDone.countDown());
+ } else if (count == 2) {
+ // First test call - term=5 (new leader)
+ return completedFuture(FACTORY.getLeaderResponse()
+ .leaderId(PeerId.fromPeer(newerLeader).toString())
+ .currentTerm(5)
+ .build());
+ }
+ // Subsequent calls - stale term=4 (should NOT update)
+ return completedFuture(FACTORY.getLeaderResponse()
+ .leaderId(PeerId.fromPeer(staleLeader).toString())
+ .currentTerm(4)
+ .build());
+ });
+
+ PhysicalTopologyAwareRaftGroupService svc = startService();
+
+ // Wait for constructor's call to complete.
+ assertTrue(constructorCallDone.await(5, TimeUnit.SECONDS),
+ "Constructor's refreshAndGetLeaderWithTerm should complete");
+
+ // Set initial leader via leader election to establish a known baseline.
+ // This sets term=2 in leaderAvailabilityState via leaderElectionListener.
+ simulateLeaderElectionAndWait(initialLeader, 2);
+ assertThat(svc.leader().consistentId(), is(initialLeader.consistentId()));
+
+ // First call: term=5, leader=node1 (new leader, newer term)
+ assertThat(svc.refreshAndGetLeaderWithTerm(TIMEOUT), willCompleteSuccessfully());
+ assertThat("After term=5, leader should be node1",
+ svc.leader().consistentId(), is(newerLeader.consistentId()));
+
+ // Second call: stale term=4, leader=node2 (should NOT update because 4 < 5)
+ assertThat(svc.refreshAndGetLeaderWithTerm(TIMEOUT), willCompleteSuccessfully());
+ assertThat("After stale term=4, leader should still be node1 (not overwritten with stale info)",
+ svc.leader().consistentId(), is(newerLeader.consistentId()));
+ }
}
diff --git a/modules/raft/src/test/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupServiceTest.java b/modules/raft/src/test/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupServiceTest.java
index c92d6b795dd..803862177d9 100644
--- a/modules/raft/src/test/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupServiceTest.java
+++ b/modules/raft/src/test/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupServiceTest.java
@@ -20,7 +20,6 @@ package org.apache.ignite.internal.raft.client;
import static java.util.stream.Collectors.toSet;
import static org.apache.ignite.internal.network.utils.ClusterServiceTestUtils.clusterService;
import static org.apache.ignite.internal.network.utils.ClusterServiceTestUtils.findLocalAddresses;
-import static org.apache.ignite.internal.raft.TestThrottlingContextHolder.throttlingContextHolder;
import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully;
import static org.apache.ignite.internal.util.IgniteUtils.stopAsync;
import static org.awaitility.Awaitility.await;
@@ -72,7 +71,6 @@ import org.apache.ignite.internal.util.CollectionUtils;
import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.network.NetworkAddress;
import org.apache.ignite.raft.jraft.RaftMessageGroup;
-import org.apache.ignite.raft.jraft.RaftMessagesFactory;
import org.apache.ignite.raft.jraft.option.NodeOptions;
import org.apache.ignite.raft.jraft.rpc.CliRequests.LeaderChangeNotification;
import org.apache.ignite.raft.jraft.rpc.impl.RaftGroupEventsClientListener;
@@ -87,9 +85,6 @@ import org.junit.jupiter.api.extension.ExtendWith;
*/
@ExtendWith(ConfigurationExtension.class)
public class PhysicalTopologyAwareRaftGroupServiceTest extends IgniteAbstractTest {
- /** RAFT message factory. */
- private static final RaftMessagesFactory FACTORY = new RaftMessagesFactory();
-
/** Base node port. */
private static final int PORT_BASE = 1234;
@@ -116,9 +111,9 @@ public class PhysicalTopologyAwareRaftGroupServiceTest extends IgniteAbstractTes
@AfterEach
public void afterTest() throws Exception {
- IgniteUtils.shutdownAndAwaitTermination(executor, 10, TimeUnit.SECONDS);
-
stopCluster();
+
+ IgniteUtils.shutdownAndAwaitTermination(executor, 10, TimeUnit.SECONDS);
}
@Test
@@ -191,7 +186,7 @@ public class PhysicalTopologyAwareRaftGroupServiceTest extends IgniteAbstractTes
assertNotNull(firstRaftClient);
- assertThat(firstRaftClient.refreshLeader(), willCompleteSuccessfully());
+ assertThat(firstRaftClient.refreshLeader(Long.MAX_VALUE), willCompleteSuccessfully());
// Start client service for the second client.
int clientPort = PORT_BASE + nodes + 1;
@@ -285,7 +280,7 @@ public class PhysicalTopologyAwareRaftGroupServiceTest extends IgniteAbstractTes
log.info("New Leader: " + secondLeaderRef.get());
- secondRaftClient.refreshLeader().get();
+ secondRaftClient.refreshLeader(Long.MAX_VALUE).get();
assertEquals(secondRaftClient.leader().consistentId(), secondLeaderRef.get().name());
}
@@ -311,7 +306,7 @@ public class PhysicalTopologyAwareRaftGroupServiceTest extends IgniteAbstractTes
log.info("Peer to transfer leader: " + newLeaderPeer);
- raftClient.transferLeadership(newLeaderPeer).get();
+ raftClient.transferLeadership(newLeaderPeer, Long.MAX_VALUE).get();
String leaderId = newLeaderPeer.consistentId();
@@ -321,7 +316,7 @@ public class PhysicalTopologyAwareRaftGroupServiceTest extends IgniteAbstractTes
log.info("New Leader: " + leaderRef.get());
- raftClient.refreshLeader().get();
+ raftClient.refreshLeader(Long.MAX_VALUE).get();
assertEquals(raftClient.leader().consistentId(), leaderRef.get().name());
}
@@ -485,7 +480,6 @@ public class PhysicalTopologyAwareRaftGroupServiceTest extends IgniteAbstractTes
eventsClientListener,
commandsMarshaller,
StoppingExceptionFactories.indicateComponentStop(),
- throttlingContextHolder(),
NOOP_FAILURE_PROCESSOR
);
}
diff --git a/modules/table/src/integrationTest/java/org/apache/ignite/internal/table/distributed/disaster/ItHighAvailablePartitionsRecoveryByFilterUpdateTest.java b/modules/table/src/integrationTest/java/org/apache/ignite/internal/table/distributed/disaster/ItHighAvailablePartitionsRecoveryByFilterUpdateTest.java
index 83eb69d7a88..94eb712c079 100644
--- a/modules/table/src/integrationTest/java/org/apache/ignite/internal/table/distributed/disaster/ItHighAvailablePartitionsRecoveryByFilterUpdateTest.java
+++ b/modules/table/src/integrationTest/java/org/apache/ignite/internal/table/distributed/disaster/ItHighAvailablePartitionsRecoveryByFilterUpdateTest.java
@@ -258,7 +258,8 @@ public class ItHighAvailablePartitionsRecoveryByFilterUpdateTest extends Abstrac
}
String globalFilter = "$[?(@.zone == \"custom\")]";
- createHaZoneWithTable(globalFilter, nodeNames(1, 2, 3, 4, 5, 6, 7));
+ // Explicit quorumSize=4 to match majority semantics (replicas/2+1) so that stopping 4 of 7 nodes triggers HA recovery.
+ createHaZoneWithTables(HA_ZONE_NAME, PARTITIONS_NUMBER, globalFilter, 4, List.of(HA_TABLE_NAME), nodeNames(1, 2, 3, 4, 5, 6, 7));
IgniteImpl node0 = igniteImpl(0);
Table table = node0.tables().table(HA_TABLE_NAME);
@@ -318,7 +319,8 @@ public class ItHighAvailablePartitionsRecoveryByFilterUpdateTest extends Abstrac
}
String globalFilter = "$[?(@.zone == \"custom\")]";
- createHaZoneWithTable(globalFilter, nodeNames(1, 2, 3, 4, 5, 6, 7));
+ // Explicit quorumSize=4 to match majority semantics (replicas/2+1) so that stopping 4 of 7 nodes triggers HA recovery.
+ createHaZoneWithTables(HA_ZONE_NAME, PARTITIONS_NUMBER, globalFilter, 4, List.of(HA_TABLE_NAME), nodeNames(1, 2, 3, 4, 5, 6, 7));
IgniteImpl node0 = igniteImpl(0);
Table table = node0.tables().table(HA_TABLE_NAME);
diff --git a/modules/table/src/integrationTest/java/org/apache/ignite/internal/table/distributed/disaster/ItHighAvailablePartitionsRecoveryTest.java b/modules/table/src/integrationTest/java/org/apache/ignite/internal/table/distributed/disaster/ItHighAvailablePartitionsRecoveryTest.java
index 499612fb46c..19631dd7ef6 100644
--- a/modules/table/src/integrationTest/java/org/apache/ignite/internal/table/distributed/disaster/ItHighAvailablePartitionsRecoveryTest.java
+++ b/modules/table/src/integrationTest/java/org/apache/ignite/internal/table/distributed/disaster/ItHighAvailablePartitionsRecoveryTest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.table.distributed.disaster;
import static java.lang.String.format;
import static org.apache.ignite.internal.ConfigTemplates.FAST_FAILURE_DETECTION_NODE_BOOTSTRAP_CFG_TEMPLATE;
import static org.apache.ignite.internal.TestWrappers.unwrapIgniteImpl;
+import static org.apache.ignite.internal.catalog.commands.CatalogUtils.DEFAULT_FILTER;
import static org.apache.ignite.internal.catalog.commands.CatalogUtils.IMMEDIATE_TIMER_VALUE;
import static org.apache.ignite.internal.catalog.commands.CatalogUtils.INFINITE_TIMER_VALUE;
import static org.apache.ignite.internal.distributionzones.DistributionZonesTestUtil.alterZone;
@@ -397,7 +398,10 @@ public class ItHighAvailablePartitionsRecoveryTest extends AbstractHighAvailable
startNode(5);
startNode(6);
- createHaZoneWithTable();
+ Set<String> allNodes = runningNodes().map(Ignite::name).collect(Collectors.toUnmodifiableSet());
+
+ // Explicit quorumSize=4 to match majority semantics (replicas/2+1) so that stopping 4 of 7 nodes triggers HA recovery.
+ createHaZoneWithTables(HA_ZONE_NAME, PARTITIONS_NUMBER, DEFAULT_FILTER, 4, List.of(HA_TABLE_NAME), allNodes);
IgniteImpl node = igniteImpl(0);
Table table = node.tables().table(HA_TABLE_NAME);
@@ -407,8 +411,6 @@ public class ItHighAvailablePartitionsRecoveryTest extends AbstractHighAvailable
changePartitionDistributionTimeout(node, (int) TimeUnit.MINUTES.toSeconds(5));
- Set<String> allNodes = runningNodes().map(Ignite::name).collect(Collectors.toUnmodifiableSet());
-
waitAndAssertStableAssignmentsOfPartitionEqualTo(node, HA_TABLE_NAME, PARTITION_IDS, allNodes);
assertValuesPresentOnNodes(node.clock().now(), table, 0, 1, 2, 3, 4, 5, 6);
@@ -499,7 +501,9 @@ public class ItHighAvailablePartitionsRecoveryTest extends AbstractHighAvailable
startNode(5);
startNode(6);
- createHaZoneWithTable();
+ // Explicit quorumSize=4 to match majority semantics (replicas/2+1) so that stopping 4 of 7 nodes triggers HA recovery.
+ Set<String> allNodes = runningNodes().map(Ignite::name).collect(Collectors.toUnmodifiableSet());
+ createHaZoneWithTables(HA_ZONE_NAME, PARTITIONS_NUMBER, DEFAULT_FILTER, 4, List.of(HA_TABLE_NAME), allNodes);
IgniteImpl node0 = igniteImpl(0);
Table table = node0.tables().table(HA_TABLE_NAME);